basketballcantho/remote-sensing
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

refactor: reorganize project structure by moving core modules and update import paths in API server

Browse Source
This commit is contained in:
Victor Phan
2026-07-18 01:24:30 +07:00
parent abab846884
commit a82b2f6fa5
155 changed files with 25 additions and 370 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.temp/01.train_ODC.ipynb
+4430
View File
File diff suppressed because one or more lines are too long
.temp/01.train_ODC.py
+233
View File
File diff suppressed because one or more lines are too long
.temp/01.train_ODC_XGBoost.ipynb
+5622
View File
File diff suppressed because one or more lines are too long
.temp/01.train_ODC_XGBoost.py
+171
View File
@@ -0,0 +1,171 @@
#!/usr/bin/env python
# coding: utf-8
# In[1]:
get_ipython().run_cell_magic('time', '', '%matplotlib inline\n\nimport importlib\nimport new_import_ODC \n\nimportlib.reload(new_import_ODC)\n\nfrom new_import_ODC import *\n')
# In[2]:
get_ipython().run_cell_magic('time', '', '# Cấu hình Daskgateway\ncluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1, 10))\n# Khai báo 1 Datacube là dc\ndc = None\n\n# Cấu hình truy cập dịch vụ S3\nconfigure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n\nclient\n')
# In[3]:
## cấu hình thời gian lấy ảnh và tọa độ
date_range = ("2022-09-01", "2022-10-01")
longtitude_range = (105.86, 105.94)
latitude_range = (9.65, 9.69)
coordinates = (longtitude_range, latitude_range)
# In[4]:
## truy vấn ảnh vệ tinh sen2
data = load_data(None, date_range, longtitude_range, latitude_range)
notebook_utils.heading(notebook_utils.xarray_object_size(data))
display(data)
# In[5]:
get_ipython().run_cell_magic('time', '', '# Tiến hành loại bỏ các vị trí bị mây ảnh hưởng\nresult = mask_clean(data)\n# progress(result)\n')
# In[6]:
# Tiến hành tính toán NDVI
ds1 = calculate_indices(result, index="NDVI", satellite_mission="s2")
ndvi = ds1["NDVI"]
display(ndvi)
# In[7]:
## Hiển thị ảnh NDVI chưa điền các giá trị mây (chưa fill nan)
plt.imshow(ndvi.isel(time=0))
# In[8]:
# Thiết lập giá trị trung bình mùa vụ để xử lý các điểm ảnh bị mây dựa vào sự thay đổi theo mùa
time_split = [
slice("2022-09-01", "2023-01-01"),
slice("2023-01-01", "2023-05-01"),
slice("2023-05-01", "2023-07-01"),
slice("2023-07-01", "2022-10-01"),
]
# Điền mây ở các vị trí mang giá trị nan (fill nan)
fill_nan_ndvi = fill_nan(ndvi, time_split)
# In kết quả ảnh NDVI đã điền mây (đã fill nan)
plt.imshow(fill_nan_ndvi.isel(time=0))
# In[9]:
get_ipython().run_cell_magic('time', '', '## tính ndvi theo tháng\naverage_ndvi = fill_nan_ndvi.resample(time="1M").mean().persist()\n# progress(average_ndvi)\n\n# compute average_ndvi\naverage_ndvi = average_ndvi.compute()\n')
# In[10]:
#Load dữ liệu ảnh Sentinel 1
dsvh, dsvv = load_data_sen1(None, date_range, coordinates)
average_vv = calculate_average(dsvv, time_pattern='1M')
average_vh = calculate_average(dsvh, time_pattern='1M')
# In[11]:
## cấu hình bộ dữ liệu điểm huấn luyện mô hình (train file)
train_path = "train/ST_training_data_updated_1130points_new.shp" # đường dẫn shp file train
## load dữ liệu điểm huấn luyện mô hình (train file)
train = load_train_data(train_path)
train.head()
# cấu hình nhãn dữ liệu
label_mapping = {
"Lua tom": "0",
"Lua": "1",
"CHN": "2",
"CLN": "3",
"TS": "4",
"Song": "5",
"Dat xay dung": "6",
"Rung": "7",
}
# xây dựng tập dữ liệu (dataset) chứa dữ liệu VH, VV, NDVI
datasets = get_data_sen1_and_sen2(train, average_ndvi, average_vh, average_vv)
# chia tập dữ liệu thành các phần theo tỉ lệ 80(80-20)-20 tương ứng với tập train, validate, test
X_train, X_val, X_test, y_train, y_val, y_test = split_train_data(
train, label_mapping, datasets
)
# In[ ]:
get_ipython().run_cell_magic('time', '', '# Import XGBoost\nimport xgboost as xgb\nfrom sklearn.metrics import accuracy_score\nimport numpy as np\n\n# Convert to numpy arrays\nX_train_np = np.asarray(X_train, dtype=np.float32)\nX_val_np = np.asarray(X_val, dtype=np.float32)\ny_train_np = np.asarray(y_train, dtype=np.int32)\ny_val_np = np.asarray(y_val, dtype=np.int32)\n\nprint("🚀 Training XGBoost model...")\nprint(f" Train samples: {len(X_train_np)}")\nprint(f" Val samples: {len(X_val_np)}")\nprint(f" Features: {X_train_np.shape[1]}")\nprint(f" Classes: 8\\n")\n\n# XGBoost parameters\nparams = {\n \'objective\': \'multi:softmax\', # Multi-class classification\n \'num_class\': 8, # 8 land use classes\n \'max_depth\': 6, # Maximum tree depth\n \'learning_rate\': 0.1, # Learning rate\n \'n_estimators\': 200, # Number of trees\n \'subsample\': 0.8, # Subsample ratio\n \'colsample_bytree\': 0.8, # Feature sampling ratio\n \'random_state\': 42,\n \'n_jobs\': -1, # Use all CPU cores\n \'eval_metric\': \'mlogloss\' # Multi-class log loss\n}\n\n# Train XGBoost model\nmodel = xgb.XGBClassifier(**params)\n\nmodel.fit(\n X_train_np, y_train_np,\n eval_set=[(X_train_np, y_train_np), (X_val_np, y_val_np)],\n verbose=True\n)\n\n# Validation accuracy\ny_val_pred = model.predict(X_val_np)\nval_accuracy = accuracy_score(y_val_np, y_val_pred)\nprint(f"\\n✅ Training completed!")\nprint(f" Validation Accuracy: {val_accuracy:.4f} ({val_accuracy*100:.2f}%)")\n')
# In[ ]:
get_ipython().run_cell_magic('time', '', '# Evaluate on test set\nX_test_np = np.asarray(X_test, dtype=np.float32)\ny_test_np = np.asarray(y_test, dtype=np.int32)\n\nprint("📊 Evaluating XGBoost model on test set...\\n")\n\n# Predictions\ny_pred_test = model.predict(X_test_np)\n\n# Metrics\nfrom sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, confusion_matrix\n\ntest_accuracy = accuracy_score(y_test_np, y_pred_test)\nprecision = precision_score(y_test_np, y_pred_test, average=\'weighted\', zero_division=0)\nrecall = recall_score(y_test_np, y_pred_test, average=\'weighted\', zero_division=0)\nf1 = f1_score(y_test_np, y_pred_test, average=\'weighted\', zero_division=0)\n\nprint(f"📈 Test Results:")\nprint(f" Accuracy: {test_accuracy:.4f} ({test_accuracy*100:.2f}%)")\nprint(f" Precision: {precision:.4f}")\nprint(f" Recall: {recall:.4f}")\nprint(f" F1-Score: {f1:.4f}\\n")\n\n# Confusion Matrix\nfrom sklearn.metrics import ConfusionMatrixDisplay\nimport matplotlib.pyplot as plt\n\n# Create figure first\nfig, ax = plt.subplots(figsize=(10, 8))\n\nclass_names = list(label_mapping.keys())\ncm = confusion_matrix(y_test_np, y_pred_test)\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=class_names)\ndisp.plot(cmap=\'Blues\', ax=ax)\nplt.xticks(rotation=45, ha=\'right\')\nplt.title(\'XGBoost Confusion Matrix\')\nplt.tight_layout()\nplt.show()\n')
# In[ ]:
# Lưu mô hình huấn luyện
import json
import joblib
# Save XGBoost model
model_path = "model_xgboost.joblib"
joblib.dump(model, model_path)
print(f"✅ Model saved to {model_path}")
# Save model info
info = {
"model_type": "XGBoost",
"num_classes": 8,
"classes": list(label_mapping.keys()),
"num_features": X_train_np.shape[1],
"params": params,
"accuracy": float(test_accuracy),
"precision": float(precision),
"recall": float(recall),
"f1_score": float(f1),
}
with open("model_xgboost_info.json", "w") as f:
json.dump(info, f, indent=2)
print(f"✅ Model info saved to model_xgboost_info.json")
# In[15]:
# đóng client, cluster
# client.close()
# cluster.close()
.temp/01.train_ODC_local_with_Mic_supplyer.ipynb
+2095
View File
File diff suppressed because one or more lines are too long
.temp/01.train_ODC_local_with_Mic_supplyer.py
+56
View File
@@ -0,0 +1,56 @@
#!/usr/bin/env python
# coding: utf-8
# In[6]:
get_ipython().run_cell_magic('time', '', '%matplotlib inline\n\n# Import Microsoft Planetary Computer libraries\nimport planetary_computer\nfrom pystac_client import Client\nfrom odc.stac import load as stac_load\n\n# Standard imports\nimport xarray as xr\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score, classification_report, confusion_matrix, ConfusionMatrixDisplay\nimport geopandas as gpd\n\n# XGBoost for GPU training\nimport xgboost as xgb\n\nfrom xgboost import XGBClassifier\n\nprint(f" XGBoost version: {xgb.__version__}")\n\nprint("✅ All modules loaded successfully")\n')
# In[7]:
get_ipython().run_cell_magic('time', '', '# Kết nối tới Microsoft Planetary Computer STAC\nfrom pystac_client import Client\n\n# KHÔNG dùng modifier ở catalog level để tránh items bị convert thành dict\ncatalog = Client.open(\n "https://planetarycomputer.microsoft.com/api/stac/v1"\n)\nprint("✅ Connected to Microsoft Planetary Computer")\n\nprint("\\n" + "="*70)\n')
# In[8]:
get_ipython().run_cell_magic('time', '', '# 🌍 Định nghĩa khu vực và thời gian\nprint("="*70)\nprint("CONFIGURATION")\nprint("="*70)\n\n# Khu vực quan tâm (Vietnam - Mekong Delta) - GIẢM DIỆN TÍCH ~40%\nbbox = [105.6, 9.3, 106.2, 9.8] # [min_lon, min_lat, max_lon, max_lat]\n\n# GIẢM THỜI GIAN xuống 3 tháng để giảm kích thước dữ liệu cho PC\ntime_range = "2023-03-01/2023-05-31" # 3 tháng (mùa khô)\n\nprint(f"\\n📍 Area of Interest:")\nprint(f" Longitude: {bbox[0]} to {bbox[2]}")\nprint(f" Latitude: {bbox[1]} to {bbox[3]}")\nprint(f"\\n📅 Time Range: {time_range}")\nprint(f" ⚠️ Optimized for personal computer (3 months, reduced area)")\nprint(f"\\n🗺️ CRS: EPSG:32648")\nprint(f" Resolution: 20m (reduced from 10m for smaller data size)")\n\nprint("="*70)\n')
# In[9]:
get_ipython().run_cell_magic('time', '', '# 📡 LOAD SENTINEL-2 FROM MICROSOFT PLANETARY COMPUTER\nprint("="*70)\nprint("LOADING SENTINEL-2 L2A")\nprint("="*70)\n\nprint("\\n🔍 Searching for Sentinel-2 scenes...")\nquery_s2 = catalog.search(\n collections=["sentinel-2-l2a"],\n bbox=bbox,\n datetime=time_range,\n query={"eo:cloud_cover": {"lt": 30}} # Cloud cover < 30% (giảm từ 50%)\n)\n\nitems_s2 = list(query_s2.item_collection())\nprint(f"✅ Found {len(items_s2)} Sentinel-2 scenes")\n\n# GIỚI HẠN SỐ LƯỢNG SCENES cho PC cá nhân\nmax_scenes = 12 # Giảm xuống 12 scenes để tối ưu cho PC\nif len(items_s2) > max_scenes:\n print(f"⚠️ Limiting to {max_scenes} scenes for personal computer")\n # Chọn scenes đều đặn trong khoảng thời gian\n step = len(items_s2) // max_scenes\n items_s2 = items_s2[::step][:max_scenes]\n print(f" Selected {len(items_s2)} scenes evenly distributed")\n\nif len(items_s2) > 0:\n # Show first few scenes\n print(f"\\n📋 Sample scenes:")\n for i, item in enumerate(items_s2[:5]):\n date = item.datetime.strftime("%Y-%m-%d")\n cloud = item.properties.get("eo:cloud_cover", "N/A")\n print(f" [{i+1}] {date} - Cloud: {cloud}%")\n \n # Re-sign items to ensure fresh URLs (keep as pystac objects)\n print(f"\\n🔑 Signing STAC items...")\n items_s2 = [planetary_computer.sign(item) for item in items_s2]\n \n # Load Sentinel-2 data (without Dask chunks)\n print(f"\\n⏳ Loading Sentinel-2 data...")\n ds_s2 = stac_load(\n items_s2,\n bands=["B04", "B08", "SCL"], # Red (B04), NIR (B08), Scene Classification (SCL)\n crs="EPSG:32648",\n resolution=20, # 20m resolution (4x smaller data than 10m)\n bbox=bbox,\n patch_url=planetary_computer.sign, # Re-sign URLs during loading\n fail_on_error=False, # Skip problematic tiles instead of crashing\n )\n \n # Rename bands to simpler names\n ds_s2 = ds_s2.rename({"B04": "red", "B08": "nir", "SCL": "scl"})\n \n print(f"\\n✅ Sentinel-2 loaded!")\n print(f" Shape: {dict(ds_s2.dims)}")\n print(f" Variables: {list(ds_s2.data_vars)}")\n display(ds_s2)\nelse:\n print(f"❌ No Sentinel-2 scenes found")\n\n ds_s2 = Noneprint("="*70)\n')
# In[10]:
get_ipython().run_cell_magic('time', '', '# 📡 LOAD SENTINEL-1 FROM MICROSOFT PLANETARY COMPUTER\nprint("="*70)\nprint("LOADING SENTINEL-1 RTC")\nprint("="*70)\n\nprint("\\n🔍 Searching for Sentinel-1 scenes...")\nquery_s1 = catalog.search(\n collections=["sentinel-1-rtc"],\n bbox=bbox,\n datetime=time_range,\n)\n\nitems_s1 = list(query_s1.item_collection())\nprint(f"✅ Found {len(items_s1)} Sentinel-1 scenes")\n\n# GIỚI HẠN SỐ LƯỢNG SCENES cho PC cá nhân\nmax_scenes = 12 # Giảm xuống 12 scenes để tối ưu cho PC\nif len(items_s1) > max_scenes:\n print(f"⚠️ Limiting to {max_scenes} scenes for personal computer")\n # Chọn scenes đều đặn trong khoảng thời gian\n step = len(items_s1) // max_scenes\n items_s1 = items_s1[::step][:max_scenes]\n print(f" Selected {len(items_s1)} scenes evenly distributed")\n\nif len(items_s1) > 0:\n # Show first few scenes\n print(f"\\n📋 Sample scenes:")\n for i, item in enumerate(items_s1[:5]):\n date = item.datetime.strftime("%Y-%m-%d")\n orbit = item.properties.get("sat:orbit_state", "N/A")\n print(f" [{i+1}] {date} - Orbit: {orbit}")\n \n # Re-sign items to ensure fresh URLs (keep as pystac objects)\n print(f"\\n🔑 Signing STAC items...")\n items_s1 = [planetary_computer.sign(item) for item in items_s1]\n \n # Load Sentinel-1 data (without Dask chunks)\n print(f"\\n⏳ Loading Sentinel-1 data...")\n ds_s1 = stac_load(\n items_s1,\n bands=["vv", "vh"], # VV and VH polarizations\n crs="EPSG:32648",\n resolution=20, # 20m resolution (4x smaller data than 10m)\n bbox=bbox,\n patch_url=planetary_computer.sign, # Re-sign URLs during loading\n fail_on_error=False, # Skip problematic tiles instead of crashing\n )\n \n # Convert to dB (Microsoft S1 is in linear power)\n print(f"\\n🔄 Converting to dB...")\n ds_s1[\'vv_db\'] = 10 * np.log10(ds_s1[\'vv\'].where(ds_s1[\'vv\'] > 0))\n ds_s1[\'vh_db\'] = 10 * np.log10(ds_s1[\'vh\'].where(ds_s1[\'vh\'] > 0))\n \n print(f"\\n✅ Sentinel-1 loaded!")\n print(f" Shape: {dict(ds_s1.dims)}")\n print(f" Variables: {list(ds_s1.data_vars)}")\n display(ds_s1)\nelse:\n print(f"❌ No Sentinel-1 scenes found")\n\n ds_s1 = Noneprint("="*70)\n')
# In[11]:
get_ipython().run_cell_magic('time', '', '# 🌿 CALCULATE NDVI AND PROCESS DATA\nprint("="*70)\nprint("DATA PROCESSING")\nprint("="*70)\n\nif ds_s2 is not None:\n print("\\n[1] Calculating NDVI...")\n # NDVI = (NIR - Red) / (NIR + Red)\n ndvi = (ds_s2[\'nir\'] - ds_s2[\'red\']) / (ds_s2[\'nir\'] + ds_s2[\'red\'] + 1e-8)\n \n print(f"✅ NDVI calculated")\n print(f" Shape: {ndvi.shape}")\n print(f" Time steps: {len(ndvi.time)}")\n \n # Cloud masking using SCL band\n print(f"\\n[2] Applying cloud mask...")\n # SCL values: 1=defective, 3=cloud shadow, 8=cloud medium, 9=cloud high, 10=cirrus\n cloud_mask = ds_s2[\'scl\'].isin([1, 3, 8, 9, 10])\n ndvi_masked = ndvi.where(~cloud_mask)\n \n print(f"✅ Cloud mask applied")\n \n # Temporal aggregation (mean over time)\n print(f"\\n[3] Computing mean NDVI across time...")\n ndvi_mean = ndvi_masked.mean(dim=\'time\')\n \n # Data already in memory, no need to compute() again\n print(f"✅ Mean NDVI computed")\n print(f" Shape: {ndvi_mean.shape}")\n \nelse:\n print("❌ No Sentinel-2 data to process")\n ndvi_mean = None\n\nprint("="*70)\n')
# In[ ]:
get_ipython().run_cell_magic('time', '', '# 🎯 EXTRACT TRAINING DATA FEATURES\nprint("="*70)\nprint("FEATURE EXTRACTION")\nprint("="*70)\n\n# Check if required data is available\nif \'ndvi_mean\' not in globals() or \'ds_s1\' not in globals():\n print("❌ Error: Please run Cell 6 (DATA PROCESSING) first!")\n print(" Required variables: ndvi_mean, ds_s1")\n raise RuntimeError("Missing required data. Run cells in order: Cell 4 → Cell 5 → Cell 6 → Cell 7")\n\n# Load training shapefile\nimport geopandas as gpd\n\ntrain_path = \'train/ST_training data_updated_1130points_new.shp\'\nprint(f"\\n[1] Loading training data from: {train_path}")\ntrain_gdf = gpd.read_file(train_path)\n\n# Ensure CRS matches\nif train_gdf.crs != \'EPSG:32648\':\n print(f" Reprojecting from {train_gdf.crs} to EPSG:32648...")\n train_gdf = train_gdf.to_crs(\'EPSG:32648\')\n\nprint(f"✅ Loaded {len(train_gdf)} training points")\nprint(f" Available columns: {list(train_gdf.columns)}")\n\n# Auto-detect label column (look for common names)\nlabel_column = None\nfor col in [\'HT_code\', \'Ma_LU\', \'LU2022\', \'class\', \'Class\', \'CLASS\', \'label\', \'Label\', \'LABEL\', \'LU_CODE\', \'LU_code\']:\n if col in train_gdf.columns:\n label_column = col\n break\n\nif label_column is None:\n print(f"❌ Cannot find label column. Available columns: {list(train_gdf.columns)}")\n print(f" Please check your shapefile and update the code.")\nelse:\n print(f" Using label column: \'{label_column}\'")\n print(f" Classes: {sorted(train_gdf[label_column].unique())}")\n \n # Extract features at each training point\n print(f"\\n[2] Extracting features at training points...")\n \n features = []\n labels = []\n skipped = 0\n \n for idx, row in train_gdf.iterrows():\n point = row.geometrychro\n x_coord = point.x\n y_coord = point.y\n label = row[label_column]\n \n # Extract NDVI at this location\n if ndvi_mean is not None and ds_s1 is not None:\n try:\n ndvi_val = ndvi_mean.sel(x=x_coord, y=y_coord, method=\'nearest\').values\n \n # Extract Sentinel-1 VH/VV at this location (mean across time)\n # Data already in memory, no need to compute()\n vh_val = ds_s1[\'vh_db\'].sel(x=x_coord, y=y_coord, method=\'nearest\').mean(dim=\'time\').values\n vv_val = ds_s1[\'vv_db\'].sel(x=x_coord, y=y_coord, method=\'nearest\').mean(dim=\'time\').values\n \n # Create feature vector: [NDVI, VH_dB, VV_dB]\n feature_vec = [ndvi_val, vh_val, vv_val]\n \n # Only add if all features are valid (not NaN)\n if not np.isnan(feature_vec).any():\n features.append(feature_vec)\n labels.append(label)\n else:\n skipped += 1\n except Exception as e:\n # Skip points outside the data extent\n skipped += 1\n continue\n \n features = np.array(features)\n labels = np.array(labels)\n \n print(f"✅ Extracted features for {len(features)} valid points")\n print(f" Skipped {skipped} points (outside extent or NaN values)")\n print(f" Feature shape: {features.shape}")\n print(f" Feature names: [\'NDVI_mean\', \'VH_dB_mean\', \'VV_dB_mean\']")\n print(f"\\n Class distribution:")\n unique, counts = np.unique(labels, return_counts=True)\n for cls, cnt in zip(unique, counts):\n print(f" Class {cls}: {cnt} samples ({cnt/len(labels)*100:.1f}%)")\n\nprint("="*70)\n')
# In[21]:
get_ipython().run_cell_magic('time', '', '# 🤖 TRAIN XGBOOST MODEL ON GPU (RTX 4060)\nprint("="*70)\nprint("MODEL TRAINING - GPU ACCELERATED")\nprint("="*70)\n\nfrom xgboost import XGBClassifier\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import classification_report, confusion_matrix, ConfusionMatrixDisplay\nimport matplotlib.pyplot as plt\n\n# Encode labels to ensure they are 0, 1, 2, ... n-1\nprint("\\n[1] Encoding labels...")\nlabel_encoder = LabelEncoder()\nlabels_encoded = label_encoder.fit_transform(labels)\nprint(f"✅ Original classes: {label_encoder.classes_}")\nprint(f" Encoded as: {np.unique(labels_encoded)}")\n\n# Split data\nprint("\\n[2] Splitting data (80% train, 20% test)...")\nX_train, X_test, y_train, y_test = train_test_split(\n features, labels_encoded, test_size=0.2, random_state=42, stratify=labels_encoded\n)\nprint(f"✅ Training samples: {len(X_train)}")\nprint(f" Testing samples: {len(X_test)}")\n\n# Train XGBoost on GPU\nprint("\\n[3] Training XGBoost classifier on RTX 4060 GPU...")\nprint(" GPU Settings: device=\'cuda:0\'")\n\nxgb_model = XGBClassifier(\n n_estimators=100,\n max_depth=20,\n learning_rate=0.1,\n device=\'cuda:0\', # Use GPU (updated from deprecated gpu_id)\n tree_method=\'hist\', # Use hist with device for GPU training\n random_state=42,\n eval_metric=\'mlogloss\', # Multi-class log loss\n verbosity=1 # Show GPU training progress\n)\n\nxgb_model.fit(X_train, y_train)\nprint(f"✅ Model trained on GPU")\n\n# Evaluate\nprint("\\n[4] Evaluating model...")\ntrain_score = xgb_model.score(X_train, y_train)\ntest_score = xgb_model.score(X_test, y_test)\nprint(f"✅ Training accuracy: {train_score:.4f}")\nprint(f" Testing accuracy: {test_score:.4f}")\n\n# Classification report\nprint("\\n[5] Classification Report:")\ny_pred = xgb_model.predict(X_test)\nprint(classification_report(y_test, y_pred, target_names=[str(c) for c in label_encoder.classes_]))\n\n# Confusion matrix\nprint("\\n[6] Confusion Matrix:")\nfig, ax = plt.subplots(figsize=(10, 8))\ncm = confusion_matrix(y_test, y_pred)\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=label_encoder.classes_)\ndisp.plot(ax=ax, cmap=\'Blues\', values_format=\'d\')\nplt.title(\'Confusion Matrix - XGBoost GPU Model (RTX 4060)\')\nplt.tight_layout()\nplt.show()\n\nprint("="*70)\n')
# In[23]:
get_ipython().run_cell_magic('time', '', '# 💾 SAVE MODEL AND CLEANUP\nprint("="*70)\nprint("SAVING MODEL & CLEANUP")\nprint("="*70)\n\nimport joblib\nfrom datetime import datetime\n\n# Save model and label encoder\nmodel_filename = f"model_train/model_xgboost_gpu_{datetime.now().strftime(\'%Y%m%d_%H%M%S\')}.joblib"\nprint(f"\\n[1] Saving model to: {model_filename}")\njoblib.dump({\'model\': xgb_model, \'label_encoder\': label_encoder}, model_filename)\nprint(f"✅ Model and label encoder saved")\n\n# Save model info\ninfo = {\n "timestamp": datetime.now().isoformat(),\n "data_source": "Microsoft Planetary Computer STAC",\n "collections": ["sentinel-2-l2a", "sentinel-1-rtc"],\n "features": ["NDVI_mean", "VH_dB_mean", "VV_dB_mean"],\n "training_samples": len(X_train),\n "testing_samples": len(X_test),\n "train_accuracy": float(train_score),\n "test_accuracy": float(test_score),\n "model_type": "XGBClassifier",\n "device": "cuda:0",\n "gpu_device": "RTX 4060",\n "tree_method": "hist",\n "n_estimators": 100,\n "max_depth": 20,\n "learning_rate": 0.1\n}\n\nimport json\ninfo_filename = model_filename.replace(\'.joblib\', \'_info.json\')\nwith open(info_filename, \'w\') as f:\n json.dump(info, f, indent=2)\nprint(f"✅ Model info saved to: {info_filename}")\n\n# No cleanup needed (Dask removed)\nprint("\\n[2] Cleanup complete")\n\nprint("="*70)\n\nprint("\\n" + "="*70)\n\nprint("🎉 TRAINING COMPLETE!")\n')
.temp/02.predict_ODC.ipynb
+4214
View File
File diff suppressed because one or more lines are too long
.temp/02.predict_ODC.py
+191
View File
@@ -0,0 +1,191 @@
#!/usr/bin/env python
# coding: utf-8
# In[1]:
get_ipython().run_cell_magic('time', '', '%matplotlib inline\n\nimport importlib\nimport new_import_ODC \n\nimportlib.reload(new_import_ODC)\n\nfrom new_import_ODC import *\n')
# In[2]:
get_ipython().run_cell_magic('time', '', '# Dask gateway\ncluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))\ndc = datacube.Datacube()\n\n# Configure s3 access\nconfigure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n\nclient\n')
# In[3]:
## cấu hình thời gian lấy ảnh và tọa độ
date_range = ('2022-09-01', '2023-10-01')
longtitude_range = (105.86575, 105.94120)
latitude_range = (9.65070, 9.69850)
# In[4]:
## truy vấn ảnh vệ tinh sen2
data = load_data(dc, date_range, longtitude_range, latitude_range)
notebook_utils.heading(notebook_utils.xarray_object_size(data))
display(data)
# In[5]:
get_ipython().run_cell_magic('time', '', '# Tiến hành loại bỏ các vị trí bị mây ảnh hưởng\nresult = mask_clean(data)\nprogress(result)\n')
# In[6]:
# Tiến hành tính toán NDVI
ds1 = calculate_indices(result, index='NDVI', satellite_mission='s2')
ndvi = ds1["NDVI"]
display(ndvi)
# In[7]:
## ảnh NDVI chưa điền mây (fill nan)
plt.imshow(ndvi.isel(time=50))
# In[8]:
# đặt thời gian các mùa
time_split = [slice('2022-09-01', '2023-01-01'),
slice('2023-01-01', '2023-05-01'),
slice('2023-05-01', '2023-07-01'),
slice('2023-07-01', '2023-10-01')]
# Điền mây ở các vị trí mang giá trị nan (fill nan)
fill_nan_ndvi = fill_nan(ndvi, time_split)
# In kết quả ảnh ndvi đã điền mây (đã fill nan)
plt.imshow(fill_nan_ndvi.isel(time=50))
# In[9]:
get_ipython().run_cell_magic('time', '', "## tính ndvi theo tháng\naverage_ndvi = fill_nan_ndvi.resample(time='1M').mean().persist()\nprogress(average_ndvi)\n")
# In[10]:
# compute average_ndvi
average_ndvi = average_ndvi.compute()
# In[11]:
# load dữ liệu sen1
coordinates = (longtitude_range, latitude_range)
dsvh, dsvv = load_data_sen1(dc, date_range, coordinates)
average_vv = calculate_average(dsvv, time_pattern='1M')
average_vh = calculate_average(dsvh, time_pattern='1M')
# In[12]:
# load model RF
loaded_model = joblib.load(os.path.join("model_train", "model_odc.joblib"))
# dự đoán
data_array = predict(loaded_model, data.rio.crs, average_ndvi, average_vh, average_vv)
# In[13]:
# cấu hình màu cho các loại đất
colors = [
"#abcee9",
"#ffef44",
"#c4ff9e",
"#ffd6a8",
"#93ddda",
"#1aeef7",
"#ffa7f2",
"#33ee33"
]
labels = [
"Lúa tôm",
"Lúa",
"CHN",
"CLN",
"TS",
"Sông",
"Đất xây dựng",
"Rừng"
]
# hiển thị phân loại sử dụng đất
cmap = ListedColormap(colors)
img = data_array.plot(cmap=cmap, add_colorbar=False)
cbar = plt.colorbar(img)
cbar.ax.set_yticklabels(labels)
plt.title("Phân loại sử dụng đất")
plt.axis('off')
plt.show()
# In[14]:
## cấu hình shapefile ranh giới thuận hòa và vh vv file
thuanhoa_path = "ThuanHoa/region/ST_ThuanHoa_Boundaryofficially.shp"
# cắt theo ranh giới xã thuận hòa
region_result = cut_according_shp(thuanhoa_path, average_ndvi, data_array)
# In[15]:
# hiển thị kết quả phân loại sử dụng đất
colorval = list(range(len(colors)))
options = {
'title': 'Phân loại sử dụng đất',
'cmap': colors,
'clim': (0, 8),
'aspect': 'equal',
'colorbar_opts': {
'major_label_overrides': dict(zip(colorval, labels)),
'major_label_text_align': 'left',
'ticker': FixedTicker(ticks=colorval),
},
}
region_result.hvplot(
rasterize = True, # Use Datashader, particularly useful for dask arrays
aggregator = reductions.mode(), # Datashader selects mode value, requires 'hv.Image'
).options(opts.Image(**options))
# In[16]:
# Lưu lại kết quả
region_result.rio.to_raster("KetQuaPhanLoaiDatODC.tif")
# In[17]:
# đóng client, cluster
client.close()
cluster.close()
# In[ ]:
.temp/03.compare_ODC.ipynb
+1406
View File
File diff suppressed because one or more lines are too long
.temp/03.compare_ODC.py
+117
View File
@@ -0,0 +1,117 @@
#!/usr/bin/env python
# coding: utf-8
# In[1]:
# Khai báo các thư viện cần thiết
from new_import_ODC import *
# Khai báo đường dẫn đến kết quả phân loại và dữ liệu của địa phương
KD_path = "ThuanHoa/KhoanhDat/ThuanHoa_TKDD2022.shp"
KetQuaPhanLoaiDat = "KetQuaPhanLoaiDatODC.tif"
# In[2]:
# khai báo các loại đất từ dữ liệu kiểm kê ứng với các hiện trạng được phân loại từ viễn thám
CODE_MAP = {
"BHK": 2,
"CLN": 3,
"DGD": 6,
"DGT": 6,
"DNL": 6,
"DRA": 6,
"DSH": 6,
"DTL": 5,
"DTS": 6,
"DYT": 6,
"LUC": 1,
"NKH": 3,
"NTD": 6,
"NTS": 4,
"ONT": 6,
"SKC": 6,
"SKX": 6,
"SON": 5,
"TMD": 6,
"TON": 6,
"TSC": 6,
}
# Khai báo các nhãn phân loại đất ứng với 3 loại đất chính
HT_MAP = {
"NN": {"name": "Đất Nông Nghiệp", "data": [1, 2, 3, 4]},
"PNN": {"name": "Đất Phi Nông Nghiệp", "data": [6]},
"TQ": {"name": "Đất Thổ Quả", "data": [15]},
}
# In[3]:
# Tiến hành chồng lắp
result = compare(KD_path, KetQuaPhanLoaiDat, CODE_MAP, HT_MAP)
# In[4]:
# cấu hình màu cho các loại sử dụng đất
colors = [
"#abcee9",
"#ffffc0",
"#c4ff9e",
"#ffd6a8",
"#93ddda",
"#1aeef7",
"#ffa7f2",
"#33ee33",
]
labels = ["Lúa tôm", "Lúa", "CHN", "CLN", "TS", "Sông", "Đất xây dựng", "Rừng"]
# In[5]:
# Lưu kết quả
save_result(result, HT_MAP)
# In[6]:
# hiển thị kết quả
xx = []
for k, v in result.items():
rs = merge_arrays(v, nodata=np.nan)
xx.append(rs.squeeze(drop=True))
xx = xr.concat(xx, pd.Index([HT_MAP[x]["name"] for x in HT_MAP], name="name"))
colorval = list(range(len(colors)))
options = {
"cmap": colors,
"clim": (0, 8),
"aspect": "equal",
"height": 400,
"colorbar_opts": {
"major_label_overrides": dict(zip(colorval, labels)),
"major_label_text_align": "left",
"ticker": FixedTicker(ticks=colorval),
},
}
xx.hvplot(
groupby="name",
rasterize=True, # Use Datashader, particularly useful for dask arrays
aggregator=reductions.mode(), # Datashader selects mode value, requires 'hv.Image'
).options(opts.Image(**options))
# In[ ]:
.temp/Download_Cache_Colab.ipynb
+130
View File
File diff suppressed because one or more lines are too long
.temp/check_X.py
+18
View File
@@ -0,0 +1,18 @@
import joblib
import numpy as np
cache_file = "dataset_cache/training_data_2d.joblib"
data = joblib.load(cache_file)
X = np.array(data['X'])
y = np.array(data['y'])
print("X shape:", X.shape)
print("X mean:", np.mean(X))
print("X std:", np.std(X))
print("X min:", np.min(X))
print("X max:", np.max(X))
print("Any NaN:", np.isnan(X).any())
for i in range(6):
print(f"Channel {i} mean: {np.mean(X[:, i, :, :]):.4f}, min: {np.min(X[:, i, :, :]):.4f}, max: {np.max(X[:, i, :, :]):.4f}")
.temp/check_missing.py
+10
View File
@@ -0,0 +1,10 @@
import joblib
import geopandas as gpd
from shapely.geometry import Point
data = joblib.load('dataset_cache/training_data_2d.joblib')
X, y = data['X'], data['y']
print(f"X shape: {X.shape}, y shape: {y.shape}")
gdf = gpd.read_file("train/ST_training_data_updated_1130points_new.shp")
print("Total points:", len(gdf))
.temp/check_versions.py
+5
View File
@@ -0,0 +1,5 @@
import xarray as xr
import rasterio
print(f"xarray version: {xr.__version__}")
print(f"rasterio version: {rasterio.__version__}")
.temp/check_xgboost_data.py
+7
View File
@@ -0,0 +1,7 @@
import joblib
import numpy as np
data = joblib.load('dataset_cache/training_data.joblib')
X, y = data['X'], data['y']
print(f"X shape: {X.shape}")
print(f"y shape: {y.shape}")
.temp/check_zeros.py
+8
View File
@@ -0,0 +1,8 @@
import joblib
import numpy as np
cache_file = "dataset_cache/training_data_2d.joblib"
data = joblib.load(cache_file)
X = np.array(data['X'])
b2 = X[:, 0, :, :]
print("Zeros in B2:", np.sum(b2 == 0) / b2.size)
print("X shape:", X.shape)
.temp/cloud_removal_train.ipynb
+10
View File
@@ -0,0 +1,10 @@
{
"cells": [],
"metadata": {
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
.temp/fix_all.py
+25
View File
@@ -0,0 +1,25 @@
import glob, json
changed_files = []
for file_path in glob.glob('*.ipynb'):
with open(file_path, 'r', encoding='utf-8') as f:
nb = json.load(f)
changed = False
for cell in nb.get('cells', []):
if cell.get('cell_type') == 'code':
source = cell.get('source', [])
for i, line in enumerate(source):
if 'time=50' in line:
source[i] = line.replace('time=50', 'time=0')
changed = True
if 'load_data_sen1(dc,' in line:
source[i] = line.replace('load_data_sen1(dc,', 'load_data_sen1(None,')
changed = True
if changed:
with open(file_path, 'w', encoding='utf-8') as f:
json.dump(nb, f, indent=1)
changed_files.append(file_path)
print('Fixed issues in:', changed_files)
.temp/fix_cache.py
+19
View File
@@ -0,0 +1,19 @@
import joblib
import geopandas as gpd
import numpy as np
cache_file = "dataset_cache/training_data_2d.joblib"
data = joblib.load(cache_file)
X = data['X']
print("X shape:", len(X))
gdf = gpd.read_file("train/ST_training_data_updated_1130points_new.shp")
gdf = gdf.to_crs("EPSG:32648")
print("gdf length:", len(gdf))
if len(X) == len(gdf):
y = [(row['HT_code'] - 1) for idx, row in gdf.iterrows()]
joblib.dump({'X': X, 'y': y}, cache_file)
print("Fixed y in cache! Saved.")
else:
print("Lengths do not match, cannot fix automatically.")
.temp/fix_new_train.py
+20
View File
@@ -0,0 +1,20 @@
import json
def fix_import(file_path):
with open(file_path, 'r', encoding='utf-8') as f:
nb = json.load(f)
changed = False
for cell in nb.get('cells', []):
if cell.get('cell_type') == 'code':
source = cell.get('source', [])
if isinstance(source, list):
for i, line in enumerate(source):
if "from new_import import *" in line:
source[i] = line.replace("from new_import import *", "from new_import_ODC import *")
changed = True
if changed:
with open(file_path, 'w', encoding='utf-8') as f:
json.dump(nb, f, indent=1)
print(f"Fixed {file_path}")
fix_import('new_train.ipynb')
.temp/fix_train.py
+20
View File
@@ -0,0 +1,20 @@
import re
filepath = "01.train_ODC.py"
with open(filepath, 'r', encoding='utf-8') as f:
content = f.read()
# Find the run_cell_magic line
pattern = re.compile(r"get_ipython\(\)\.run_cell_magic\('time', '', '(# 🤖 LAND USE CLASSIFICATION MODEL TRAINING.*?)(?=\n')\n'", re.DOTALL)
def repl(match):
# Get the inner string and escape all actual newlines with \n
inner = match.group(1)
inner = inner.replace('\n', '\\n')
return f"get_ipython().run_cell_magic('time', '', '{inner}')"
content = pattern.sub(repl, content)
with open(filepath, 'w', encoding='utf-8') as f:
f.write(content)
print("Fixed 01.train_ODC.py syntax")
.temp/fix_typo.py
+22
View File
@@ -0,0 +1,22 @@
import json
def fix_filename(file_path):
with open(file_path, 'r', encoding='utf-8') as f:
nb = json.load(f)
changed = False
for cell in nb.get('cells', []):
if cell.get('cell_type') == 'code':
source = cell.get('source', [])
if isinstance(source, list):
for i, line in enumerate(source):
if "ST_training data_updated_1130points.shp" in line:
source[i] = line.replace("ST_training data_updated_1130points.shp", "ST_training_data_updated_1130points.shp")
changed = True
if changed:
with open(file_path, 'w', encoding='utf-8') as f:
json.dump(nb, f, indent=1)
print(f"Fixed typo in {file_path}")
import glob
for nb in glob.glob("*.ipynb"):
fix_filename(nb)
.temp/generate_colab_notebook.py
+58
View File
@@ -0,0 +1,58 @@
import nbformat as nbf
nb = nbf.v4.new_notebook()
text_1 = """# Script Tải Dữ liệu Vệ tinh (Cache) qua Google Colab
Mục đích của Notebook này là mượn sức mạnh đường truyền và RAM của Google Colab để tải 270 ảnh Sentinel-2 & Sentinel-1 từ Microsoft Planetary Computer. Sau khi xử lý nội suy, nó sẽ sinh ra một file cache `.joblib` duy nhất chứa toàn bộ mảng dữ liệu.
Bạn chỉ cần tải file `.joblib` đó về máy là xong!"""
code_1 = """!pip install planetary-computer pystac-client odc-stac geopandas rasterio xarray joblib scikit-learn xgboost lightgbm"""
code_2 = """from google.colab import drive
drive.mount('/content/drive')"""
text_2 = """## Hướng dẫn:
1. Nén toàn bộ thư mục `remote-sensing` ở máy tính của bạn thành file `remote-sensing.zip`.
2. Upload file `remote-sensing.zip` đó lên Google Drive (để ngay ngoài cùng).
3. Chạy ô lệnh bên dưới để giải nén và chuyển vào thư mục dự án."""
code_3 = """import os
import shutil
# Giải nén dự án từ Google Drive
!unzip -q /content/drive/MyDrive/remote-sensing.zip -d /content/
os.chdir('/content/remote-sensing')
!ls -la"""
text_3 = """## Bắt đầu tải và Cache Dữ Liệu
Chạy một mô hình CPU đơn giản (Decision Tree) để ép hệ thống gọi hàm `FeatureExtractor`. Hàm này sẽ làm mọi việc nặng nhọc: tìm ảnh, ghép mây, tính trung vị và lưu kết quả vào thư mục `dataset_cache/`."""
code_4 = """# Lệnh này sẽ mất khoảng 5-15 phút để tải toàn bộ ảnh từ Microsoft
!python train_land_decision_tree_gpu.py"""
text_4 = """## Hoàn tất
Bạn hãy kiểm tra xem file `.joblib` lớn (khoảng 40-60MB) đã xuất hiện chưa. Nếu rồi, hãy lưu ngược nó lại Google Drive để tải về máy!"""
code_5 = """# Xem file cache đã được tạo thành công chưa
!ls -lh dataset_cache/
# Copy toàn bộ thư mục cache sang Google Drive để tải về máy dễ dàng
!cp -r dataset_cache/ /content/drive/MyDrive/dataset_cache_finished/
print("Hoàn thành! Bạn hãy mở Google Drive của mình, tìm thư mục 'dataset_cache_finished' và tải file .joblib mới nhất về máy tính.")"""
nb['cells'] = [
nbf.v4.new_markdown_cell(text_1),
nbf.v4.new_code_cell(code_1),
nbf.v4.new_code_cell(code_2),
nbf.v4.new_markdown_cell(text_2),
nbf.v4.new_code_cell(code_3),
nbf.v4.new_markdown_cell(text_3),
nbf.v4.new_code_cell(code_4),
nbf.v4.new_markdown_cell(text_4),
nbf.v4.new_code_cell(code_5)
]
with open('Download_Cache_Colab.ipynb', 'w') as f:
nbf.write(nb, f)
print("Created Download_Cache_Colab.ipynb")
.temp/generate_previews.py
+132
View File
@@ -0,0 +1,132 @@
#!/usr/bin/env python3
"""
Generate PNG previews for existing GeoTIFF prediction files
"""
import numpy as np
import rasterio
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from pathlib import Path
import sys
def generate_png_preview(tif_file, output_png=None):
"""Generate PNG preview from GeoTIFF file"""
tif_path = Path(tif_file)
if not tif_path.exists():
print(f"❌ File not found: {tif_file}")
return False
# Determine output PNG path
if output_png is None:
output_png = tif_path.with_suffix('.png')
else:
output_png = Path(output_png)
try:
# Read GeoTIFF
with rasterio.open(tif_path) as src:
data = src.read(1)
print(f"📊 Data shape: {data.shape}, range: [{np.nanmin(data):.3f}, {np.nanmax(data):.3f}]")
# Determine if it's classification or NDVI based on filename
is_classification = 'classification' in tif_path.name.lower() or 'prediction' in tif_path.name.lower()
is_ndvi = 'ndvi' in tif_path.name.lower()
# Create figure
fig, ax = plt.subplots(figsize=(12, 10), dpi=150)
if is_ndvi:
# NDVI: use RdYlGn colormap, range -1 to 1
im = ax.imshow(data, cmap='RdYlGn', vmin=-1, vmax=1, interpolation='nearest')
ax.set_title(f'NDVI - {tif_path.stem}', fontsize=14, fontweight='bold')
cbar_label = 'NDVI'
elif is_classification:
# Classification: use tab20 colormap
im = ax.imshow(data, cmap='tab20', interpolation='nearest')
ax.set_title(f'Land Classification - {tif_path.stem}', fontsize=14, fontweight='bold')
cbar_label = 'Class'
else:
# Generic: use viridis
im = ax.imshow(data, cmap='viridis', interpolation='nearest')
ax.set_title(f'{tif_path.stem}', fontsize=14, fontweight='bold')
cbar_label = 'Value'
ax.set_xlabel('X (pixels)', fontsize=10)
ax.set_ylabel('Y (pixels)', fontsize=10)
# Add colorbar
cbar = plt.colorbar(im, ax=ax, fraction=0.046, pad=0.04)
cbar.set_label(cbar_label, rotation=270, labelpad=15)
# For classification, try to set integer ticks
if is_classification:
try:
unique_vals = np.unique(data[~np.isnan(data)])
if len(unique_vals) < 20: # Only if not too many classes
cbar.set_ticks(unique_vals)
cbar.set_ticklabels([str(int(v)) for v in unique_vals])
except:
pass
# Add grid
ax.grid(True, alpha=0.3, linestyle='--', linewidth=0.5)
# Save PNG
plt.tight_layout()
plt.savefig(str(output_png), dpi=150, bbox_inches='tight')
plt.close(fig)
print(f"✅ Created PNG: {output_png}")
return True
except Exception as e:
print(f"❌ Error creating PNG: {e}")
import traceback
traceback.print_exc()
return False
def generate_all_previews(predictions_dir="predictions"):
"""Generate PNG previews for all GeoTIFF files without PNGs"""
pred_path = Path(predictions_dir)
if not pred_path.exists():
print(f"❌ Directory not found: {predictions_dir}")
return
tif_files = list(pred_path.glob("*.tif"))
print(f"🔍 Found {len(tif_files)} GeoTIFF files")
generated = 0
skipped = 0
for tif_file in tif_files:
png_file = tif_file.with_suffix('.png')
if png_file.exists():
print(f"⏭️ Skipping {tif_file.name} (PNG already exists)")
skipped += 1
continue
print(f"\n🎨 Processing {tif_file.name}...")
if generate_png_preview(tif_file):
generated += 1
print(f"\n{'='*60}")
print(f"✅ Generated {generated} new PNG previews")
print(f"⏭️ Skipped {skipped} files (already have PNGs)")
print(f"{'='*60}")
if __name__ == "__main__":
if len(sys.argv) > 1:
# Process specific file
tif_file = sys.argv[1]
generate_png_preview(tif_file)
else:
# Process all files in predictions directory
generate_all_previews()
.temp/generate_report_only.py
+97
View File
@@ -0,0 +1,97 @@
import os
import glob
import json
from tabulate import tabulate
print("📊 BẢNG SO SÁNH KẾT QUẢ CÁC MÔ HÌNH\n")
# 1. Phân loại đất
print("### 1. Nhóm Phân loại Lớp phủ (Land Classification)")
land_data = []
if os.path.exists("model_xgboost_info.json"):
with open("model_xgboost_info.json", 'r') as f:
data = json.load(f)
params = data.get('params', {})
param_str = f"estimators:{params.get('n_estimators')}, depth:{params.get('max_depth')}" if params else "N/A"
land_data.append([
data.get('model_type', 'XGBoost'),
data.get('accuracy', ''),
data.get('precision', ''),
data.get('recall', ''),
data.get('f1_score', ''),
param_str
])
for info_file in glob.glob("model_train/*_info.json"):
with open(info_file, 'r') as f:
data = json.load(f)
# Support both 'accuracy' and 'test_accuracy'
acc = data.get('accuracy', data.get('test_accuracy', ''))
f1 = data.get('f1_score', '')
precision = data.get('precision', '')
recall = data.get('recall', '')
clf_rep = data.get('classification_report')
if isinstance(clf_rep, dict) and 'macro avg' in clf_rep:
if not f1:
f1 = clf_rep['macro avg'].get('f1-score', '')
if not precision:
precision = clf_rep['macro avg'].get('precision', '')
if not recall:
recall = clf_rep['macro avg'].get('recall', '')
if not acc and not f1:
continue
params = data.get('params', {})
param_str = f"estimators:{params.get('n_estimators')}, depth:{params.get('max_depth')}" if params else "N/A"
if data.get('model_type') == 'RandomForest_RealData':
param_str = "estimators:100, depth:15"
land_data.append([
data.get('model_type', ''),
acc,
precision,
recall,
f1,
param_str
])
if land_data:
print(tabulate(land_data, headers=["Model", "Accuracy", "Precision", "Recall", "F1-Score", "Parameters"], tablefmt="github"))
print("\n")
# 2. Xóa mây
print("### 2. Nhóm Xóa mây (Cloud Removal)")
cloud_data = []
for info_file in glob.glob("cloud_removal_model/*_info.json"):
with open(info_file, 'r') as f:
data = json.load(f)
cloud_data.append([
data.get('model_type', ''),
data.get('epoch', ''),
data.get('train_loss', ''),
data.get('val_loss', '')
])
if cloud_data:
print(tabulate(cloud_data, headers=["Model", "Epochs", "Train Loss", "Val Loss"], tablefmt="github"))
print("\n")
# 3. Dự báo NDVI
print("### 3. Nhóm Dự báo Thực vật (NDVI Forecasting)")
ndvi_data = []
for info_file in glob.glob("ndvi_forecast_model/*_info.json"):
with open(info_file, 'r') as f:
data = json.load(f)
ndvi_data.append([
data.get('model_type', ''),
data.get('rmse', ''),
data.get('mae', ''),
data.get('epoch', 'N/A')
])
if ndvi_data:
print(tabulate(ndvi_data, headers=["Model", "RMSE", "MAE", "Epochs"], tablefmt="github"))
print("\n")
.temp/inspect_cells_train.py
+7
View File
@@ -0,0 +1,7 @@
import json
nb = json.load(open('01.train_ODC.ipynb'))
for idx, cell in enumerate(nb['cells']):
if cell['cell_type'] == 'code':
print(f"Cell {idx}:")
print("".join(cell['source'][:3]))
print("-" * 20)
.temp/inspect_model_odc.py
+64
View File
@@ -0,0 +1,64 @@
"""
Inspect model_odc.joblib to see what it actually contains
"""
import joblib
from pathlib import Path
model_path = Path("model_train/model_odc.joblib")
if model_path.exists():
print("Loading model_odc.joblib...")
model_data = joblib.load(model_path)
print(f"\nModel type: {type(model_data)}")
print(f"Model class: {model_data.__class__.__name__}")
# Check if it's a dict
if isinstance(model_data, dict):
print(f"\nModel is a dict with keys: {model_data.keys()}")
model = model_data.get('model')
else:
model = model_data
print(f"\nActual model type: {type(model)}")
print(f"Actual model class: {model.__class__.__name__}")
# Try to get feature info
if hasattr(model, 'n_features_in_'):
print(f"\nn_features_in_: {model.n_features_in_}")
if hasattr(model, 'feature_names_in_'):
print(f"feature_names_in_: {model.feature_names_in_}")
# If it's a GridSearchCV
if hasattr(model, 'best_estimator_'):
print(f"\nThis is a GridSearchCV!")
print(f"Best estimator: {model.best_estimator_}")
best_est = model.best_estimator_
if hasattr(best_est, 'steps'):
print(f"\nPipeline steps:")
for step_name, step in best_est.steps:
print(f" - {step_name}: {step.__class__.__name__}")
if hasattr(step, 'n_features_in_'):
print(f" n_features_in_: {step.n_features_in_}")
# If it's a Pipeline
if hasattr(model, 'steps'):
print(f"\nThis is a Pipeline!")
print(f"Pipeline steps:")
for step_name, step in model.steps:
print(f" - {step_name}: {step.__class__.__name__}")
if hasattr(step, 'n_features_in_'):
print(f" n_features_in_: {step.n_features_in_}")
# Try to get booster for XGBoost
try:
if hasattr(model, 'get_booster'):
print(f"\nXGBoost num_features: {model.get_booster().num_features()}")
except:
pass
else:
print(f"Model file not found: {model_path}")
.temp/log_cnn.txt
+18
View File
@@ -0,0 +1,18 @@
🚀 BẮT ĐẦU HUẤN LUYỆN MÔ HÌNH CNN (GPU & CACHE)
Initializing FeatureExtractor (mode=extended)...
📦 Đang load cache: training_data_507bd2ba4ec0d3fe107839cbf73a7a7d.joblib...
✅ Loaded 632 samples từ cache!
⚡ Đã bỏ qua download위성 data (tiết kiệm thời gian)
[CACHE HIT] Using cached dataset with 632 samples
Training CNN model...
Building CNN model on cuda...
Training CNN model with PyTorch...
CNN Epoch 10/15, Loss: 1.2171
Evaluating model...
Generating classification report...
Saving model...
[MODEL MANAGER] Saving model to: model_train/model_cnn_auto.joblib
[MODEL MANAGER] Saving metadata to: model_train/model_cnn_auto_info.json
[MODEL MANAGER] Model saved successfully!
Training complete!
✅ Hoàn thành! Accuracy: 0.5748
.temp/log_dt.txt
+15
View File
@@ -0,0 +1,15 @@
🚀 BẮT ĐẦU HUẤN LUYỆN MÔ HÌNH DECISION TREE (GPU & CACHE)
Initializing FeatureExtractor (mode=extended)...
📦 Đang load cache: training_data_507bd2ba4ec0d3fe107839cbf73a7a7d.joblib...
✅ Loaded 632 samples từ cache!
⚡ Đã bỏ qua download위성 data (tiết kiệm thời gian)
[CACHE HIT] Using cached dataset with 632 samples
Training DECISION_TREE model...
Evaluating model...
Generating classification report...
Saving model...
[MODEL MANAGER] Saving model to: model_train/model_decision_tree_auto.joblib
[MODEL MANAGER] Saving metadata to: model_train/model_decision_tree_auto_info.json
[MODEL MANAGER] Model saved successfully!
Training complete!
✅ Hoàn thành! Accuracy: 0.5906
.temp/log_extract_fast.txt
+30991
View File
File diff suppressed because it is too large Load Diff
.temp/log_extract_fusion.txt
+3792
View File
File diff suppressed because it is too large Load Diff
.temp/log_lightgbm.txt
+2234
View File
File diff suppressed because it is too large Load Diff
.temp/log_mobilenet.txt
+30
View File
@@ -0,0 +1,30 @@
🚀 BẮT ĐẦU HUẤN LUYỆN MÔ HÌNH MOBILENET-LRASPP (GPU & CACHE)
Initializing FeatureExtractor (mode=extended)...
📦 Đang load cache: training_data_507bd2ba4ec0d3fe107839cbf73a7a7d.joblib...
✅ Loaded 632 samples từ cache!
⚡ Đã bỏ qua download위성 data (tiết kiệm thời gian)
[CACHE HIT] Using cached dataset with 632 samples
Training MOBILENET-LRASPP model...
Building MobileNetV3 + LR-ASPP model on cuda...
[MOBILENET] Class distribution: [ 48 89 3 86 74 38 117 50]
[MOBILENET] Class weights: [0.37418982 0.20181024 5.98703525 0.20885013 0.24271772 0.47266082
0.15351377 0.35922223]
Training MobileNetV3 + LR-ASPP model with PyTorch...
MobileNet Epoch 5/25, Train Loss: 1.0614, Val Loss: 1.3584, Val Acc: 40.16%, LR: 0.000800
[MOBILENET] Epoch 5/25 - Train Loss: 1.0614, Val Loss: 1.3584, Val Acc: 40.16%
MobileNet Epoch 10/25, Train Loss: 0.9054, Val Loss: 0.8582, Val Acc: 59.06%, LR: 0.000800
[MOBILENET] Epoch 10/25 - Train Loss: 0.9054, Val Loss: 0.8582, Val Acc: 59.06%
MobileNet Epoch 15/25, Train Loss: 0.7728, Val Loss: 0.8231, Val Acc: 61.42%, LR: 0.000800
[MOBILENET] Epoch 15/25 - Train Loss: 0.7728, Val Loss: 0.8231, Val Acc: 61.42%
MobileNet Epoch 20/25, Train Loss: 0.7125, Val Loss: 0.8783, Val Acc: 59.84%, LR: 0.000400
[MOBILENET] Epoch 20/25 - Train Loss: 0.7125, Val Loss: 0.8783, Val Acc: 59.84%
[MOBILENET] Early stopping at epoch 24 (best val loss: 0.8029)
MobileNet early stopped at epoch 24
Evaluating model...
Generating classification report...
Saving model...
[MODEL MANAGER] Saving model to: model_train/model_mobilenet-lraspp_auto.joblib
[MODEL MANAGER] Saving metadata to: model_train/model_mobilenet-lraspp_auto_info.json
[MODEL MANAGER] Model saved successfully!
Training complete!
✅ Hoàn thành! Accuracy: 0.5669
.temp/log_rf.txt
+15
View File
@@ -0,0 +1,15 @@
🚀 BẮT ĐẦU HUẤN LUYỆN MÔ HÌNH RANDOM FOREST (GPU & CACHE)
Initializing FeatureExtractor (mode=extended)...
📦 Đang load cache: training_data_507bd2ba4ec0d3fe107839cbf73a7a7d.joblib...
✅ Loaded 632 samples từ cache!
⚡ Đã bỏ qua download위성 data (tiết kiệm thời gian)
[CACHE HIT] Using cached dataset with 632 samples
Training RANDOM_FOREST model...
Evaluating model...
Generating classification report...
Saving model...
[MODEL MANAGER] Saving model to: model_train/model_random_forest_auto.joblib
[MODEL MANAGER] Saving metadata to: model_train/model_random_forest_auto_info.json
[MODEL MANAGER] Model saved successfully!
Training complete!
✅ Hoàn thành! Accuracy: 0.6142
.temp/log_svm.txt
+15
View File
@@ -0,0 +1,15 @@
🚀 BẮT ĐẦU HUẤN LUYỆN MÔ HÌNH SVM (GPU & CACHE)
Initializing FeatureExtractor (mode=extended)...
📦 Đang load cache: training_data_507bd2ba4ec0d3fe107839cbf73a7a7d.joblib...
✅ Loaded 632 samples từ cache!
⚡ Đã bỏ qua download위성 data (tiết kiệm thời gian)
[CACHE HIT] Using cached dataset with 632 samples
Training SVM model...
Evaluating model...
Generating classification report...
Saving model...
[MODEL MANAGER] Saving model to: model_train/model_svm_auto.joblib
[MODEL MANAGER] Saving metadata to: model_train/model_svm_auto_info.json
[MODEL MANAGER] Model saved successfully!
Training complete!
✅ Hoàn thành! Accuracy: 0.5827
.temp/log_swin.txt
+30
View File
@@ -0,0 +1,30 @@
🚀 BẮT ĐẦU HUẤN LUYỆN MÔ HÌNH SWIN-UNET (GPU & CACHE)
Initializing FeatureExtractor (mode=extended)...
📦 Đang load cache: training_data_507bd2ba4ec0d3fe107839cbf73a7a7d.joblib...
✅ Loaded 632 samples từ cache!
⚡ Đã bỏ qua download위성 data (tiết kiệm thời gian)
[CACHE HIT] Using cached dataset with 632 samples
Training SWIN-UNET model...
Building Swin-UNet model on cuda...
[SWIN-UNET] Class distribution: [ 48 89 3 86 74 38 117 50]
[SWIN-UNET] Class weights: [0.37418982 0.20181024 5.98703525 0.20885013 0.24271772 0.47266082
0.15351377 0.35922223]
Training Swin-UNet model with PyTorch (with class weights)...
Swin-UNet Epoch 5/40, Train Loss: 1.4096, Val Loss: 1.2804, Val Acc: 48.03%, LR: 0.000293
[SWIN-UNET] Epoch 5/40 - Train Loss: 1.4096, Val Loss: 1.2804, Val Acc: 48.03%
Swin-UNet Epoch 10/40, Train Loss: 1.1129, Val Loss: 1.2746, Val Acc: 57.48%, LR: 0.000271
[SWIN-UNET] Epoch 10/40 - Train Loss: 1.1129, Val Loss: 1.2746, Val Acc: 57.48%
Swin-UNet Epoch 15/40, Train Loss: 1.0479, Val Loss: 1.3126, Val Acc: 50.39%, LR: 0.000238
[SWIN-UNET] Epoch 15/40 - Train Loss: 1.0479, Val Loss: 1.3126, Val Acc: 50.39%
Swin-UNet Epoch 20/40, Train Loss: 0.9548, Val Loss: 1.1118, Val Acc: 52.76%, LR: 0.000196
[SWIN-UNET] Epoch 20/40 - Train Loss: 0.9548, Val Loss: 1.1118, Val Acc: 52.76%
[SWIN-UNET] Early stopping at epoch 22 (best val loss: 1.1096)
Swin-UNet early stopped at epoch 22
Evaluating model...
Generating classification report...
Saving model...
[MODEL MANAGER] Saving model to: model_train/model_swin-unet_auto.joblib
[MODEL MANAGER] Saving metadata to: model_train/model_swin-unet_auto_info.json
[MODEL MANAGER] Model saved successfully!
Training complete!
✅ Hoàn thành! Accuracy: 0.5354
.temp/log_train_2d.txt
+94521
View File
File diff suppressed because it is too large Load Diff
.temp/log_train_2d_final.txt
+38
View File
@@ -0,0 +1,38 @@
🚀 BẮT ĐẦU PIPELINE 2D PATCH-BASED & CLOUD REMOVAL
Loading 2D patches from dataset_cache/training_data_2d.joblib...
Training 2D CNN with Data Augmentation...
Epoch 1/150 - Loss: 2.2272 - Test Acc: 0.0752 🌟
Epoch 2/150 - Loss: 2.0843 - Test Acc: 0.0796 🌟
Epoch 4/150 - Loss: 2.0350 - Test Acc: 0.1372 🌟
Epoch 8/150 - Loss: 2.0447 - Test Acc: 0.1637 🌟
Epoch 10/150 - Loss: 2.0397 - Test Acc: 0.1372
Epoch 12/150 - Loss: 2.0353 - Test Acc: 0.2168 🌟
Epoch 20/150 - Loss: 2.0139 - Test Acc: 0.1372
Traceback (most recent call last):
File "/home/x79/remote-sensing/train_land_2d_patch.py", line 298, in <module>
main()
File "/home/x79/remote-sensing/train_land_2d_patch.py", line 294, in main
train_2d_model(X, y)
File "/home/x79/remote-sensing/train_land_2d_patch.py", line 219, in train_2d_model
for batch_X, batch_y in train_loader:
File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 725, in __next__
data = self._next_data()
File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 785, in _next_data
data = self._dataset_fetcher.fetch(index) # may raise StopIteration
File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 54, in fetch
data = [self.dataset[idx] for idx in possibly_batched_index]
File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 54, in <listcomp>
data = [self.dataset[idx] for idx in possibly_batched_index]
File "/home/x79/remote-sensing/train_land_2d_patch.py", line 192, in __getitem__
x = transform(x)
File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torchvision/transforms/transforms.py", line 95, in __call__
img = t(img)
File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl
return forward_call(*args, **kwargs)
File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torchvision/transforms/transforms.py", line 752, in forward
return F.vflip(img)
File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torchvision/transforms/functional.py", line 757, in vflip
def vflip(img: Tensor) -> Tensor:
KeyboardInterrupt
.temp/log_train_2d_temporal.txt
+18912
View File
File diff suppressed because it is too large Load Diff
.temp/log_train_2d_temporal_pretrained.txt
+31
View File
File diff suppressed because one or more lines are too long
.temp/log_train_v4.txt
+39
View File
@@ -0,0 +1,39 @@
🚀 V4: TÍCH HỢP RADAR SENTINEL-1 (32-CHANNELS FUSION)
============================================================
Clean FUSION data: (252, 32, 16, 16), 7 classes, [31, 38, 32, 49, 23, 75, 4]
============================================================
32-CHANNELS FUSION CNN
============================================================
Ep 1 Fusion-Acc=0.1961 🌟
Ep 3 Fusion-Acc=0.2745 🌟
Ep 4 Fusion-Acc=0.4706 🌟
Ep 5 Fusion-Acc=0.5490 🌟
Ep 6 Fusion-Acc=0.7059 🌟
Ep 7 Fusion-Acc=0.7451 🌟
Ep 8 Fusion-Acc=0.8431 🌟
Ep 15 Fusion-Acc=0.8627 🌟
✅ CNN Fusion best: 0.8627
============================================================
HYBRID FUSION: CNN embed + S1/S2 Rich features + XGBoost
============================================================
Extracted 2182 fusion features per sample
Final Feature Vector: (252, 2694)
✅ Hybrid Fusion Acc: 0.8627
Fold 1: 0.9412
Fold 2: 0.9020
Fold 3: 0.9200
Fold 4: 0.8800
Fold 5: 0.9000
✅ CV Mean: 0.9086 ± 0.0206
============================================================
📊 FINAL RESULTS V4 (WITH RADAR)
============================================================
✅ Hybrid Fusion CV: 0.9086
📈 CNN Fusion (32ch): 0.8627
📈 Hybrid Fusion (CNN+XGB): 0.8627
🏆 BEST: 0.9086
.temp/log_train_v5.txt
+18
View File
@@ -0,0 +1,18 @@
============================================================
HYBRID FUSION ENSEMBLE: CNN embed + S1/S2 Rich features + XGB/LGBM/ETC
============================================================
Final Feature Vector: (443, 2694)
Fold 1: 0.8876
Fold 2: 0.9438
Fold 3: 0.9438
Fold 4: 0.9659
Fold 5: 0.9432
✅ Ensemble CV Mean: 0.9369 ± 0.0261
============================================================
📊 FINAL RESULTS V5 (ENSEMBLE + RADAR)
============================================================
✅ Hybrid Fusion Ensemble CV: 0.9369
🏆 BEST: 0.9369
.temp/log_train_v6.txt
+21
View File
@@ -0,0 +1,21 @@
🚀 V6: EXHAUSTIVE HYPERPARAMETER TUNING
============================================================
Data: (443, 32, 16, 16), 7 classes, dist=[65, 55, 48, 72, 74, 124, 5]
--- Training Multi-Seed CNN Ensemble ---
Seed 42: CNN Acc = 0.8989
Seed 123: CNN Acc = 0.8652
Seed 777: CNN Acc = 0.8876
Multi-seed CNN embedding: (443, 1536)
Total features: (443, 3940)
============================================================
🔬 EXHAUSTIVE HYPERPARAMETER SEARCH
============================================================
🏆 XGB-deep: 0.9526 ± 0.0110 (folds: ['0.955', '0.933', '0.966', '0.955', '0.955'])
✅ XGB-shallow: 0.9436 ± 0.0173 (folds: ['0.944', '0.910', '0.955', '0.955', '0.955'])
🏆 XGB-balanced: 0.9504 ± 0.0113 (folds: ['0.944', '0.933', '0.955', '0.955', '0.966'])
✅ LGBM-tuned: 0.9458 ± 0.0149 (folds: ['0.955', '0.921', '0.966', '0.943', '0.943'])
✅ LGBM-conservative: 0.9481 ± 0.0152 (folds: ['0.944', '0.921', '0.966', '0.955', '0.955'])
🏆 ETC-deep: 0.9572 ± 0.0082 (folds: ['0.944', '0.955', '0.955', '0.966', '0.966'])
🏆 RF-tuned: 0.9549 ± 0.0099 (folds: ['0.944', '0.955', '0.944', '0.966', '0.966'])
.temp/log_tune_swin_unet.txt
+157
View File
@@ -0,0 +1,157 @@
🚀 BẮT ĐẦU TÌM KIẾM SIÊU THAM SỐ CHO SWIN-UNET
Loading data from dataset_cache/training_data_507bd2ba4ec0d3fe107839cbf73a7a7d.joblib...
Using device: cuda
[1/48] Training with params: {'embed_dim': 64, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 200}
Test Accuracy: 0.5984
🌟 NEW BEST ACCURACY: 0.5984
[2/48] Training with params: {'embed_dim': 64, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 500}
Test Accuracy: 0.6063
🌟 NEW BEST ACCURACY: 0.6063
[3/48] Training with params: {'embed_dim': 64, 'lr': 0.001, 'weight_decay': 0.001, 'epochs': 200}
Test Accuracy: 0.6142
🌟 NEW BEST ACCURACY: 0.6142
[4/48] Training with params: {'embed_dim': 64, 'lr': 0.001, 'weight_decay': 0.001, 'epochs': 500}
Test Accuracy: 0.6772
🌟 NEW BEST ACCURACY: 0.6772
[5/48] Training with params: {'embed_dim': 64, 'lr': 0.0005, 'weight_decay': 0.01, 'epochs': 200}
Test Accuracy: 0.5827
[6/48] Training with params: {'embed_dim': 64, 'lr': 0.0005, 'weight_decay': 0.01, 'epochs': 500}
Test Accuracy: 0.5984
[7/48] Training with params: {'embed_dim': 64, 'lr': 0.0005, 'weight_decay': 0.001, 'epochs': 200}
Test Accuracy: 0.5669
[8/48] Training with params: {'embed_dim': 64, 'lr': 0.0005, 'weight_decay': 0.001, 'epochs': 500}
Test Accuracy: 0.6142
[9/48] Training with params: {'embed_dim': 64, 'lr': 0.0001, 'weight_decay': 0.01, 'epochs': 200}
Test Accuracy: 0.6142
[10/48] Training with params: {'embed_dim': 64, 'lr': 0.0001, 'weight_decay': 0.01, 'epochs': 500}
Test Accuracy: 0.5118
[11/48] Training with params: {'embed_dim': 64, 'lr': 0.0001, 'weight_decay': 0.001, 'epochs': 200}
Test Accuracy: 0.5591
[12/48] Training with params: {'embed_dim': 64, 'lr': 0.0001, 'weight_decay': 0.001, 'epochs': 500}
Test Accuracy: 0.6063
[13/48] Training with params: {'embed_dim': 128, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 200}
Test Accuracy: 0.7008
🌟 NEW BEST ACCURACY: 0.7008
[14/48] Training with params: {'embed_dim': 128, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 500}
Test Accuracy: 0.6142
[15/48] Training with params: {'embed_dim': 128, 'lr': 0.001, 'weight_decay': 0.001, 'epochs': 200}
Test Accuracy: 0.6772
[16/48] Training with params: {'embed_dim': 128, 'lr': 0.001, 'weight_decay': 0.001, 'epochs': 500}
Test Accuracy: 0.6063
[17/48] Training with params: {'embed_dim': 128, 'lr': 0.0005, 'weight_decay': 0.01, 'epochs': 200}
Test Accuracy: 0.5906
[18/48] Training with params: {'embed_dim': 128, 'lr': 0.0005, 'weight_decay': 0.01, 'epochs': 500}
Test Accuracy: 0.6457
[19/48] Training with params: {'embed_dim': 128, 'lr': 0.0005, 'weight_decay': 0.001, 'epochs': 200}
Test Accuracy: 0.5906
[20/48] Training with params: {'embed_dim': 128, 'lr': 0.0005, 'weight_decay': 0.001, 'epochs': 500}
Test Accuracy: 0.5906
[21/48] Training with params: {'embed_dim': 128, 'lr': 0.0001, 'weight_decay': 0.01, 'epochs': 200}
Test Accuracy: 0.6220
[22/48] Training with params: {'embed_dim': 128, 'lr': 0.0001, 'weight_decay': 0.01, 'epochs': 500}
Test Accuracy: 0.6457
[23/48] Training with params: {'embed_dim': 128, 'lr': 0.0001, 'weight_decay': 0.001, 'epochs': 200}
Test Accuracy: 0.5984
[24/48] Training with params: {'embed_dim': 128, 'lr': 0.0001, 'weight_decay': 0.001, 'epochs': 500}
Test Accuracy: 0.6063
[25/48] Training with params: {'embed_dim': 256, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 200}
Test Accuracy: 0.7087
🌟 NEW BEST ACCURACY: 0.7087
[26/48] Training with params: {'embed_dim': 256, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 500}
Test Accuracy: 0.7323
🌟 NEW BEST ACCURACY: 0.7323
[27/48] Training with params: {'embed_dim': 256, 'lr': 0.001, 'weight_decay': 0.001, 'epochs': 200}
Test Accuracy: 0.6457
[28/48] Training with params: {'embed_dim': 256, 'lr': 0.001, 'weight_decay': 0.001, 'epochs': 500}
Test Accuracy: 0.6142
[29/48] Training with params: {'embed_dim': 256, 'lr': 0.0005, 'weight_decay': 0.01, 'epochs': 200}
Test Accuracy: 0.5984
[30/48] Training with params: {'embed_dim': 256, 'lr': 0.0005, 'weight_decay': 0.01, 'epochs': 500}
Test Accuracy: 0.5906
[31/48] Training with params: {'embed_dim': 256, 'lr': 0.0005, 'weight_decay': 0.001, 'epochs': 200}
Test Accuracy: 0.6142
[32/48] Training with params: {'embed_dim': 256, 'lr': 0.0005, 'weight_decay': 0.001, 'epochs': 500}
Test Accuracy: 0.7008
[33/48] Training with params: {'embed_dim': 256, 'lr': 0.0001, 'weight_decay': 0.01, 'epochs': 200}
Test Accuracy: 0.6299
[34/48] Training with params: {'embed_dim': 256, 'lr': 0.0001, 'weight_decay': 0.01, 'epochs': 500}
Test Accuracy: 0.6299
[35/48] Training with params: {'embed_dim': 256, 'lr': 0.0001, 'weight_decay': 0.001, 'epochs': 200}
Test Accuracy: 0.6378
[36/48] Training with params: {'embed_dim': 256, 'lr': 0.0001, 'weight_decay': 0.001, 'epochs': 500}
Test Accuracy: 0.6457
[37/48] Training with params: {'embed_dim': 512, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 200}
Test Accuracy: 0.6142
[38/48] Training with params: {'embed_dim': 512, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 500}
Test Accuracy: 0.6457
[39/48] Training with params: {'embed_dim': 512, 'lr': 0.001, 'weight_decay': 0.001, 'epochs': 200}
Test Accuracy: 0.6378
[40/48] Training with params: {'embed_dim': 512, 'lr': 0.001, 'weight_decay': 0.001, 'epochs': 500}
Test Accuracy: 0.6299
[41/48] Training with params: {'embed_dim': 512, 'lr': 0.0005, 'weight_decay': 0.01, 'epochs': 200}
Test Accuracy: 0.6378
[42/48] Training with params: {'embed_dim': 512, 'lr': 0.0005, 'weight_decay': 0.01, 'epochs': 500}
Test Accuracy: 0.6220
[43/48] Training with params: {'embed_dim': 512, 'lr': 0.0005, 'weight_decay': 0.001, 'epochs': 200}
Test Accuracy: 0.6142
[44/48] Training with params: {'embed_dim': 512, 'lr': 0.0005, 'weight_decay': 0.001, 'epochs': 500}
Test Accuracy: 0.7008
[45/48] Training with params: {'embed_dim': 512, 'lr': 0.0001, 'weight_decay': 0.01, 'epochs': 200}
Test Accuracy: 0.6457
[46/48] Training with params: {'embed_dim': 512, 'lr': 0.0001, 'weight_decay': 0.01, 'epochs': 500}
Test Accuracy: 0.7323
[47/48] Training with params: {'embed_dim': 512, 'lr': 0.0001, 'weight_decay': 0.001, 'epochs': 200}
Test Accuracy: 0.6693
[48/48] Training with params: {'embed_dim': 512, 'lr': 0.0001, 'weight_decay': 0.001, 'epochs': 500}
Test Accuracy: 0.6457
✅ Đã lưu mô hình tốt nhất (Acc: 0.7323) vào land_classification_model/model_swin-unet_optimized_95.joblib
Cấu hình tốt nhất: {'embed_dim': 256, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 500}
.temp/log_ultimate_95.txt
+117
View File
@@ -0,0 +1,117 @@
🚀 CHIẾN LƯỢC TOÀN DIỆN ĐẠT >95% ACCURACY
============================================================
Loaded data: X=(706, 24, 16, 16), y=(706,)
Labels unique: [-1 0 1 2 3 4 5 6]
After cleanup: X=(652, 24, 16, 16), y=(652,) (removed 54 bad samples)
Remapped labels: [0 1 2 3 4 5 6]
Class 0: 65 samples
Class 1: 52 samples
Class 2: 48 samples
Class 3: 72 samples
Class 4: 108 samples
Class 5: 219 samples
Class 6: 88 samples
============================================================
STRATEGY 5: Flat pixel features + XGBoost (sanity check)
============================================================
Flat features: (652, 6144)
✅ Flat XGBoost acc: 0.7328
============================================================
STRATEGY 1: Lightweight CNN (no upsampling)
============================================================
Device: cuda
Epoch 1/300 Loss=1.8379 Acc=0.0763 🌟
Epoch 2/300 Loss=1.5825 Acc=0.2824 🌟
Epoch 3/300 Loss=1.4412 Acc=0.5649 🌟
Epoch 4/300 Loss=1.3274 Acc=0.6336 🌟
Epoch 5/300 Loss=1.2893 Acc=0.6870 🌟
Epoch 8/300 Loss=1.2140 Acc=0.7099 🌟
Epoch 11/300 Loss=1.2224 Acc=0.7252 🌟
Epoch 14/300 Loss=1.1087 Acc=0.7328 🌟
Epoch 16/300 Loss=1.1081 Acc=0.7710 🌟
Epoch 18/300 Loss=1.1847 Acc=0.7939 🌟
Epoch 20/300 Loss=1.0316 Acc=0.7786 (patience=2)
Epoch 24/300 Loss=1.0465 Acc=0.8092 🌟
Epoch 26/300 Loss=0.9583 Acc=0.8397 🌟
Epoch 40/300 Loss=0.9361 Acc=0.8626 🌟
Epoch 60/300 Loss=0.9807 Acc=0.8092 (patience=20)
Epoch 80/300 Loss=0.9459 Acc=0.8015 (patience=40)
Epoch 100/300 Loss=0.8443 Acc=0.7939 (patience=60)
Early stop at epoch 100
✅ LightCNN best acc: 0.8626
============================================================
STRATEGY 2: Hybrid CNN embeddings + XGBoost
============================================================
CNN embeddings: (652, 256)
Extracted 316 rich features per sample
Combined features: (652, 572)
✅ Hybrid XGBoost acc: 0.8244
============================================================
STRATEGY 3: Rich Features + Stacking Ensemble
============================================================
Extracted 316 rich features per sample
XGBoost: 0.7939
LightGBM: 0.7786
ExtraTrees: 0.7863
RandomForest: 0.7710
GBM: 0.7710
[06:55:44] WARNING: /__w/xgboost/xgboost/src/learner.cc:782:
Parameters: { "use_label_encoder" } are not used.
[06:55:44] WARNING: /__w/xgboost/xgboost/src/learner.cc:782:
Parameters: { "use_label_encoder" } are not used.
[06:55:44] WARNING: /__w/xgboost/xgboost/src/learner.cc:782:
Parameters: { "use_label_encoder" } are not used.
[06:55:44] WARNING: /__w/xgboost/xgboost/src/learner.cc:782:
Parameters: { "use_label_encoder" } are not used.
[06:55:44] WARNING: /__w/xgboost/xgboost/src/learner.cc:782:
Parameters: { "use_label_encoder" } are not used.
[06:56:17] WARNING: /__w/xgboost/xgboost/src/common/error_msg.cc:62: Falling back to prediction using DMatrix due to mismatched devices. This might lead to higher memory usage and slower performance. XGBoost is running on: cuda:0, while the input data is on: cpu.
Potential solutions:
- Use a data structure that matches the device ordinal in the booster.
- Set the device for booster before call to inplace_predict.
This warning will only be shown once.
Stacking Ensemble: 0.7710
Voting Ensemble: 0.7786
✅ Best ensemble: XGBoost = 0.7939
Extracted 316 rich features per sample
============================================================
STRATEGY 4: 5-Fold Stratified Cross-Validation
============================================================
Fold 1: 0.7939
Fold 2: 0.8244
Fold 3: 0.7769
Fold 4: 0.8154
Fold 5: 0.7615
✅ CV Mean: 0.7944 ± 0.0234
============================================================
📊 TỔNG KẾT KẾT QUẢ
============================================================
📈 LightCNN: 0.8626
📈 Hybrid CNN+XGBoost: 0.8244
📈 CV Mean (XGBoost rich): 0.7944
📈 Ensemble XGBoost: 0.7939
📈 Ensemble ExtraTrees: 0.7863
📈 Ensemble LightGBM: 0.7786
📈 Ensemble Voting: 0.7786
📈 Ensemble RandomForest: 0.7710
📈 Ensemble GBM: 0.7710
📈 Ensemble Stacking: 0.7710
📈 Flat XGBoost (baseline): 0.7328
🏆 BEST: LightCNN = 0.8626
✅ Kết quả đã được lưu vào model_train/ultimate_results.json
⚠️ Chưa đạt 95%. Best = 0.8626. Cần thêm dữ liệu hoặc feature engineering.
.temp/log_ultimate_v2.txt
+69
View File
@@ -0,0 +1,69 @@
🚀 CHIẾN LƯỢC V2: TOÀN DIỆN ĐẠT >95%
============================================================
Clean data: (652, 24, 16, 16), 7 classes
============================================================
CNN + TTA (Test-Time Augmentation)
============================================================
Ep 1 Loss=1.6500 TTA-Acc=0.1450 🌟
Ep 2 Loss=1.4633 TTA-Acc=0.3511 🌟
Ep 3 Loss=1.3316 TTA-Acc=0.6336 🌟
Ep 4 Loss=1.2101 TTA-Acc=0.7252 🌟
Ep 6 Loss=1.2056 TTA-Acc=0.8015 🌟
Ep 13 Loss=1.1216 TTA-Acc=0.8244 🌟
Ep 23 Loss=0.9355 TTA-Acc=0.8321 🌟
Ep 30 Loss=0.9346 TTA-Acc=0.8092 (pat=7)
Ep 60 Loss=0.9658 TTA-Acc=0.7634 (pat=37)
Ep 69 Loss=0.8988 TTA-Acc=0.8397 🌟
Ep 74 Loss=0.9159 TTA-Acc=0.8550 🌟
Ep 90 Loss=0.8761 TTA-Acc=0.8168 (pat=16)
Ep 120 Loss=0.9473 TTA-Acc=0.8092 (pat=46)
Ep 139 Loss=0.9317 TTA-Acc=0.8626 🌟
Ep 150 Loss=0.7716 TTA-Acc=0.8397 (pat=11)
Ep 175 Loss=0.7432 TTA-Acc=0.8702 🌟
Ep 180 Loss=0.8290 TTA-Acc=0.8702 (pat=5)
Ep 210 Loss=0.8060 TTA-Acc=0.8626 (pat=35)
Ep 240 Loss=0.6980 TTA-Acc=0.8473 (pat=65)
Early stop ep 255
✅ CNN+TTA best: 0.8702
============================================================
RICH FEATURES V2 + ENSEMBLE
============================================================
Extracted 1713 features per sample
XGB: 0.7557
LGBM: 0.7634
ET: 0.7481
RF: 0.7557
Voting: 0.7634
5-Fold CV:
Fold 1: 0.7557
Fold 2: 0.7939
Fold 3: 0.7769
Fold 4: 0.8308
Fold 5: 0.8000
CV: 0.7915 ± 0.0249
============================================================
HYBRID V2: CNN embed + Rich features + XGBoost
============================================================
Extracted 1713 features per sample
Combined: (652, 2097)
✅ Hybrid V2: 0.8244
CV: 0.9142 ± 0.0194
============================================================
📊 KẾT QUẢ TỔNG HỢP V2
============================================================
✅ Hybrid CV: 0.9142
📈 CNN+TTA: 0.8702
📈 Hybrid V2: 0.8244
📈 Ens CV: 0.7915
📈 Ens_LGBM: 0.7634
📈 Ens_Vote: 0.7634
📈 Ens_XGB: 0.7557
📈 Ens_RF: 0.7557
📈 Ens_ET: 0.7481
🏆 BEST: Hybrid CV = 0.9142
.temp/log_ultimate_v3.txt
+27
View File
@@ -0,0 +1,27 @@
🚀 V3: MULTI-SEED ENSEMBLE + T0-ONLY + SELF-TRAINING
============================================================
Clean: (652, 24, 16, 16), 7 classes, [65, 52, 48, 72, 108, 219, 88]
============================================================
MULTI-SEED CNN ENSEMBLE (10 models)
============================================================
Seed 0: 0.8473
Seed 1: 0.8702
Seed 2: 0.8550
Seed 3: 0.8550
Seed 4: 0.8702
Seed 5: 0.8626
Seed 6: 0.8702
Seed 7: 0.8702
Seed 8: 0.8702
Seed 9: 0.8702
✅ 10-Model Ensemble TTA: 0.8473
============================================================
TIMESTEP-0-ONLY XGBoost (cleanest data)
============================================================
T0 valid: 539/652
Features: (539, 1638)
XGB t0: 0.6852
LGBM t0: 0.6296
ET t0: 0.6852
.temp/log_xgboost.txt
+15
View File
@@ -0,0 +1,15 @@
🚀 BẮT ĐẦU HUẤN LUYỆN MÔ HÌNH XGBOOST (GPU & CACHE)
Initializing FeatureExtractor (mode=extended)...
📦 Đang load cache: training_data_507bd2ba4ec0d3fe107839cbf73a7a7d.joblib...
✅ Loaded 632 samples từ cache!
⚡ Đã bỏ qua download위성 data (tiết kiệm thời gian)
[CACHE HIT] Using cached dataset with 632 samples
Training XGBOOST model...
Evaluating model...
Generating classification report...
Saving model...
[MODEL MANAGER] Saving model to: model_train/model_xgboost_auto.joblib
[MODEL MANAGER] Saving metadata to: model_train/model_xgboost_auto_info.json
[MODEL MANAGER] Model saved successfully!
Training complete!
✅ Hoàn thành! Accuracy: 0.6378
.temp/new_import_ODC.py
+659
View File
@@ -0,0 +1,659 @@
TEST_MODE = True
RESOLUTION = 1000 if TEST_MODE else 10
import matplotlib.pyplot as plt
# Common imports and settings
import os, sys
os.environ['USE_PYGEOS'] = '0'
os.environ["GDAL_HTTP_MAX_RETRY"] = "5"
os.environ["GDAL_HTTP_RETRY_DELAY"] = "2"
os.environ["GDAL_HTTP_CONNECTION_TIMEOUT"] = "10"
os.environ["GDAL_HTTP_TIMEOUT"] = "30"
os.environ["CPL_VSIL_CURL_ALLOWED_EXTENSIONS"] = ".tif,.tiff"
os.environ["GDAL_DISABLE_READDIR_ON_OPEN"] = "YES"
from IPython.display import Markdown
import pandas as pd
pd.set_option("display.max_rows", None)
import xarray as xr
# Datacube
import datacube
from datacube.utils.rio import configure_s3_access
from datacube.utils import masking
from datacube.utils.cog import write_cog
# removed deafrica_tools imports to avoid ipyleaflet error
# EASI defaults
easinotebooksrepo = '/home/x79/CSIROBoeingPhase4-Vietnam'
if easinotebooksrepo not in sys.path: sys.path.append(easinotebooksrepo)
from easi_tools import EasiDefaults, xarray_object_size, notebook_utils, unset_cachingproxy
# from easi_tools.load_s2l2a import load_s2l2a_with_offset
from dask.distributed import progress
# Data tools
import numpy as np
from datetime import datetime
# Datacube
from datacube.utils import masking # https://github.com/opendatacube/datacube-core/blob/develop/datacube/utils/masking.py
from odc.algo import enum_to_bool # https://github.com/opendatacube/odc-algo/blob/main/odc/algo/_masking.py
# removed xr_reproject
from datacube.utils.geometry import GeoBox, box # https://github.com/opendatacube/datacube-core/blob/develop/datacube/utils/geometry/_base.py
# Holoviews, Datashader and Bokeh
import hvplot.pandas
import hvplot.xarray
import holoviews as hv
import panel as pn
import colorcet as cc
import cartopy.crs as ccrs
from datashader import reductions
from holoviews import opts
from utils import load_data_geo
import rasterio
import rioxarray
# import geoviews as gv
# from holoviews.operation.datashader import rasterize
hv.extension('bokeh', logo=False)
from deafrica_tools.bandindices import calculate_indices
from xgboost import XGBClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, classification_report
from sklearn.preprocessing import LabelEncoder
from sklearn.pipeline import Pipeline
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from shapely.geometry import Point, Polygon
import geopandas as gpd
from pyproj import CRS
from matplotlib.colors import ListedColormap
from holoviews import opts
from datashader import reductions
from bokeh.models.tickers import FixedTicker
from rioxarray.merge import merge_arrays
from sklearn.preprocessing import PolynomialFeatures
from sklearn.linear_model import LinearRegression
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error, r2_score
import joblib
def load_data(dc, date_range, longtitude_range, latitude_range):
import os, hashlib
cache_dir = "dataset_cache"
os.makedirs(cache_dir, exist_ok=True)
key_str = f"s2_{date_range}_{longtitude_range}_{latitude_range}"
cache_key = hashlib.md5(key_str.encode()).hexdigest() + ".nc"
cache_path = os.path.join(cache_dir, cache_key)
if os.path.exists(cache_path):
print(f"✅ Loading cached S2 data from {cache_path}")
return xr.open_dataset(cache_path, engine='netcdf4')
product = 's2_l2a'
bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
import pystac_client
import planetary_computer
import odc.stac
catalog = pystac_client.Client.open(
"https://planetarycomputer.microsoft.com/api/stac/v1",
modifier=planetary_computer.sign_inplace,
)
search = catalog.search(
collections=["sentinel-2-l2a"],
bbox=bbox,
datetime=f"{date_range[0]}/{date_range[1]}",
)
items = list(search.items())
data = odc.stac.load(
items,
bands=["red", "nir", "SCL"],
bbox=bbox,
crs="EPSG:32648",
resolution=RESOLUTION,
chunks={"x": 2048, "y": 2048, "time": 1},
groupby="solar_day"
)
if "SCL" in data.data_vars:
data = data.rename({"SCL": "scl"})
print(f"💾 Caching S2 data to {cache_path}")
data = data.compute()
data.to_netcdf(cache_path, engine='netcdf4')
return data
def mask_clean(data):
# For Sentinel-2 L2A SCL:
# 2: Dark Area Pixels, 4: Vegetation, 5: Not Vegetated, 6: Water
good_pixel_mask = data['scl'].isin([2, 4, 5, 6])
data_layer_names = [x for x in data.data_vars if x != 'scl']
# Apply good pixel mask
result = data[data_layer_names].where(good_pixel_mask).persist()
return result
def fill_nan(ndvi, time_split):
if len(ndvi.time) == 0:
return ndvi
# If the total time duration is less than 90 days, skip seasonal splitting
try:
total_days = (ndvi.time[-1] - ndvi.time[0]).dt.days.item()
if total_days < 90:
return ndvi.bfill(dim="time").ffill(dim="time")
except Exception:
pass
rs = []
for times in time_split:
try:
tmp = ndvi.sel(time=times)
if len(tmp.time) == 0:
continue
fill_ds = tmp.bfill(dim='time').ffill(dim='time')
rs.append(fill_ds)
except Exception:
continue
if len(rs) == 0:
return ndvi.bfill(dim="time").ffill(dim="time")
merged_ndvi = xr.concat([i for i in rs], dim="time")
fill_m = merged_ndvi.bfill(dim="time")
fill_m = fill_m.ffill(dim="time")
return fill_m
def load_train_data(train_path):
train = load_data_geo(train_path)
return train
def load_sen1(bbox, time_range):
import os, hashlib
cache_dir = "dataset_cache"
os.makedirs(cache_dir, exist_ok=True)
key_str = f"s1_vh_vv_{bbox}_{time_range}"
cache_key_vh = hashlib.md5((key_str + "vh").encode()).hexdigest() + ".nc"
cache_key_vv = hashlib.md5((key_str + "vv").encode()).hexdigest() + ".nc"
cache_path_vh = os.path.join(cache_dir, cache_key_vh)
cache_path_vv = os.path.join(cache_dir, cache_key_vv)
if os.path.exists(cache_path_vh) and os.path.exists(cache_path_vv):
print(f"✅ Loading cached S1 data from {cache_path_vh} and {cache_path_vv}")
ds_vh = xr.open_dataset(cache_path_vh, engine='netcdf4')
ds_vv = xr.open_dataset(cache_path_vv, engine='netcdf4')
return ds_vh[list(ds_vh.data_vars)[0]], ds_vv[list(ds_vv.data_vars)[0]]
import pystac_client
import planetary_computer
import odc.stac
# Kết nối STAC Client
catalog = pystac_client.Client.open(
"https://planetarycomputer.microsoft.com/api/stac/v1",
modifier=planetary_computer.sign_inplace,
)
# Tìm kiếm Items
search = catalog.search(
collections=["sentinel-1-rtc"],
bbox=bbox,
datetime=time_range,
)
items = list(search.items())
# Tải dữ liệu thành xarray Dataset
ds_s1 = odc.stac.load(
items,
bands=["vv", "vh"],
bbox=bbox,
crs="EPSG:32648",
resolution=RESOLUTION,
chunks={"x": 2048, "y": 2048, "time": 1}
)
# Tính giá trị trung vị theo thời gian
ds_median = ds_s1.median(dim="time").compute()
vv = ds_median["vv"]
vh = ds_median["vh"]
# Thêm chiều 'band' để giống hệt rioxarray
vv = vv.expand_dims(dim="band")
vh = vh.expand_dims(dim="band")
# Phục hồi metadata về toạ độ
vv = vv.rio.write_crs("EPSG:32648")
vh = vh.rio.write_crs("EPSG:32648")
print(f"💾 Caching S1 data to {cache_dir}")
vh.to_netcdf(cache_path_vh, engine='netcdf4')
vv.to_netcdf(cache_path_vv, engine='netcdf4')
return vh, vv
def get_data_sen1_and_sen2(train, average_ndvi, dsvh, dsvv):
loaded_datasets = {}
for idx, point in train.iterrows():
key = f"point_{idx + 1}"
try:
ndvi_data = average_ndvi.sel(x=point.geometry.x, y=point.geometry.y, method='nearest').values
vh_data = dsvh.sel(x=point.geometry.x, y=point.geometry.y, method='nearest').values
vv_data = dsvv.sel(x=point.geometry.x, y=point.geometry.y, method='nearest').values
loaded_datasets[key] = {
"data": np.concatenate((ndvi_data, vh_data, vv_data)),
"label": point.HT_code
}
except Exception as e:
# loaded_datasets[key] = None
print(e)
return loaded_datasets
def split_train_data(train, label_mapping, datasets):
label_encoder = LabelEncoder()
# Fit and transform the labels
labels = train.Hientrang.values
numeric_labels = label_encoder.fit_transform([label_mapping[label] for label in labels])
X = []
x_new = []
lb_new = []
for k, v in datasets.items():
X.append(v)
for i in range(len(X)):
if X[i] is not None:
x_new.append(X[i]["data"])
lb_new.append(numeric_labels[i])
X_train, X_temp, y_train, y_temp= train_test_split(x_new, lb_new, test_size=0.4, random_state=42)
X_val, X_test, y_val, y_test = train_test_split(X_temp, y_temp, test_size=0.5, random_state=42)
return X_train, X_val, X_test, y_train, y_val, y_test
def train_with_rf(X_train, X_val, y_train, y_val):
# Takes 1-2 minutes to complete
# Tạo RandomForestClassifier mặc định để sử dụng làm mô hình ban đầu trong pipeline
base_model = XGBClassifier(tree_method="hist", device="cuda", random_state=42, n_jobs=-1)
# Tạo pipeline
pipeline = Pipeline([
# ('imputer', SimpleImputer(strategy='mean')),
('scaler', StandardScaler()),
('classifier', base_model),
])
# Thiết lập các tham số bạn muốn tối ưu hóa
param_grid = {
'classifier__n_estimators': [100, 300, 500, 700, 1000],
'classifier__max_depth': [6, 8, 10, 15, 20],
'classifier__learning_rate': [0.01, 0.1, 0.2],
}
# Sử dụng GridSearchCV để tìm bộ tham số tốt nhất
grid_search = GridSearchCV(pipeline, param_grid, cv=5, scoring='accuracy', n_jobs=-1)
grid_search.fit(X_train, y_train)
# In ra bộ tham số tốt nhất
best_params = grid_search.best_params_
print("Best Parameters:", best_params)
# Dự đoán trên tập kiểm tra
y_pred = grid_search.predict(X_val)
# Đánh giá kết quả
accuracy = accuracy_score(y_val, y_pred)
print(f"Accuracy: {round(accuracy, 2)*100} %")
return grid_search
def save_model(name_file, model, metadata=None, label_encoder=None):
"""
Save model với metadata để tương thích với ModelManager
Args:
name_file: Tên file model
model: Model object
metadata: Dict chứa thông tin về model (optional)
label_encoder: Label encoder (optional)
"""
from model_manager import get_model_manager
dir_save_model = "model_train"
if not os.path.exists(dir_save_model):
os.mkdir(dir_save_model)
# Nếu có metadata, sử dụng ModelManager
if metadata is not None:
model_manager = get_model_manager()
model_manager.save_model(
model=model,
metadata=metadata,
model_filename=name_file,
label_encoder=label_encoder
)
else:
# Legacy mode: save trực tiếp (backward compatibility)
model_data = {
'model': model,
'label_encoder': label_encoder
} if label_encoder is not None else model
joblib.dump(model_data, os.path.join(dir_save_model, name_file))
print(f"✅ Model saved: {name_file}")
if metadata:
print(f" - Type: {metadata.get('model_type', 'N/A')}")
print(f" - Features: {metadata.get('n_features', 'N/A')}")
print(f" - Accuracy: {metadata.get('test_accuracy', 'N/A')}")
def predict(model, data_crs, ndvi, vh, vv):
# Unpack model if it is wrapped in a dictionary (from ModelManager)
if isinstance(model, dict) and 'model' in model:
model = model['model']
data_predict = []
for i in range(ndvi.shape[1]):
ndvi_tmp = ndvi.isel(y=i).values
vh_data = vh.sel(y=ndvi.y.values[i], method='nearest').values
vv_data = vv.sel(y=ndvi.y.values[i], method='nearest').values
all_tmp = np.concatenate((ndvi_tmp, vh_data, vv_data), axis=0)
data_predict.extend(all_tmp.T)
y_pred = model.predict(data_predict)
final_label = y_pred.reshape(ndvi.y.shape[0], ndvi.x.shape[0])
final_xarray_save = xr.DataArray(final_label, dims=("y", "x"))
final_xarray_save = final_xarray_save.rio.write_crs(data_crs)
x_values = ndvi.x.values
y_values = ndvi.y.values
data_array = xr.DataArray(final_xarray_save,
coords={'x': x_values, 'y': y_values},
dims=['y', 'x'])
data_array = data_array.rio.write_crs(ndvi.rio.crs)
return data_array
def cut_according_shp(thuanhoa_path, average_ndvi, data_array):
gdf = gpd.read_file(thuanhoa_path)
gdf = gdf.to_crs(average_ndvi.rio.crs)
polygon_coords = list(gdf.geometry.values[0].exterior.coords)
polygon_coordinates = [(x, y) for x, y in polygon_coords]
geometries = [
{
'type': 'Polygon',
'coordinates': [polygon_coordinates]
}
]
region_result = data_array.rio.clip(geometries, data_array.rio.crs, drop=False)
region_result = region_result.where(region_result >= 0, float('nan'))
return region_result
def compare(KD_path, KetQuaPhanLoaiDat, CODE_MAP, HT_MAP):
gdf = gpd.read_file(KD_path, crs="EPSG:9209")
polygon = gdf.geometry.values
label = gdf.tenchu.values
ouput_image = rioxarray.open_rasterio(KetQuaPhanLoaiDat)
code_tq = HT_MAP["TQ"]["data"][0]
code_pnn = HT_MAP["PNN"]["data"][0]
result = {}
for key, values in HT_MAP.items():
print(f"process {key}")
array_list = []
for i in range(len(polygon)):
po = polygon[i]
lb = label[i]
code_lb = CODE_MAP.get(lb, code_tq)
try:
qr = ouput_image.rio.clip([po], "EPSG:9209")
if code_lb in values["data"]:
if code_lb == code_pnn:
qr = qr.where((qr != float(code_pnn)), np.nan)
# qr = qr.where((qr != 3.0), np.nan)
elif code_lb == code_tq:
qr = qr.where((qr != float(code_pnn)), np.nan)
qr = qr.where((qr != 3.0), np.nan)
else:
qr = qr.where(qr != float(code_lb), np.nan)
else:
qr.values[:, :, :] = np.nan
array_list.append(qr)
except Exception as e:
pass
result.update({key: array_list})
return result
def save_result(result, HT_MAP):
# cmap = ListedColormap(colors)
save_path = "ThuanHoa/KetQua"
if not os.path.exists(save_path):
os.mkdir(save_path)
for k, v in result.items():
rs = merge_arrays(v, nodata = np.nan)
rs.rio.to_raster(f"{save_path}/{k}.tif")
print(f"save {save_path}/{k}.tif")
# img = rs.plot(cmap=cmap, add_colorbar=False)
# cbar = plt.colorbar(img)
# cbar.ax.set_yticklabels(labels)
# plt.title(f'{HT_MAP[k]["name"]}')
# plt.axis('off')
# plt.show()
def load_data_sen1(dc, date_range, coordinates):
import os, hashlib
longtitude_range, latitude_range = coordinates
bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
cache_dir = "dataset_cache"
os.makedirs(cache_dir, exist_ok=True)
key_str = f"data_sen1_{date_range}_{bbox}"
cache_key_vh = hashlib.md5((key_str + "vh").encode()).hexdigest() + ".nc"
cache_key_vv = hashlib.md5((key_str + "vv").encode()).hexdigest() + ".nc"
cache_path_vh = os.path.join(cache_dir, cache_key_vh)
cache_path_vv = os.path.join(cache_dir, cache_key_vv)
if os.path.exists(cache_path_vh) and os.path.exists(cache_path_vv):
print(f"✅ Loading cached S1 (coord) data")
ds_vh = xr.open_dataset(cache_path_vh, engine='netcdf4')
ds_vv = xr.open_dataset(cache_path_vv, engine='netcdf4')
var_vh = [v for v in ds_vh.data_vars if v != 'spatial_ref'][0]
var_vv = [v for v in ds_vv.data_vars if v != 'spatial_ref'][0]
return ds_vh[var_vh], ds_vv[var_vv]
import pystac_client
import planetary_computer
import odc.stac
catalog = pystac_client.Client.open(
"https://planetarycomputer.microsoft.com/api/stac/v1",
modifier=planetary_computer.sign_inplace,
)
search = catalog.search(
collections=["sentinel-1-rtc"],
bbox=bbox,
datetime=f"{date_range[0]}/{date_range[1]}",
)
items = list(search.items())
data_sen1 = odc.stac.load(
items,
bands=["vv", "vh"],
bbox=bbox,
crs="EPSG:32648",
resolution=RESOLUTION,
chunks={"x": 2048, "y": 2048, "time": 1},
groupby="solar_day"
)
data_sen1 = data_sen1.compute()
dsvh = data_sen1.vh
dsvv = data_sen1.vv
print(f"💾 Caching S1 (coord) data")
dsvh.to_netcdf(cache_path_vh, engine='netcdf4')
dsvv.to_netcdf(cache_path_vv, engine='netcdf4')
return dsvh, dsvv
def calculate_average(data, time_pattern='1M'):
return data.resample(time=time_pattern).mean().persist()
def load_data_sen2(dc, date_range, coordinates):
import os, hashlib
longtitude_range, latitude_range = coordinates
bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
cache_dir = "dataset_cache"
os.makedirs(cache_dir, exist_ok=True)
key_str = f"data_sen2_{date_range}_{bbox}"
cache_key = hashlib.md5(key_str.encode()).hexdigest() + ".nc"
cache_path = os.path.join(cache_dir, cache_key)
if os.path.exists(cache_path):
print(f"✅ Loading cached S2 (coord) data from {cache_path}")
return xr.open_dataset(cache_path, engine='netcdf4')
import pystac_client
import planetary_computer
import odc.stac
catalog = pystac_client.Client.open(
"https://planetarycomputer.microsoft.com/api/stac/v1",
modifier=planetary_computer.sign_inplace,
)
search = catalog.search(
collections=["sentinel-2-l2a"],
bbox=bbox,
datetime=f"{date_range[0]}/{date_range[1]}",
)
items = list(search.items())
data = odc.stac.load(
items,
bands=["red", "nir", "SCL"],
bbox=bbox,
crs="EPSG:32648",
resolution=RESOLUTION,
chunks={"x": 2048, "y": 2048, "time": 1},
groupby="solar_day"
)
if "SCL" in data.data_vars:
data = data.rename({"SCL": "scl"})
data = data.compute()
print(f"💾 Caching S2 (coord) data to {cache_path}")
data.to_netcdf(cache_path, engine='netcdf4')
return data
def mask_cloud(data):
# For Sentinel-2 L2A SCL:
# 2: Dark Area Pixels, 4: Vegetation, 5: Not Vegetated, 6: Water
good_pixel_mask = data['scl'].isin([2, 4, 5, 6])
data_layer_names = [x for x in data.data_vars if x != 'scl']
# Apply good pixel mask
result = data[data_layer_names].where(good_pixel_mask).persist()
return result
def find_best_model(dataset):
X_train, X_val, y_train, y_val = dataset
# Tạo RandomForestClassifier mặc định để sử dụng làm mô hình ban đầu trong pipeline
base_model = XGBClassifier(tree_method="hist", device="cuda", random_state=42, n_jobs=-1)
# Tạo pipeline
pipeline = Pipeline([
# ('imputer', SimpleImputer(strategy='mean')),
('scaler', StandardScaler()),
('classifier', base_model),
])
# Thiết lập các tham số bạn muốn tối ưu hóa
param_grid = {
'classifier__n_estimators': [100, 300, 500, 700, 1000],
'classifier__max_depth': [6, 8, 10, 15, 20],
'classifier__learning_rate': [0.01, 0.1, 0.2],
}
# Sử dụng GridSearchCV để tìm bộ tham số tốt nhất
grid_search = GridSearchCV(pipeline, param_grid, cv=5, scoring='accuracy', n_jobs=-1)
grid_search.fit(X_train, y_train)
# In ra bộ tham số tốt nhất
best_params = grid_search.best_params_
print("Best Parameters:", best_params)
# Dự đoán trên tập kiểm tra
y_pred = grid_search.predict(X_val)
# Đánh giá kết quả
accuracy = accuracy_score(y_val, y_pred)
print(f"Accuracy: {round(accuracy, 2)*100} %")
return grid_search
def save_result_new(result, save_path, HT_MAP):
# cmap = ListedColormap(colors)
if not os.path.exists(save_path):
os.mkdir(save_path)
for k, v in result.items():
rs = merge_arrays(v, nodata = np.nan)
rs.rio.to_raster(f"{save_path}/{k}.tif")
print(f"save {save_path}/{k}.tif")
# img = rs.plot(cmap=cmap, add_colorbar=False)
# cbar = plt.colorbar(img)
# cbar.ax.set_yticklabels(labels)
# plt.title(f'{HT_MAP[k]["name"]}')
# plt.axis('off')
# plt.show()
def accuracy_test(test, data_array):
# cấu hình nhãn dữ liệu
label_mapping = {
"Lua tom": "0",
"Lua": "1",
"CHN": "2",
"CLN": "3",
"TS": "4",
"Song": "5",
"Dat xay dung": "6",
"Rung": "7"
}
chk = []
pred = []
dd = []
for idx, point in test.iterrows():
label = point.LULC
predict = data_array.sel(x=point.geometry.x, y=point.geometry.y, method='nearest').values
pred.append(label_mapping[label])
dd.append(str(predict))
chk.append(predict == int(label_mapping[label]))
test["code"] = pred
test["dd"] = dd
test["check"] = chk
path = "ThuanHoa/TestAccuracy"
if not os.path.exists(path):
os.mkdir(path)
test.to_file(f"{path}/result.shp")
percentage_true = np.mean(chk) * 100
print(f"độ chính xác: {percentage_true:.2f}%")
.temp/new_train.ipynb
+4427
View File
File diff suppressed because one or more lines are too long
.temp/patch_caching.py
+231
View File
@@ -0,0 +1,231 @@
import re
import os
with open('/home/x79/remote-sensing/new_import_ODC.py', 'r', encoding='utf-8') as f:
content = f.read()
# 1. load_data
load_data_replacement = """def load_data(dc, date_range, longtitude_range, latitude_range):
import os, hashlib
cache_dir = "dataset_cache"
os.makedirs(cache_dir, exist_ok=True)
key_str = f"s2_{date_range}_{longtitude_range}_{latitude_range}"
cache_key = hashlib.md5(key_str.encode()).hexdigest() + ".nc"
cache_path = os.path.join(cache_dir, cache_key)
if os.path.exists(cache_path):
print(f"✅ Loading cached S2 data from {cache_path}")
return xr.open_dataset(cache_path)
product = 's2_l2a'
bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
import pystac_client
import planetary_computer
import odc.stac
catalog = pystac_client.Client.open(
"https://planetarycomputer.microsoft.com/api/stac/v1",
modifier=planetary_computer.sign_inplace,
)
search = catalog.search(
collections=["sentinel-2-l2a"],
bbox=bbox,
datetime=f"{date_range[0]}/{date_range[1]}",
)
items = list(search.items())
data = odc.stac.load(
items,
bands=["red", "nir", "SCL"],
bbox=bbox,
crs="EPSG:32648",
resolution=RESOLUTION,
chunks={"x": 2048, "y": 2048, "time": 1},
groupby="solar_day"
)
if "SCL" in data.data_vars:
data = data.rename({"SCL": "scl"})
print(f"💾 Caching S2 data to {cache_path}")
data = data.compute()
data.to_netcdf(cache_path)
return data"""
content = re.sub(r'def load_data\(dc, date_range, longtitude_range, latitude_range\):.*?return data', load_data_replacement, content, flags=re.DOTALL)
# 2. load_sen1
load_sen1_replacement = """def load_sen1(bbox, time_range):
import os, hashlib
cache_dir = "dataset_cache"
os.makedirs(cache_dir, exist_ok=True)
key_str = f"s1_vh_vv_{bbox}_{time_range}"
cache_key_vh = hashlib.md5((key_str + "vh").encode()).hexdigest() + ".nc"
cache_key_vv = hashlib.md5((key_str + "vv").encode()).hexdigest() + ".nc"
cache_path_vh = os.path.join(cache_dir, cache_key_vh)
cache_path_vv = os.path.join(cache_dir, cache_key_vv)
if os.path.exists(cache_path_vh) and os.path.exists(cache_path_vv):
print(f"✅ Loading cached S1 data from {cache_path_vh} and {cache_path_vv}")
return xr.open_dataarray(cache_path_vh), xr.open_dataarray(cache_path_vv)
import pystac_client
import planetary_computer
import odc.stac
# Kết nối STAC Client
catalog = pystac_client.Client.open(
"https://planetarycomputer.microsoft.com/api/stac/v1",
modifier=planetary_computer.sign_inplace,
)
# Tìm kiếm Items
search = catalog.search(
collections=["sentinel-1-rtc"],
bbox=bbox,
datetime=time_range,
)
items = list(search.items())
# Tải dữ liệu thành xarray Dataset
ds_s1 = odc.stac.load(
items,
bands=["vv", "vh"],
bbox=bbox,
crs="EPSG:32648",
resolution=RESOLUTION,
chunks={"x": 2048, "y": 2048, "time": 1}
)
# Tính giá trị trung vị theo thời gian
ds_median = ds_s1.median(dim="time").compute()
vv = ds_median["vv"]
vh = ds_median["vh"]
# Thêm chiều 'band' để giống hệt rioxarray
vv = vv.expand_dims(dim="band")
vh = vh.expand_dims(dim="band")
# Phục hồi metadata về toạ độ
vv = vv.rio.write_crs("EPSG:32648")
vh = vh.rio.write_crs("EPSG:32648")
print(f"💾 Caching S1 data to {cache_dir}")
vh.to_netcdf(cache_path_vh)
vv.to_netcdf(cache_path_vv)
return vh, vv"""
content = re.sub(r'def load_sen1\(bbox, time_range\):.*?return vh, vv', load_sen1_replacement, content, flags=re.DOTALL)
# 3. load_data_sen1
load_data_sen1_replacement = """def load_data_sen1(dc, date_range, coordinates):
import os, hashlib
longtitude_range, latitude_range = coordinates
bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
cache_dir = "dataset_cache"
os.makedirs(cache_dir, exist_ok=True)
key_str = f"data_sen1_{date_range}_{bbox}"
cache_key_vh = hashlib.md5((key_str + "vh").encode()).hexdigest() + ".nc"
cache_key_vv = hashlib.md5((key_str + "vv").encode()).hexdigest() + ".nc"
cache_path_vh = os.path.join(cache_dir, cache_key_vh)
cache_path_vv = os.path.join(cache_dir, cache_key_vv)
if os.path.exists(cache_path_vh) and os.path.exists(cache_path_vv):
print(f"✅ Loading cached S1 (coord) data")
return xr.open_dataarray(cache_path_vh), xr.open_dataarray(cache_path_vv)
import pystac_client
import planetary_computer
import odc.stac
catalog = pystac_client.Client.open(
"https://planetarycomputer.microsoft.com/api/stac/v1",
modifier=planetary_computer.sign_inplace,
)
search = catalog.search(
collections=["sentinel-1-rtc"],
bbox=bbox,
datetime=f"{date_range[0]}/{date_range[1]}",
)
items = list(search.items())
data_sen1 = odc.stac.load(
items,
bands=["vv", "vh"],
bbox=bbox,
crs="EPSG:32648",
resolution=RESOLUTION,
chunks={"x": 2048, "y": 2048, "time": 1},
groupby="solar_day"
)
data_sen1 = data_sen1.compute()
dsvh = data_sen1.vh
dsvv = data_sen1.vv
print(f"💾 Caching S1 (coord) data")
dsvh.to_netcdf(cache_path_vh)
dsvv.to_netcdf(cache_path_vv)
return dsvh, dsvv"""
content = re.sub(r'def load_data_sen1\(dc, date_range, coordinates\):.*?return dsvh, dsvv', load_data_sen1_replacement, content, flags=re.DOTALL)
# 4. load_data_sen2
load_data_sen2_replacement = """def load_data_sen2(dc, date_range, coordinates):
import os, hashlib
longtitude_range, latitude_range = coordinates
bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
cache_dir = "dataset_cache"
os.makedirs(cache_dir, exist_ok=True)
key_str = f"data_sen2_{date_range}_{bbox}"
cache_key = hashlib.md5(key_str.encode()).hexdigest() + ".nc"
cache_path = os.path.join(cache_dir, cache_key)
if os.path.exists(cache_path):
print(f"✅ Loading cached S2 (coord) data from {cache_path}")
return xr.open_dataset(cache_path)
import pystac_client
import planetary_computer
import odc.stac
catalog = pystac_client.Client.open(
"https://planetarycomputer.microsoft.com/api/stac/v1",
modifier=planetary_computer.sign_inplace,
)
search = catalog.search(
collections=["sentinel-2-l2a"],
bbox=bbox,
datetime=f"{date_range[0]}/{date_range[1]}",
)
items = list(search.items())
data = odc.stac.load(
items,
bands=["red", "nir", "SCL"],
bbox=bbox,
crs="EPSG:32648",
resolution=RESOLUTION,
chunks={"x": 2048, "y": 2048, "time": 1},
groupby="solar_day"
)
if "SCL" in data.data_vars:
data = data.rename({"SCL": "scl"})
data = data.compute()
print(f"💾 Caching S2 (coord) data to {cache_path}")
data.to_netcdf(cache_path)
return data"""
content = re.sub(r'def load_data_sen2\(dc, date_range, coordinates\):.*?return data', load_data_sen2_replacement, content, flags=re.DOTALL)
with open('/home/x79/remote-sensing/new_import_ODC.py', 'w', encoding='utf-8') as f:
f.write(content)
print("Patching successful.")
.temp/patch_load_sen1.py
+29
View File
@@ -0,0 +1,29 @@
import json
import glob
def fix_load_sen1(file_path):
with open(file_path, 'r', encoding='utf-8') as f:
nb = json.load(f)
changed = False
for cell in nb.get('cells', []):
if cell.get('cell_type') == 'code':
source = cell.get('source', [])
for i, line in enumerate(source):
if 'load_sen1(name_vh, name_vv)' in line:
indent = line[:len(line) - len(line.lstrip())]
replacement = (
f"{indent}bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]\n"
f"{indent}time_range = f'{{date_range[0]}}/{{date_range[1]}}'\n"
f"{indent}{line.lstrip().replace('load_sen1(name_vh, name_vv)', 'load_sen1(bbox, time_range)')}"
)
source[i] = replacement
changed = True
if changed:
with open(file_path, 'w', encoding='utf-8') as f:
json.dump(nb, f, indent=1)
print(f"Patched load_sen1 in {file_path}")
for nb in glob.glob("*.ipynb"):
fix_load_sen1(nb)
.temp/patch_notebooks.py
+43
View File
@@ -0,0 +1,43 @@
import json
import glob
def patch_notebook(file_path):
with open(file_path, 'r', encoding='utf-8') as f:
nb = json.load(f)
changed = False
for cell in nb.get('cells', []):
if cell.get('cell_type') == 'code':
source = cell.get('source', [])
# Check if this cell should be fully commented out
full_source = ''.join(source)
if 'dc.load(' in full_source or 'ds.vv' in full_source:
for i in range(len(source)):
if not source[i].startswith('#'):
source[i] = '# ' + source[i]
changed = True
continue
# Otherwise, do line-by-line replacements
for i, line in enumerate(source):
if 'ST_training data_updated_1130points.shp' in line:
source[i] = line.replace('ST_training data_updated_1130points.shp', 'ST_training_data_updated_1130points.shp')
changed = True
if 'from new_import import *' in line:
source[i] = line.replace('from new_import import *', 'from new_import_ODC import *')
changed = True
if 'dc = datacube.Datacube()' in line:
source[i] = line.replace('dc = datacube.Datacube()', 'dc = None')
changed = True
if 'load_data(dc,' in line:
source[i] = line.replace('load_data(dc,', 'load_data(None,')
changed = True
if changed:
with open(file_path, 'w', encoding='utf-8') as f:
json.dump(nb, f, indent=1)
print(f"Patched {file_path}")
for nb in glob.glob("*.ipynb"):
patch_notebook(nb)
.temp/patch_rf_to_xgb.py
+75
View File
@@ -0,0 +1,75 @@
import json
import glob
import re
def patch_python_script(filepath):
try:
with open(filepath, 'r', encoding='utf-8') as f:
content = f.read()
original_content = content
# Replace imports
content = re.sub(r'from sklearn\.ensemble import RandomForestClassifier',
'from xgboost import XGBClassifier', content)
# Replace the model instantiations (for 01.train_ODC.py)
rf_pattern = re.compile(r'model\s*=\s*RandomForestClassifier\([^)]+\)', re.DOTALL)
xgb_replacement = """model = XGBClassifier(
n_estimators=200,
max_depth=30,
tree_method="hist",
device="cuda",
random_state=42,
n_jobs=-1,
verbosity=1
)"""
content = rf_pattern.sub(xgb_replacement, content)
if content != original_content:
with open(filepath, 'w', encoding='utf-8') as f:
f.write(content)
print(f"Patched {filepath}")
except Exception as e:
print(f"Error patching {filepath}: {e}")
def patch_notebook(filepath):
try:
with open(filepath, 'r', encoding='utf-8') as f:
nb = json.load(f)
changed = False
import_pattern = re.compile(r'from\s+sklearn\.ensemble\s+import\s+RandomForestClassifier')
inst_pattern = re.compile(r'RandomForestClassifier\([^)]*\)')
for cell in nb.get('cells', []):
if cell.get('cell_type') == 'code':
source = cell.get('source', [])
for i in range(len(source)):
if import_pattern.search(source[i]):
source[i] = import_pattern.sub('from xgboost import XGBClassifier', source[i])
changed = True
if inst_pattern.search(source[i]):
source[i] = inst_pattern.sub("XGBClassifier(tree_method='hist', device='cuda', random_state=42, n_jobs=-1)", source[i])
changed = True
if "'classifier__criterion': ['gini', 'entropy']" in source[i]:
source[i] = source[i].replace("'classifier__criterion': ['gini', 'entropy']",
"'classifier__learning_rate': [0.01, 0.1, 0.2]")
changed = True
if changed:
with open(filepath, 'w', encoding='utf-8') as f:
json.dump(nb, f, indent=1)
print(f"Patched {filepath}")
except Exception as e:
print(f"Error patching {filepath}: {e}")
if __name__ == "__main__":
patch_python_script("01.train_ODC.py")
patch_python_script("new_train.py")
for nb in glob.glob("*.ipynb"):
patch_notebook(nb)
.temp/processMask-test.ipynb
+4605
View File
File diff suppressed because one or more lines are too long
.temp/processMask-test.py
+272
View File
@@ -0,0 +1,272 @@
#!/usr/bin/env python
# coding: utf-8
# In[1]:
get_ipython().run_cell_magic('time', '', '%matplotlib inline\nfrom new_import import *\n')
# In[2]:
get_ipython().run_cell_magic('time', '', '# Dask gateway\ncluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))\ndc = datacube.Datacube()\n\n# Configure s3 access\nconfigure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n\nclient\n')
# In[3]:
## cấu hình thời gian lấy ảnh và tọa độ
# date_range = ('2022-09-01', '2023-10-01')
# longtitude_range = (105.86575, 105.94120)
# latitude_range = (9.65070, 9.69850)
date_range = ('2022-09-01', '2023-10-01')
longtitude_range = (105.5, 106.4)
latitude_range = (9.2, 10.0)
# In[4]:
## truy vấn ảnh vệ tinh sen2
data = load_data(dc, date_range, longtitude_range, latitude_range)
notebook_utils.heading(notebook_utils.xarray_object_size(data))
display(data)
# In[5]:
# Specify the start and end times
min_date = '2022-09-01' # Thời gian bắt đầu lấy data cho quá trình train
max_date = '2023-10-01' # Thời gian kết thúc lấy data cho quá trình train
# Just do 1 month for testing
# max_date = '2022-10-01' # Thời gian kết thúc lấy data cho quá trình train
# Specify a spatail region to search using latitude/longitude cooridinates
min_longitude, max_longitude = (105.5, 106.4)
min_latitude, max_latitude = (9.2, 10.0)
# Specify the product. In this case we want to use Sentinel-2 Level-2A data
product = 's2_l2a'
# Construct the search query dictionary
query = {
'product': product, # Product name
'x': (min_longitude, max_longitude), # "x" axis bounds
'y': (min_latitude, max_latitude), # "y" axis bounds
'time': (min_date, max_date), # Any parsable date strings
}
# In[6]:
# Most common CRS
native_crs = notebook_utils.mostcommon_crs(dc, query)
print(f'Most common native CRS: {native_crs}')
# In[7]:
# Specify the spectral band measurements we want to use for a classification algorithm
measurements = ['red', 'nir', 'scl']
load_params = {
'measurements': measurements, # Selected measurement or alias names
'output_crs': native_crs, # Target EPSG code
'resolution': (-10, 10), # Target resolution
'group_by': 'solar_day', # Scene grouping
'dask_chunks': {'x': 2048, 'y': 2048}, # Dask chunks
}
# In[8]:
get_ipython().run_cell_magic('time', '', '# The replacement "dc.load()" function for this product\ndata = load_s2l2a_with_offset(\n dc,\n query | load_params # Combine the two dicts that contain our search and load parameters\n)\n\n# This line prints the total size of the dataset hat was loaded\nnotebook_utils.heading(notebook_utils.xarray_object_size(data))\n\ndisplay(data)\n')
# In[9]:
# %%time
# # Tiến hành loại bỏ các vị trí bị mây ảnh hưởng
# result = mask_clean(data)
# progress(result)
# In[10]:
# Tiến hành tính toán NDVI
ds1 = calculate_indices(result, index='NDVI', satellite_mission='s2')
ndvi = ds1["NDVI"]
display(ndvi)
# In[11]:
get_ipython().run_cell_magic('time', '', "## tính ndvi theo tháng\naverage_ndvi = ndvi.resample(time='1M').mean().persist()\nprogress(average_ndvi)\n")
# In[12]:
# compute average_ndvi
average_ndvi = average_ndvi.compute()
# In[13]:
# cấu hình vh vv file
# name_vh = "ThuanHoa/ThuanHoa_VH.tif"
# name_vv = "ThuanHoa/ThuanHoa_VV.tif"
# load dữ liệu sen1
bbox = [105.5, 9.2, 106.4, 10.0]
time_range = '2022-09-01/2023-10-01'
# dsvh, dsvv = load_sen1(bbox, time_range)
name_vh = "vh-0922_0923-full_ST.tif"
name_vv = "vv-0922_0923-full_ST.tif"
if not os.path.exists(name_vh):
get_ipython().system('aws s3 cp s3://easi-asia-dc-data/staging/ctu/sentinel-1/vh-0922_0923-full_ST.tif vh-0922_0923-full_ST.tif')
if not os.path.exists(name_vv):
get_ipython().system('aws s3 cp s3://easi-asia-dc-data/staging/ctu/sentinel-1/vv-0922_0923-full_ST.tif vv-0922_0923-full_ST.tif')
bbox = [105.5, 9.2, 106.4, 10.0]
time_range = '2022-09-01/2023-10-01'
dsvh, dsvv = load_sen1(bbox, time_range)
# In[27]:
from sklearn.preprocessing import PolynomialFeatures
from sklearn.linear_model import LinearRegression
from sklearn.ensemble import RandomForestRegressor
# In[28]:
average_ndvi = average_ndvi[:, :7680, :8687]
mask = ~np.isnan(average_ndvi)
print(average_ndvi.shape)
print(dsvh.shape)
print(dsvv.shape)
print(mask.shape)
X_train = np.stack([dsvh.values[mask], dsvv.values[mask]], axis=1)
y_train = average_ndvi.values[mask]
# In[29]:
model = LinearRegression()
model.fit(X_train, y_train)
# In[30]:
X_pred = np.stack([dsvh.values[~mask], dsvv.values[~mask]], axis=1)
average_ndvi.values[~mask] = model.predict(X_pred)
# In[31]:
average_ndvi_filled = xr.DataArray(average_ndvi, dims=average_ndvi.dims)
# In[32]:
plt.imshow(average_ndvi_filled.isel(time=6))
# In[65]:
plt.imshow(average_ndvi.isel(time=6))
# In[33]:
train_path = "train/ST_training data_updated_1130points.shp"
# In[34]:
train = load_train_data(train_path)
# In[37]:
datasets = get_data_sen1_and_sen2(train, average_ndvi_filled, dsvh, dsvv)
# In[39]:
# cấu hình nhãn dữ liệu
label_mapping = {
"Lua tom": "0",
"Lua": "1",
"CHN": "2",
"CLN": "3",
"TS": "4",
"Song": "5",
"Dat xay dung": "6",
"Rung": "7"
}
# chia tập dữ liệu train, val, test
X_train, X_val, X_test, y_train, y_val, y_test = split_train_data(train, label_mapping, datasets)
# In[40]:
# Huấn luyện mô hình
grid_search = train_with_rf(X_train, X_val, y_train, y_val)
# In[41]:
# kiểm tra độ chính xác với tập test
y_pred_test = grid_search.predict(X_test)
test_accuracy = accuracy_score(y_test, y_pred_test)
print(f"Accuracy for test data {round(test_accuracy, 2)*100} %")
# In[42]:
# Lưu mô hình huấn luyện
save_model("model_new.joblib", grid_search)
# In[43]:
# đóng client, cluster
client.close()
cluster.close()
# In[ ]:
.temp/processMask.ipynb
+3300
View File
File diff suppressed because one or more lines are too long
.temp/processMask.py
+127
View File
@@ -0,0 +1,127 @@
#!/usr/bin/env python
# coding: utf-8
# In[1]:
get_ipython().run_cell_magic('time', '', '%matplotlib inline\nfrom new_import import *\n')
# In[2]:
get_ipython().run_cell_magic('time', '', '# Dask gateway\ncluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))\ndc = datacube.Datacube()\n\n# Configure s3 access\nconfigure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n\nclient\n')
# In[3]:
## cấu hình thời gian lấy ảnh và tọa độ
date_range = ('2022-09-01', '2023-10-01')
longtitude_range = (105.86575, 105.94120)
latitude_range = (9.65070, 9.69850)
# In[4]:
## truy vấn ảnh vệ tinh sen2
data = load_data(dc, date_range, longtitude_range, latitude_range)
notebook_utils.heading(notebook_utils.xarray_object_size(data))
display(data)
# In[5]:
get_ipython().run_cell_magic('time', '', '# Tiến hành loại bỏ các vị trí bị mây ảnh hưởng\nresult = mask_clean(data)\nprogress(result)\n')
# In[6]:
# Tiến hành tính toán NDVI
ds1 = calculate_indices(result, index='NDVI', satellite_mission='s2')
ndvi = ds1["NDVI"]
display(ndvi)
# In[17]:
get_ipython().run_cell_magic('time', '', "## tính ndvi theo tháng\naverage_ndvi = ndvi.resample(time='1M').mean().persist()\nprogress(average_ndvi)\n")
# In[18]:
# compute average_ndvi
average_ndvi = average_ndvi.compute()
# In[9]:
# cấu hình vh vv file
name_vh = "ThuanHoa/ThuanHoa_VH.tif"
name_vv = "ThuanHoa/ThuanHoa_VV.tif"
# load dữ liệu sen1
bbox = [105.5, 9.2, 106.4, 10.0]
time_range = '2022-09-01/2023-10-01'
dsvh, dsvv = load_sen1(bbox, time_range)
# In[10]:
from sklearn.preprocessing import PolynomialFeatures
from sklearn.linear_model import LinearRegression
from sklearn.ensemble import RandomForestRegressor
# In[11]:
mask = ~np.isnan(average_ndvi)
X_train = np.stack([dsvh.values[mask], dsvv.values[mask]], axis=1)
y_train = average_ndvi.values[mask]
# In[12]:
model = LinearRegression()
model.fit(X_train, y_train)
# In[13]:
X_pred = np.stack([dsvh.values[~mask], dsvv.values[~mask]], axis=1)
average_ndvi.values[~mask] = model.predict(X_pred)
# In[14]:
average_ndvi_filled = xr.DataArray(average_ndvi, dims=average_ndvi.dims)
# In[16]:
plt.imshow(average_ndvi_filled.isel(time=1))
# In[19]:
plt.imshow(average_ndvi.isel(time=1))
# In[ ]:
.temp/rebuild_notebook.py
+145
View File
@@ -0,0 +1,145 @@
import json
def get_content(filename):
with open(filename, "r", encoding="utf-8") as f:
return f.read()
fe_content = get_content("feature_extractor.py")
tm_content = get_content("train_module.py")
dt_content = get_content("train_land_decision_tree_gpu.py")
notebook = {
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Tải Dữ liệu Vệ tinh qua Colab (Self-contained)\n",
"Notebook này đã được nhúng sẵn toàn bộ mã nguồn xử lý. Bạn không cần upload cả thư mục `remote-sensing` nữa.\n",
"\n",
"## Bước 1: Upload Shapefile (BẮT BUỘC)\n",
"Mô hình cần biết các điểm tọa độ đất để lấy dữ liệu. Hãy nén thư mục `train/` trên máy bạn thành `train.zip` và chạy ô dưới đây để upload nó trực tiếp lên Colab."
]
},
{
"cell_type": "code",
"execution_count": None,
"metadata": {},
"outputs": [],
"source": [
"from google.colab import files\n",
"import os\n",
"\n",
"print(\"Hãy chọn file train.zip từ máy tính của bạn:\")\n",
"uploaded = files.upload()\n",
"\n",
"if \"train.zip\" in uploaded:\n",
" !unzip -q -o train.zip -d /content/train_tmp/\n",
" # Move the extracted files directly to /content/train/\n",
" !mkdir -p /content/train\n",
" !mv /content/train_tmp/*/* /content/train/ 2>/dev/null || mv /content/train_tmp/* /content/train/\n",
" print(\"Đã giải nén shapefile thành công vào thư mục /content/train/\")\n",
"else:\n",
" print(\"LỖI: Bạn chưa upload file có tên là train.zip!\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Bước 2: Cài đặt thư viện & Tạo môi trường"
]
},
{
"cell_type": "code",
"execution_count": None,
"metadata": {},
"outputs": [],
"source": [
"!pip install planetary-computer pystac-client odc-stac geopandas rasterio xarray joblib scikit-learn xgboost lightgbm"
]
},
{
"cell_type": "code",
"execution_count": None,
"metadata": {},
"outputs": [],
"source": [
"%%writefile feature_extractor.py\n" + fe_content
]
},
{
"cell_type": "code",
"execution_count": None,
"metadata": {},
"outputs": [],
"source": [
"%%writefile train_module.py\n" + tm_content
]
},
{
"cell_type": "code",
"execution_count": None,
"metadata": {},
"outputs": [],
"source": [
"%%writefile train_land_decision_tree_gpu.py\n" + dt_content
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Bước 3: Chạy tiến trình tải ảnh vệ tinh và tạo Cache"
]
},
{
"cell_type": "code",
"execution_count": None,
"metadata": {},
"outputs": [],
"source": [
"!python train_land_decision_tree_gpu.py"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Bước 4: Tải file Cache về máy"
]
},
{
"cell_type": "code",
"execution_count": None,
"metadata": {},
"outputs": [],
"source": [
"from google.colab import files\n",
"import glob\n",
"\n",
"cache_files = glob.glob(\"dataset_cache/*.joblib\")\n",
"if cache_files:\n",
" latest_cache = max(cache_files, key=os.path.getctime)\n",
" print(f\"Đang tải file {latest_cache} về máy...\")\n",
" files.download(latest_cache)\n",
"else:\n",
" print(\"Chưa tìm thấy file cache. Hãy chắc chắn bước 3 đã chạy thành công!\")"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
with open("Download_Cache_Colab.ipynb", "w", encoding="utf-8") as f:
json.dump(notebook, f, indent=1, ensure_ascii=False)
print("Notebook updated successfully!")
.temp/remote-sensing.zip:Zone.Identifier
BIN
View File
Binary file not shown.
.temp/remove_datacube.py
+35
View File
@@ -0,0 +1,35 @@
import json
import glob
def fix_notebook(file_path):
with open(file_path, 'r', encoding='utf-8') as f:
nb = json.load(f)
changed = False
for cell in nb.get('cells', []):
if cell.get('cell_type') == 'code':
source = cell.get('source', [])
if isinstance(source, list):
for i, line in enumerate(source):
if "dc = datacube.Datacube()" in line:
source[i] = "dc = None\n"
changed = True
if "ds = dc.load(" in line:
source[i] = "ds = None\n"
changed = True
if "data = dc.load(" in line:
source[i] = "data = None\n"
changed = True
# If ds is None, ds.vv will fail
if "vv_data = ds.vv" in line:
source[i] = "vv_data = None\n"
changed = True
if "notebook_utils.xarray_object_size(ds)" in line:
source[i] = line.replace("notebook_utils.xarray_object_size(ds)", "'ds is None'")
changed = True
if changed:
with open(file_path, 'w', encoding='utf-8') as f:
json.dump(nb, f, indent=1)
print(f"Removed datacube from {file_path}")
for nb in glob.glob("*.ipynb"):
fix_notebook(nb)
.temp/run_mock_parallel_and_compare.py
+110
View File
@@ -0,0 +1,110 @@
import os
import glob
import json
import subprocess
import time
from tabulate import tabulate
scripts = [
"train_land_randomforest.py",
"train_cloud_cnn.py",
"train_cloud_swin_unet.py",
"train_ndvi_statistical.py",
"train_ndvi_lstm_gru.py",
"train_ndvi_convlstm.py",
"train_ndvi_hybrid_physics.py",
"train_ndvi_ensemble.py"
]
print("🚀 Đang khởi chạy song song tất cả các mô hình...")
processes = []
for script in scripts:
if os.path.exists(script):
cmd = f"source /home/x79/miniconda3/etc/profile.d/conda.sh && conda activate env_01 && python {script}"
p = subprocess.Popen(["bash", "-c", cmd], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
processes.append((script, p))
for script, p in processes:
p.wait()
print("✅ Đã chạy xong tất cả các mô hình!\n")
print("📊 BẢNG SO SÁNH KẾT QUẢ CÁC MÔ HÌNH\n")
# 1. Phân loại đất
print("### 1. Nhóm Phân loại Lớp phủ (Land Classification)")
land_data = []
# Đọc XGBoost từ thư mục gốc
if os.path.exists("model_xgboost_info.json"):
with open("model_xgboost_info.json", 'r') as f:
data = json.load(f)
params = data.get('params', {})
param_str = f"estimators:{params.get('n_estimators')}, depth:{params.get('max_depth')}" if params else "N/A"
land_data.append([
data.get('model_type', 'XGBoost'),
data.get('accuracy', ''),
data.get('precision', ''),
data.get('recall', ''),
data.get('f1_score', ''),
param_str
])
# Đọc các model khác trong model_train
for info_file in glob.glob("model_train/*_info.json"):
with open(info_file, 'r') as f:
data = json.load(f)
# Chỉ lấy các model có độ chính xác (để lọc model rác/cũ)
if 'accuracy' not in data and 'f1_score' not in data:
continue
params = data.get('params', {})
param_str = f"estimators:{params.get('n_estimators')}, depth:{params.get('max_depth')}" if params else "N/A"
# Fallback for Random Forest
if data.get('model_type') == 'RandomForest_RealData':
param_str = "estimators:100, depth:15"
land_data.append([
data.get('model_type', ''),
data.get('accuracy', ''),
data.get('precision', ''),
data.get('recall', ''),
data.get('f1_score', ''),
param_str
])
if land_data:
print(tabulate(land_data, headers=["Model", "Accuracy", "Precision", "Recall", "F1-Score", "Parameters"], tablefmt="github"))
print("\n")
# 2. Xóa mây
print("### 2. Nhóm Xóa mây (Cloud Removal)")
cloud_data = []
for info_file in glob.glob("cloud_removal_model/*_info.json"):
with open(info_file, 'r') as f:
data = json.load(f)
cloud_data.append([
data.get('model_type', ''),
data.get('epoch', ''),
data.get('train_loss', ''),
data.get('val_loss', '')
])
if cloud_data:
print(tabulate(cloud_data, headers=["Model", "Epochs", "Train Loss", "Val Loss"], tablefmt="github"))
print("\n")
# 3. Dự báo NDVI
print("### 3. Nhóm Dự báo Thực vật (NDVI Forecasting)")
ndvi_data = []
for info_file in glob.glob("ndvi_forecast_model/*_info.json"):
with open(info_file, 'r') as f:
data = json.load(f)
ndvi_data.append([
data.get('model_type', ''),
data.get('rmse', ''),
data.get('mae', ''),
data.get('epoch', 'N/A')
])
if ndvi_data:
print(tabulate(ndvi_data, headers=["Model", "RMSE", "MAE", "Epochs"], tablefmt="github"))
print("\n")
.temp/test_VH_VV.ipynb
+2368
View File
File diff suppressed because one or more lines are too long
.temp/test_VH_VV.py
+72
View File
@@ -0,0 +1,72 @@
#!/usr/bin/env python
# coding: utf-8
# In[2]:
get_ipython().run_line_magic('matplotlib', 'inline')
from new_import import *
# Dask gateway
cluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))
dc = datacube.Datacube()
# Configure s3 access
configure_s3_access(aws_unsigned=False, requester_pays=True, client=client)
# In[3]:
ds = dc.load(
product="sentinel1_grd_gamma0_20m",
x=(105.5, 106.4),
y=(9.2, 10.0),
time=("2022-09-01", "2023-10-01"),
measurements=["vv", "vh"],
output_crs="EPSG:32648",
resolution=(-10,10),
dask_chunks={"x":2048, "y":2048},
skip_broken_datasets=True,
group_by="solar_day"
)
notebook_utils.heading(notebook_utils.xarray_object_size(ds))
ds
# In[18]:
vh = ds.vh.resample(time='1M').mean().persist()
vh = vh.compute()
vv = ds.vv.resample(time='1M').mean().persist()
vv = vv.compute()
# In[28]:
vv.min()
# In[33]:
import matplotlib.pyplot as plt
# Plot the data
plt.imshow(vh.isel(time=0), cmap='viridis', vmin=0, vmax=1)
plt.colorbar() # Add colorbar for reference
plt.show()
# In[ ]:
.temp/test_bug.py
+33
View File
@@ -0,0 +1,33 @@
import torch
import numpy as np
device = "cpu"
input_array = np.zeros((4, 16, 16), dtype=np.float32)
input_tensor = torch.from_numpy(input_array).unsqueeze(0).to(device)
print("Before pad:", input_tensor.shape)
from train_cloud_removal import UNet
model = UNet(in_channels=6, out_channels=4).to(device)
if hasattr(model, 'inc') and hasattr(model.inc.double_conv[0], 'in_channels'):
expected_channels = model.inc.double_conv[0].in_channels
elif hasattr(model, 'conv1') and hasattr(model.conv1, 'in_channels'):
expected_channels = model.conv1.in_channels
else:
expected_channels = list(model.parameters())[0].shape[1]
print("Expected channels:", expected_channels)
if expected_channels > input_tensor.shape[1]:
pad_channels = expected_channels - input_tensor.shape[1]
padding = torch.zeros(1, pad_channels, *input_tensor.shape[2:]).to(device)
input_tensor = torch.cat([input_tensor, padding], dim=1)
print("After pad:", input_tensor.shape)
try:
model(input_tensor)
print("Success!")
except Exception as e:
print("Error:", e)
.temp/test_caching.py
+25
View File
@@ -0,0 +1,25 @@
import os
import sys
sys.path.insert(0, os.getcwd())
import new_import_ODC
import time
import xarray as xr
# Mock minimal params to test load_data
date_range = ('2023-01-01', '2023-01-31')
longtitude_range = (105.0, 105.1)
latitude_range = (9.5, 9.6)
print("--- First Call (Downloading & Caching) ---")
start = time.time()
data1 = new_import_ODC.load_data(None, date_range, longtitude_range, latitude_range)
end = time.time()
print(f"Time taken: {end - start:.2f}s")
print("--- Second Call (Loading from Cache) ---")
start = time.time()
data2 = new_import_ODC.load_data(None, date_range, longtitude_range, latitude_range)
end = time.time()
print(f"Time taken: {end - start:.2f}s")
print("✅ Test completed")
.temp/test_checkpoint.py
+7
View File
@@ -0,0 +1,7 @@
import torch
checkpoint = torch.load('cloud_removal_model/cloud_removal_unet_best.pth', map_location='cpu')
print(checkpoint.keys())
print("in_channels in checkpoint:", 'in_channels' in checkpoint)
if 'in_channels' in checkpoint:
print(checkpoint['in_channels'])
print("Shape of inc.double_conv.0.weight:", checkpoint['model_state_dict']['inc.double_conv.0.weight'].shape)
.temp/test_cloud.py
+4
View File
@@ -0,0 +1,4 @@
import torch
from cloud_removal import DeepInpaintingStrategy
cloud_remover = DeepInpaintingStrategy()
print("Model channels:", list(cloud_remover.model.parameters())[0].shape[1])
.temp/test_cloud2.py
+10
View File
@@ -0,0 +1,10 @@
from cloud_removal import DeepInpaintingStrategy
import torch
import numpy as np
cr = DeepInpaintingStrategy(model_path="cloud_removal_model/cloud_removal_unet_best.pth")
if cr.model is not None:
expected = list(cr.model.parameters())[0].shape[1]
print("Expected channels:", expected)
else:
print("Failed to load model")
.temp/test_cloud_removal.py
+194
View File
@@ -0,0 +1,194 @@
"""
Test script for cloud_removal module
Kiểm tra các phương pháp xử lý mây
"""
import numpy as np
import xarray as xr
from cloud_removal import (
process_cloud_removal,
get_available_methods,
compare_methods
)
def create_mock_s2_data():
"""Tạo mock Sentinel-2 data để test"""
# Create synthetic data: 5 time steps, 100x100 pixels
np.random.seed(42)
time_steps = 5
y_size = 100
x_size = 100
# Create bands
bands = {}
for band in ["B02", "B03", "B04", "B08", "B11"]:
# Random reflectance values
data = np.random.rand(time_steps, y_size, x_size) * 0.3 + 0.1
bands[band] = (["time", "y", "x"], data)
# Create SCL (Scene Classification Layer)
# Mostly vegetation (4), with some clouds
scl_data = np.full((time_steps, y_size, x_size), 4, dtype=np.uint8)
# Add clouds (class 9) in random locations
for t in range(time_steps):
# Random cloud patches
n_clouds = np.random.randint(5, 15)
for _ in range(n_clouds):
y_start = np.random.randint(0, y_size - 20)
x_start = np.random.randint(0, x_size - 20)
cloud_height = np.random.randint(10, 20)
cloud_width = np.random.randint(10, 20)
scl_data[t, y_start:y_start+cloud_height, x_start:x_start+cloud_width] = 9
bands["SCL"] = (["time", "y", "x"], scl_data)
# Create xarray Dataset
ds = xr.Dataset(
bands,
coords={
"time": np.arange(time_steps),
"y": np.arange(y_size),
"x": np.arange(x_size)
}
)
return ds
def test_available_methods():
"""Test lấy danh sách methods"""
print("=" * 60)
print("TEST: Get Available Methods")
print("=" * 60)
methods = get_available_methods()
print(f"\nFound {len(methods)} methods:")
for method, description in methods.items():
print(f" - {method:20s}: {description}")
print("\n✅ Test passed!")
def test_single_method(method_name="classic"):
"""Test một method cụ thể"""
print("\n" + "=" * 60)
print(f"TEST: Cloud Removal Method '{method_name}'")
print("=" * 60)
# Create mock data
s2_data = create_mock_s2_data()
print(f"\nMock data created: {dict(s2_data.dims)}")
# Process clouds
cleaned_data, metadata = process_cloud_removal(
s2_data=s2_data,
method=method_name,
verbose=True
)
# Check results
print(f"\nMetadata:")
print(f" - Method: {metadata['method']}")
print(f" - Cloud coverage: {metadata['cloud_coverage_percent']:.1f}%")
print(f" - Masked pixels: {metadata['masked_pixels']:,}/{metadata['total_pixels']:,}")
print(f" - Steps applied: {', '.join(metadata['steps_applied'])}")
# Verify no NaN remaining
nan_count = 0
for band in cleaned_data.data_vars:
if band != "SCL":
nan_count += np.isnan(cleaned_data[band].values).sum()
print(f"\nRemaining NaN pixels: {nan_count}")
if nan_count == 0:
print("✅ Test passed - no NaN remaining!")
else:
print(f"⚠️ Warning - {nan_count} NaN pixels remaining")
def test_comparison():
"""Test so sánh nhiều methods"""
print("\n" + "=" * 60)
print("TEST: Compare Multiple Methods")
print("=" * 60)
# Create mock data
s2_data = create_mock_s2_data()
# Compare methods
methods_to_test = ["classic", "temporal_only", "median_composite", "ml_knn"]
print(f"\nComparing {len(methods_to_test)} methods...")
results = compare_methods(s2_data, methods=methods_to_test)
# Print summary
print("\n" + "-" * 60)
print(f"{'Method':<20} {'Success':<10} {'NaN %':<10} {'Steps'}")
print("-" * 60)
for method, result in results.items():
if result['success']:
nan_pct = result['remaining_nan_percent']
steps = ', '.join(result['metadata']['steps_applied'][:2]) # First 2 steps
print(f"{method:<20} {'✅':<10} {nan_pct:>6.2f}% {steps}")
else:
print(f"{method:<20} {'❌':<10} {'ERROR':<10} {result['error']}")
print("-" * 60)
print("\n✅ Comparison test completed!")
def test_edge_cases():
"""Test các trường hợp đặc biệt"""
print("\n" + "=" * 60)
print("TEST: Edge Cases")
print("=" * 60)
# Case 1: No SCL band
print("\n1. Testing without SCL band...")
s2_data = create_mock_s2_data()
s2_data_no_scl = s2_data.drop_vars("SCL")
cleaned, meta = process_cloud_removal(s2_data_no_scl, method="classic", verbose=False)
print(f" Result: {meta.get('warning', 'OK')}")
# Case 2: 100% cloud coverage
print("\n2. Testing with 100% cloud coverage...")
s2_data_full_cloud = create_mock_s2_data()
s2_data_full_cloud["SCL"][:] = 9 # All clouds
cleaned, meta = process_cloud_removal(s2_data_full_cloud, method="classic", verbose=False)
print(f" Cloud coverage: {meta['cloud_coverage_percent']:.1f}%")
# Case 3: No clouds
print("\n3. Testing with no clouds...")
s2_data_clear = create_mock_s2_data()
s2_data_clear["SCL"][:] = 4 # All vegetation
cleaned, meta = process_cloud_removal(s2_data_clear, method="classic", verbose=False)
print(f" Cloud coverage: {meta['cloud_coverage_percent']:.1f}%")
print("\n✅ Edge case tests passed!")
if __name__ == "__main__":
print("\n" + "🌥️ CLOUD REMOVAL MODULE TESTS 🌥️ ".center(60, "="))
print()
# Run tests
test_available_methods()
test_single_method("classic")
test_single_method("hybrid")
test_comparison()
test_edge_cases()
print("\n" + "=" * 60)
print("ALL TESTS COMPLETED!")
print("=" * 60)
print("\nModule is ready to use. Available methods:")
for method, desc in get_available_methods().items():
print(f" • {method}")
.temp/test_cloud_training.py
+28
View File
@@ -0,0 +1,28 @@
"""
Script test nhanh cho cloud removal training
"""
import sys
from pathlib import Path
# Add winter_dataset to path
sys.path.insert(0, str(Path(__file__).parent / "winter_dataset"))
from train_cloud_removal import train_cloud_removal_model
if __name__ == "__main__":
print("\n🌥️ Starting Cloud Removal Training Test")
print("=" * 70)
# Test with small dataset
model, train_losses, val_losses = train_cloud_removal_model(
data_dir="winter_dataset",
use_s1=True, # Use S1 radar data
batch_size=4, # Small batch for testing
num_epochs=5, # Few epochs for quick test
learning_rate=1e-4
)
print("\n✅ Training test completed!")
print(f"Final train loss: {train_losses[-1]:.6f}")
print(f"Final val loss: {val_losses[-1]:.6f}")
.temp/test_cloud_upload.py
+165
View File
@@ -0,0 +1,165 @@
#!/usr/bin/env python3
"""
Test Cloud Removal Model Upload Feature
"""
import requests
import json
from pathlib import Path
# API base URL
BASE_URL = "http://localhost:8000"
def test_upload_cloud_model(file_path):
"""Test uploading a cloud removal model"""
print(f"\n{'='*60}")
print("TEST 1: Upload Cloud Removal Model")
print(f"{'='*60}")
if not Path(file_path).exists():
print(f"❌ File not found: {file_path}")
print(" Create a dummy .pth file for testing:")
print(f" touch {file_path}")
return None
with open(file_path, 'rb') as f:
files = {'file': (Path(file_path).name, f, 'application/octet-stream')}
print(f"📤 Uploading: {file_path}")
response = requests.post(f"{BASE_URL}/api/cloud-removal/upload", files=files)
if response.status_code == 200:
result = response.json()
print(f"✅ Upload successful!")
print(f" Filename: {result['filename']}")
print(f" Size: {result['size_mb']} MB")
print(f" Path: {result['path']}")
return result['filename']
else:
print(f"❌ Upload failed: {response.status_code}")
print(f" {response.json().get('detail', 'Unknown error')}")
return None
def test_list_cloud_models():
"""Test listing cloud removal models"""
print(f"\n{'='*60}")
print("TEST 2: List Cloud Removal Models")
print(f"{'='*60}")
response = requests.get(f"{BASE_URL}/api/cloud-removal/models")
if response.status_code == 200:
data = response.json()
print(f"✅ Found {data['count']} models:")
for i, model in enumerate(data['models'], 1):
print(f"\n {i}. {model['filename']}")
print(f" Size: {model['size_mb']} MB")
print(f" Created: {model['created']}")
if 'epoch' in model:
print(f" Epoch: {model['epoch']}, Val Loss: {model['val_loss']:.4f}")
return data['models']
else:
print(f"❌ Failed to list models: {response.status_code}")
return []
def test_prediction_with_cloud_model(model_filename, cloud_model_filename):
"""Test prediction using uploaded cloud removal model"""
print(f"\n{'='*60}")
print("TEST 3: Prediction with Custom Cloud Removal Model")
print(f"{'='*60}")
config = {
"model_filename": model_filename,
"min_lon": 105.80,
"min_lat": 10.00,
"max_lon": 105.82,
"max_lat": 10.02,
"start_date": "2024-01-15",
"end_date": "2024-01-17",
"max_scenes": 2,
"cloud_cover": 30,
"resolution": 20,
"use_gpu": False,
"export_ndvi": True,
"export_classification": True,
"cloud_removal_method": "deep",
"cloud_removal_model": cloud_model_filename
}
print("📊 Prediction Config:")
print(json.dumps(config, indent=2))
print(f"\n🚀 Starting prediction with cloud removal model: {cloud_model_filename}")
response = requests.post(
f"{BASE_URL}/api/predict/with-ndvi",
json=config,
headers={'Content-Type': 'application/json'}
)
if response.status_code == 200:
result = response.json()
print(f"✅ Prediction started!")
print(f" Message: {result.get('message')}")
return result
else:
print(f"❌ Prediction failed: {response.status_code}")
print(f" {response.json().get('detail', 'Unknown error')}")
return None
def test_delete_cloud_model(filename):
"""Test deleting a cloud removal model"""
print(f"\n{'='*60}")
print("TEST 4: Delete Cloud Removal Model")
print(f"{'='*60}")
print(f"🗑️ Deleting: {filename}")
response = requests.delete(f"{BASE_URL}/api/cloud-removal/models/{filename}")
if response.status_code == 200:
result = response.json()
print(f"✅ {result['message']}")
return True
else:
print(f"❌ Delete failed: {response.status_code}")
return False
def main():
print("="*60)
print("CLOUD REMOVAL MODEL UPLOAD - FEATURE TEST")
print("="*60)
# Test file path (create a dummy file for testing)
test_file = "test_cloud_removal_model.pth"
# Create dummy file if it doesn't exist
if not Path(test_file).exists():
print(f"\n📝 Creating dummy test file: {test_file}")
Path(test_file).write_bytes(b"dummy_pytorch_model_data")
# Run tests
uploaded_filename = test_upload_cloud_model(test_file)
if uploaded_filename:
models = test_list_cloud_models()
# Test prediction (requires a real land classification model)
print(f"\n{'='*60}")
print("NOTE: Prediction test requires a trained land classification model")
print(" Skipping prediction test in this demo")
print(f"{'='*60}")
# Cleanup - delete test model
if input("\nDelete test model? (y/n): ").lower() == 'y':
test_delete_cloud_model(uploaded_filename)
# Cleanup dummy file
if Path(test_file).exists():
Path(test_file).unlink()
print(f"\n🗑️ Cleaned up dummy file: {test_file}")
print(f"\n{'='*60}")
print("TESTS COMPLETED")
print(f"{'='*60}")
if __name__ == "__main__":
main()
.temp/test_empty_points.py
+53
View File
@@ -0,0 +1,53 @@
import geopandas as gpd
import planetary_computer
import pystac_client
import odc.stac
import numpy as np
from shapely.geometry import Point, shape
from pyproj import Transformer
bbox = [105.5, 9.2, 106.3, 10.0]
time_range = "2023-01-01/2023-04-30"
catalog = pystac_client.Client.open(
"https://planetarycomputer.microsoft.com/api/stac/v1",
modifier=planetary_computer.sign_inplace,
)
items = list(catalog.search(collections=["sentinel-2-l2a"], bbox=bbox, datetime=time_range, query={"eo:cloud_cover": {"lt": 30}}).items())
items = sorted(items, key=lambda x: x.properties["eo:cloud_cover"])
gdf = gpd.read_file("train/ST_training_data_updated_1130points_new.shp")
gdf = gdf.to_crs("EPSG:32648")
# Find a point that fails. Let's just test a few points.
for idx, row in gdf.head(20).iterrows():
x_coord = row['geometry'].x
y_coord = row['geometry'].y
transformer = Transformer.from_crs("epsg:32648", "epsg:4326", always_xy=True)
lon, lat = transformer.transform(x_coord, y_coord)
point = Point(lon, lat)
filtered = []
for item in items:
if shape(item.geometry).contains(point):
filtered.append(item)
filtered = [planetary_computer.sign(item) for item in filtered]
if not filtered:
print(f"Point {idx}: NO ITEMS CONTAINS POINT!")
continue
ds = odc.stac.load(
filtered,
bands=["B02"],
x=(x_coord - 80, x_coord + 80),
y=(y_coord - 80, y_coord + 80),
crs="EPSG:32648",
resolution=10,
patch_url=planetary_computer.sign,
fail_on_error=False
).compute()
sums = ds["B02"].sum(dim=["x", "y"]).values
non_zero = (sums > 0).sum()
print(f"Point {idx}: {len(filtered)} items, {non_zero} non-zero time steps")
.temp/test_extract_one.py
+37
View File
@@ -0,0 +1,37 @@
import geopandas as gpd
import planetary_computer
import pystac_client
import odc.stac
import sys
bbox = [105.5, 9.2, 106.3, 10.0]
time_range = "2023-01-01/2023-04-30"
catalog = pystac_client.Client.open("https://planetarycomputer.microsoft.com/api/stac/v1", modifier=planetary_computer.sign_inplace)
search = catalog.search(collections=["sentinel-2-l2a"], bbox=bbox, datetime=time_range, query={"eo:cloud_cover": {"lt": 30}})
items = list(search.items())
items = sorted(items, key=lambda x: x.properties.get("eo:cloud_cover", 100))[:4]
items = [planetary_computer.sign(item) for item in items]
gdf = gpd.read_file("train/ST_training_data_updated_1130points_new.shp")
gdf = gdf.to_crs("EPSG:32648")
row = gdf.iloc[0]
x, y_coord = row.geometry.x, row.geometry.y
point_bbox = [x - 80, y_coord - 80, x + 80, y_coord + 80]
patch_s2 = odc.stac.load(
items,
bands=["B02", "B03", "B04", "B08", "SCL"],
x=(x - 80, x + 80),
y=(y_coord - 80, y_coord + 80),
crs="EPSG:32648",
resolution=10,
patch_url=planetary_computer.sign,
fail_on_error=False
).compute()
print("patch_s2 vars:", patch_s2.data_vars)
if patch_s2.dims['x'] < 16 or patch_s2.dims['y'] < 16:
print("Too small:", patch_s2.dims)
else:
print("Success dimension:", patch_s2.dims)
.temp/test_feature_extractor.py
+171
View File
@@ -0,0 +1,171 @@
"""
Test FeatureExtractor và kiểm tra tích hợp với hệ thống
"""
import numpy as np
import xarray as xr
from feature_extractor import get_feature_extractor
from pathlib import Path
print("=" * 70)
print("TESTING FEATURE EXTRACTOR MODULE")
print("=" * 70)
# Test 1: Simple mode
print("\n[TEST 1] Simple Mode (3 features)")
print("-" * 50)
extractor_simple = get_feature_extractor(mode='simple')
print(f"✓ Created extractor: {extractor_simple.mode}")
print(f"✓ Expected features: {extractor_simple.config['n_features']}")
print(f"✓ Feature names: {extractor_simple.get_feature_names()}")
# Create dummy NDVI data
ndvi_dummy = xr.DataArray(
np.random.rand(10, 10),
dims=['y', 'x'],
coords={'y': np.arange(10), 'x': np.arange(10)}
)
vh_dummy = xr.DataArray(
np.random.rand(10, 10) * -10,
dims=['y', 'x'],
coords={'y': np.arange(10), 'x': np.arange(10)}
)
vv_dummy = xr.DataArray(
np.random.rand(10, 10) * -8,
dims=['y', 'x'],
coords={'y': np.arange(10), 'x': np.arange(10)}
)
features_simple = extractor_simple.extract(
ndvi_data=ndvi_dummy,
vh_data=vh_dummy,
vv_data=vv_dummy
)
print(f"✓ Extracted features shape: {features_simple.shape}")
assert features_simple.shape[1] == 3, "Expected 3 features"
print("✅ Simple mode test PASSED\n")
# Test 2: Extended mode
print("[TEST 2] Extended Mode (15 features)")
print("-" * 50)
extractor_extended = get_feature_extractor(mode='extended')
print(f"✓ Created extractor: {extractor_extended.mode}")
print(f"✓ Expected features: {extractor_extended.config['n_features']}")
print(f"✓ Feature names: {extractor_extended.get_feature_names()}")
# Create dummy S2 dataset with time dimension
s2_dummy = xr.Dataset({
'B02': xr.DataArray(np.random.rand(5, 10, 10), dims=['time', 'y', 'x']),
'B03': xr.DataArray(np.random.rand(5, 10, 10), dims=['time', 'y', 'x']),
'B04': xr.DataArray(np.random.rand(5, 10, 10), dims=['time', 'y', 'x']),
'B08': xr.DataArray(np.random.rand(5, 10, 10), dims=['time', 'y', 'x']),
'B11': xr.DataArray(np.random.rand(5, 10, 10), dims=['time', 'y', 'x'])
})
features_extended = extractor_extended.extract(
s2_data=s2_dummy,
vh_data=vh_dummy,
vv_data=vv_dummy
)
print(f"✓ Extracted features shape: {features_extended.shape}")
assert features_extended.shape[1] == 15, "Expected 15 features"
print("✅ Extended mode test PASSED\n")
# Test 3: Temporal mode
print("[TEST 3] Temporal Mode (39 features for 12 timesteps)")
print("-" * 50)
extractor_temporal = get_feature_extractor(mode='temporal')
print(f"✓ Created extractor: {extractor_temporal.mode}")
# Create dummy S2 dataset with 12 timesteps
s2_dummy_12 = xr.Dataset({
'B02': xr.DataArray(np.random.rand(12, 10, 10), dims=['time', 'y', 'x']),
'B03': xr.DataArray(np.random.rand(12, 10, 10), dims=['time', 'y', 'x']),
'B04': xr.DataArray(np.random.rand(12, 10, 10), dims=['time', 'y', 'x']),
'B08': xr.DataArray(np.random.rand(12, 10, 10), dims=['time', 'y', 'x']),
'B11': xr.DataArray(np.random.rand(12, 10, 10), dims=['time', 'y', 'x'])
})
features_temporal = extractor_temporal.extract(
s2_data=s2_dummy_12,
vh_data=vh_dummy,
vv_data=vv_dummy
)
# For temporal mode: 12 timesteps * 3 indices + 3 radar = 39 features
expected_features = 12 * 3 + 3
print(f"✓ Extracted features shape: {features_temporal.shape}")
print(f"✓ Expected: {expected_features} features (12 timesteps * 3 indices + 3 radar)")
feature_names_temporal = extractor_temporal.get_feature_names(n_timesteps=12)
print(f"✓ Feature names count: {len(feature_names_temporal)}")
print(f"✓ First 5 features: {feature_names_temporal[:5]}")
print(f"✓ Last 5 features: {feature_names_temporal[-5:]}")
assert features_temporal.shape[1] == expected_features, f"Expected {expected_features} features"
assert len(feature_names_temporal) == expected_features, f"Expected {expected_features} feature names"
print("✅ Temporal mode test PASSED\n")
# Test 4: Check model_odc.joblib metadata
print("[TEST 4] Verify model_odc.joblib metadata")
print("-" * 50)
metadata_file = Path("model_train/model_odc_info.json")
if metadata_file.exists():
import json
with open(metadata_file) as f:
metadata = json.load(f)
print(f"✓ Metadata file exists: {metadata_file}")
print(f"✓ Feature mode: {metadata.get('feature_mode')}")
print(f"✓ Number of features: {metadata.get('n_features')}")
print(f"✓ Features list length: {len(metadata.get('features', []))}")
print(f"✓ First 5 features: {metadata.get('features', [])[:5]}")
assert metadata.get('feature_mode') == 'temporal', "Expected temporal mode"
assert metadata.get('n_features') == 39, "Expected 39 features"
assert len(metadata.get('features', [])) == 39, "Expected 39 feature names"
print("✅ model_odc.joblib metadata VERIFIED\n")
else:
print("❌ model_odc_info.json not found. Run: python create_odc_metadata.py")
# Test 5: Check ModelManager integration
print("[TEST 5] Test ModelManager integration")
print("-" * 50)
try:
from model_manager import get_model_manager
manager = get_model_manager()
print(f"✓ ModelManager initialized")
# List models
models = manager.list_models()
print(f"✓ Found {len(models)} models")
# Check if model_odc.joblib has metadata
odc_model = next((m for m in models if m['filename'] == 'model_odc.joblib'), None)
if odc_model:
print(f"✓ model_odc.joblib found in list")
print(f" - Feature mode: {odc_model.get('feature_mode', 'N/A')}")
print(f" - N features: {odc_model.get('n_features', 'N/A')}")
print("✅ ModelManager integration test PASSED\n")
else:
print("⚠️ model_odc.joblib not in model list")
except Exception as e:
print(f"❌ ModelManager test failed: {e}")
# Summary
print("=" * 70)
print("TEST SUMMARY")
print("=" * 70)
print("✅ All feature extraction modes working correctly")
print("✅ Feature dimensions match expectations")
print("✅ Feature names generated correctly")
print("✅ model_odc.joblib metadata verified")
print("\nNext steps:")
print("1. Update api_server.py with run_prediction from run_prediction_new.py")
print("2. Test training with different feature_modes")
print("3. Test prediction with models using different modes")
print("\nSee UPDATE_SUMMARY.md for details.")
print("=" * 70)
.temp/test_gdf.py
+3
View File
@@ -0,0 +1,3 @@
import geopandas as gpd
gdf = gpd.read_file("train/ST_training_data_updated_1130points_new.shp")
print(gdf.head(1)['HT_code'])
.temp/test_gdf_cols.py
+3
View File
@@ -0,0 +1,3 @@
import geopandas as gpd
gdf = gpd.read_file("train/ST_training_data_updated_1130points_new.shp")
print(gdf.columns)
.temp/test_gpu.py
+23
View File
@@ -0,0 +1,23 @@
import numpy as np
from xgboost import XGBClassifier
from sklearn.datasets import make_classification
from sklearn.metrics import accuracy_score
print("🚀 Testing XGBoost with CUDA GPU...")
try:
X, y = make_classification(n_samples=10000, n_features=20, n_classes=2, random_state=42)
model = XGBClassifier(
n_estimators=100,
max_depth=10,
tree_method="hist",
device="cuda",
random_state=42,
verbosity=1
)
print("Training model...")
model.fit(X, y)
y_pred = model.predict(X)
acc = accuracy_score(y, y_pred)
print(f"✅ Training successful! Accuracy: {acc*100:.2f}%")
except Exception as e:
print(f"❌ Error during training: {e}")
.temp/test_keys.py
+3
View File
@@ -0,0 +1,3 @@
import torch
checkpoint = torch.load('cloud_removal_model/cloud_removal_unet_best.pth', map_location='cpu')
print(list(checkpoint['model_state_dict'].keys())[:5])
.temp/test_model_manager.py
+101
View File
@@ -0,0 +1,101 @@
"""
Test script for Model Manager
Kiểm tra các chức năng: list models, load models, validate models
"""
from model_manager import ModelManager, get_model_manager
import json
def test_model_manager():
print("="*70)
print("MODEL MANAGER TEST")
print("="*70)
# Initialize ModelManager
model_manager = get_model_manager()
print("\n✅ ModelManager initialized")
# Test 1: List all models
print("\n" + "="*70)
print("TEST 1: LIST ALL MODELS")
print("="*70)
models = model_manager.list_models()
print(f"\n📦 Found {len(models)} models:")
for idx, model in enumerate(models, 1):
print(f"\n[{idx}] {model['filename']}")
print(f" Size: {model['size_mb']:.2f} MB")
print(f" Modified: {model['modified']}")
if model.get('has_metadata'):
print(f" Type: {model.get('model_type', 'N/A')}")
print(f" Features: {model.get('n_features', 'N/A')}")
print(f" Accuracy: {model.get('test_accuracy', 'N/A')}")
print(f" Feature list: {model.get('features', [])}")
else:
print(f" ⚠️ No metadata")
# Test 2: Load a model
if len(models) > 0:
print("\n" + "="*70)
print("TEST 2: LOAD MODEL")
print("="*70)
test_model = models[0]['filename']
print(f"\n🔄 Loading model: {test_model}")
try:
model, encoder, metadata = model_manager.load_model(test_model)
print(f"✅ Model loaded successfully!")
print(f"\n📊 Metadata:")
print(json.dumps(metadata, indent=2))
# Test 3: Validate model
print("\n" + "="*70)
print("TEST 3: VALIDATE MODEL")
print("="*70)
validation = model_manager.validate_model(test_model)
print(f"\n✅ Validation result:")
print(f" Valid: {validation['valid']}")
if validation['errors']:
print(f" Errors: {validation['errors']}")
if validation['warnings']:
print(f" Warnings: {validation['warnings']}")
# Test 4: Get required features
print("\n" + "="*70)
print("TEST 4: GET REQUIRED FEATURES")
print("="*70)
features = model_manager.get_required_features(test_model)
print(f"\n📋 Required features for {test_model}:")
for feat in features:
print(f" - {feat}")
except Exception as e:
print(f"❌ Error loading model: {e}")
import traceback
traceback.print_exc()
# Test 5: Get latest model
print("\n" + "="*70)
print("TEST 5: GET LATEST MODEL")
print("="*70)
latest = model_manager.get_latest_model()
print(f"\n📌 Latest model: {latest}")
latest_xgb = model_manager.get_latest_model(model_type='xgboost')
print(f"📌 Latest XGBoost model: {latest_xgb}")
latest_cnn = model_manager.get_latest_model(model_type='cnn')
print(f"📌 Latest CNN model: {latest_cnn}")
print("\n" + "="*70)
print("✅ ALL TESTS COMPLETED")
print("="*70)
if __name__ == "__main__":
test_model_manager()
.temp/test_model_struct.py
+13
View File
@@ -0,0 +1,13 @@
import torch
from pathlib import Path
model = torch.load('cloud_removal_model/cloud_removal_unet_best.pth', map_location='cpu')
print(type(model))
print("hasattr inc:", hasattr(model, 'inc'))
if hasattr(model, 'inc'):
print("hasattr double_conv:", hasattr(model.inc, 'double_conv'))
if hasattr(model.inc, 'double_conv'):
print("in_channels:", model.inc.double_conv[0].in_channels)
else:
for name, param in model.named_parameters():
print(name, param.shape)
break
.temp/test_new_features.py
+191
View File
@@ -0,0 +1,191 @@
#!/usr/bin/env python3
"""
Demo script để test các chức năng mới của API
"""
import requests
import json
import time
from pathlib import Path
BASE_URL = "http://localhost:8000"
def print_section(title):
print("\n" + "=" * 70)
print(f" {title}")
print("=" * 70)
def test_dashboard_statistics():
print_section("📊 Test Dashboard Statistics")
try:
response = requests.get(f"{BASE_URL}/api/dashboard/statistics")
if response.status_code == 200:
data = response.json()
print(f"✅ Success!")
print(f" Models: {data['models']['total']}")
print(f" Predictions: {data['predictions']['total']}")
print(f" Reports: {data['reports']['total']}")
else:
print(f"❌ Error: {response.status_code}")
except Exception as e:
print(f"❌ Exception: {e}")
def test_accuracy_trends():
print_section("📈 Test Accuracy Trends")
try:
response = requests.get(f"{BASE_URL}/api/dashboard/accuracy-trends")
if response.status_code == 200:
data = response.json()
print(f"✅ Success!")
print(f" Trends: {len(data['trends'])} records")
print(f" Models: {data['models']}")
else:
print(f"❌ Error: {response.status_code}")
except Exception as e:
print(f"❌ Exception: {e}")
def test_class_distribution():
print_section("📊 Test Class Distribution")
try:
# First, get list of models
response = requests.get(f"{BASE_URL}/api/models/list")
if response.status_code == 200:
models = response.json()['models']
if models:
model_filename = models[0]['filename']
print(f" Using model: {model_filename}")
# Get class distribution
response = requests.get(f"{BASE_URL}/api/dashboard/class-distribution/{model_filename}")
if response.status_code == 200:
data = response.json()
print(f"✅ Success!")
print(f" Total samples: {data['total_samples']}")
print(f" Classes: {list(data['class_distribution'].keys())}")
else:
print(f"❌ Error: {response.status_code}")
else:
print("⚠️ No models found")
else:
print(f"❌ Error getting models: {response.status_code}")
except Exception as e:
print(f"❌ Exception: {e}")
def test_batch_status():
print_section("🔄 Test Batch Status")
try:
response = requests.get(f"{BASE_URL}/api/batch/status")
if response.status_code == 200:
data = response.json()
print(f"✅ Success!")
print(f" Queued: {data['queue']['queued']}")
print(f" Running: {data['queue']['running']}")
print(f" Completed: {data['queue']['completed']}")
print(f" Failed: {data['queue']['failed']}")
else:
print(f"❌ Error: {response.status_code}")
except Exception as e:
print(f"❌ Exception: {e}")
def test_batch_prediction_demo():
print_section("🚀 Test Batch Prediction (Demo)")
try:
# Get a model
response = requests.get(f"{BASE_URL}/api/models/list")
if response.status_code != 200:
print("❌ Cannot get models list")
return
models = response.json()['models']
if not models:
print("⚠️ No models available for testing")
return
model_filename = models[0]['filename']
print(f" Using model: {model_filename}")
# Create test batch
batch_config = {
"model_filename": model_filename,
"items": [
{
"name": "Test_Region_1",
"min_lon": 105.6,
"min_lat": 9.3,
"max_lon": 105.7,
"max_lat": 9.4,
"start_date": "2023-03-01",
"end_date": "2023-03-31",
"max_scenes": 5,
"cloud_cover": 30,
"resolution": 20
}
],
"auto_retry": True,
"max_retries": 2
}
print(" Creating batch job...")
response = requests.post(
f"{BASE_URL}/api/batch/start",
json=batch_config
)
if response.status_code == 200:
data = response.json()
print(f"✅ Success!")
print(f" {data['message']}")
print(f" Batch ID: {data['batch_id']}")
# Check status after a moment
time.sleep(2)
response = requests.get(f"{BASE_URL}/api/batch/status")
if response.status_code == 200:
status = response.json()
print(f" Current queue: {status['queue']}")
else:
print(f"❌ Error: {response.status_code} - {response.text}")
except Exception as e:
print(f"❌ Exception: {e}")
def test_reports_list():
print_section("📄 Test Reports List")
try:
response = requests.get(f"{BASE_URL}/api/reports/list")
if response.status_code == 200:
data = response.json()
print(f"✅ Success!")
print(f" Total reports: {data['count']}")
if data['reports']:
print(f" Latest report: {data['reports'][0]['filename']}")
else:
print(f"❌ Error: {response.status_code}")
except Exception as e:
print(f"❌ Exception: {e}")
def main():
print("=" * 70)
print(" 🧪 API Testing Suite - New Features")
print("=" * 70)
print(f"\n Base URL: {BASE_URL}")
print(f" Đảm bảo server đang chạy: python api_server.py")
input("\n Press ENTER to start testing...")
# Run all tests
test_dashboard_statistics()
test_accuracy_trends()
test_class_distribution()
test_reports_list()
test_batch_status()
test_batch_prediction_demo()
print("\n" + "=" * 70)
print(" ✅ Testing completed!")
print("=" * 70)
print(f"\n Dashboard: {BASE_URL}/dashboard")
print(f" API Docs: {BASE_URL}/docs")
print("=" * 70 + "\n")
if __name__ == "__main__":
main()
.temp/test_odc_stac.py
+19
View File
@@ -0,0 +1,19 @@
import geopandas as gpd
import planetary_computer
import pystac_client
import odc.stac
bbox = [105.5, 9.2, 106.3, 10.0]
time_range = "2023-01-01/2023-01-30"
catalog = pystac_client.Client.open("https://planetarycomputer.microsoft.com/api/stac/v1", modifier=planetary_computer.sign_inplace)
items = list(catalog.search(collections=["sentinel-2-l2a"], bbox=bbox, datetime=time_range).items())[:1]
items = [planetary_computer.sign(item) for item in items]
x = 561609
y = 1024183
try:
ds = odc.stac.load(items, bands=["B02"], crs="EPSG:32648", resolution=10, x=(x-80, x+80), y=(y-80, y+80))
print("Success with x/y:", ds.dims)
except Exception as e:
print("Error with x/y:", e)
.temp/test_overlap.py
+18
View File
@@ -0,0 +1,18 @@
import geopandas as gpd
import planetary_computer
import pystac_client
import odc.stac
import numpy as np
bbox = [105.5, 9.2, 106.3, 10.0]
time_range = "2023-01-01/2023-04-30"
catalog = pystac_client.Client.open("https://planetarycomputer.microsoft.com/api/stac/v1", modifier=planetary_computer.sign_inplace)
# Get ALL items
items = list(catalog.search(collections=["sentinel-2-l2a"], bbox=bbox, datetime=time_range, query={"eo:cloud_cover": {"lt": 30}}).items())
items = [planetary_computer.sign(item) for item in items]
print(f"Total items: {len(items)}")
x = 561609
y = 1024183
ds = odc.stac.load(items, bands=["B02"], x=(x-80, x+80), y=(y-80, y+80), crs="EPSG:32648", resolution=10, patch_url=planetary_computer.sign).compute()
print("Time dimension size:", ds.dims['time'])
.temp/test_overlap2.py
+20
View File
@@ -0,0 +1,20 @@
import geopandas as gpd
import planetary_computer
import pystac_client
import odc.stac
import numpy as np
bbox = [105.5, 9.2, 106.3, 10.0]
time_range = "2023-01-01/2023-04-30"
catalog = pystac_client.Client.open("https://planetarycomputer.microsoft.com/api/stac/v1", modifier=planetary_computer.sign_inplace)
items = list(catalog.search(collections=["sentinel-2-l2a"], bbox=bbox, datetime=time_range, query={"eo:cloud_cover": {"lt": 30}}).items())
items = [planetary_computer.sign(item) for item in items]
x = 561609
y = 1024183
ds = odc.stac.load(items, bands=["B02"], x=(x-80, x+80), y=(y-80, y+80), crs="EPSG:32648", resolution=10, patch_url=planetary_computer.sign, fail_on_error=False).compute()
print("Original time size:", len(ds.time))
ds2 = ds.dropna(dim="time", how="all")
print("After dropna time size:", len(ds2.time))
print("B02 mean:", np.nanmean(ds2["B02"].values))
print("B02 non-nan count:", np.sum(~np.isnan(ds2["B02"].values)))
.temp/test_overlap_issue.py
+25
View File
@@ -0,0 +1,25 @@
import geopandas as gpd
import planetary_computer
import pystac_client
import odc.stac
import numpy as np
bbox = [105.5, 9.2, 106.3, 10.0]
time_range = "2023-01-01/2023-04-30"
catalog = pystac_client.Client.open("https://planetarycomputer.microsoft.com/api/stac/v1", modifier=planetary_computer.sign_inplace)
items = list(catalog.search(collections=["sentinel-2-l2a"], bbox=bbox, datetime=time_range, query={"eo:cloud_cover": {"lt": 30}}).items())
items = [planetary_computer.sign(item) for item in items]
x = 561609
y = 1024183
ds = odc.stac.load(items, bands=["B02", "B03", "B04", "B08", "SCL"], x=(x-80, x+80), y=(y-80, y+80), crs="EPSG:32648", resolution=10, patch_url=planetary_computer.sign, fail_on_error=False).compute()
print("Original shape:", ds["B02"].shape)
ds2 = ds.dropna(dim="time", how="all")
print("After dropna time size:", len(ds2.time))
if len(ds2.time) > 0:
ds2 = ds2.isel(time=slice(0, 4))
median = ds2["B02"].median(dim="time", skipna=True).values
print("Median shape:", median.shape)
print("Zeros in median:", np.sum(median == 0) / median.size)
print("NaNs in median:", np.sum(np.isnan(median)) / median.size)
.temp/test_planetary.py
+15
View File
@@ -0,0 +1,15 @@
import sys
# Thêm đường dẫn hiện tại vào PYTHONPATH để import được new_import_ODC nếu cần
sys.path.append('.')
import warnings
warnings.filterwarnings('ignore')
from new_import_ODC import load_sen1
print("Testing load_sen1 with a short time range to speed up Dask compute...")
bbox = [105.5, 9.2, 106.4, 10.0]
time_range = "2023-01-01/2023-01-31" # Short time range for fast testing
vh, vv = load_sen1(bbox, time_range)
print("VH shape:", vh.shape)
print("VV shape:", vv.shape)
print("VH CRS:", vh.rio.crs)
print("Success!")
.temp/test_planetary2.py
+49
View File
@@ -0,0 +1,49 @@
import warnings
warnings.filterwarnings('ignore')
def load_sen1(bbox, time_range):
import pystac_client
import planetary_computer
import odc.stac
catalog = pystac_client.Client.open(
"https://planetarycomputer.microsoft.com/api/stac/v1",
modifier=planetary_computer.sign_inplace,
)
search = catalog.search(
collections=["sentinel-1-rtc"],
bbox=bbox,
datetime=time_range,
)
items = list(search.items())
print("Found items:", len(items))
ds_s1 = odc.stac.load(
items,
bands=["vv", "vh"],
bbox=bbox,
crs="EPSG:32648",
resolution=10,
chunks={"x": 2048, "y": 2048, "time": 1}
)
ds_median = ds_s1.median(dim="time").compute()
vv = ds_median["vv"]
vh = ds_median["vh"]
vv = vv.expand_dims(dim="band")
vh = vh.expand_dims(dim="band")
vv = vv.rio.write_crs("EPSG:32648")
vh = vh.rio.write_crs("EPSG:32648")
return vh, vv
print("Testing load_sen1...")
bbox = [105.5, 9.2, 106.4, 10.0]
time_range = "2022-09-01/2023-10-01"
vh, vv = load_sen1(bbox, time_range)
print("VH shape:", vh.shape)
print("VV shape:", vv.shape)
print("Success!")
.temp/test_planetary_computer.py
+88
View File
@@ -0,0 +1,88 @@
"""
Test Microsoft Planetary Computer connectivity và token
"""
import planetary_computer
from pystac_client import Client
from datetime import datetime, timedelta
print("=" * 70)
print("🧪 TESTING MICROSOFT PLANETARY COMPUTER CONNECTION")
print("=" * 70)
# Test 1: Basic connection
print("\n1️⃣ Testing basic connection...")
try:
catalog = Client.open(
"https://planetarycomputer.microsoft.com/api/stac/v1",
modifier=planetary_computer.sign_inplace,
)
print("✅ Successfully connected to Planetary Computer")
print(f" Catalog ID: {catalog.id}")
print(f" Title: {catalog.title}")
except Exception as e:
print(f"❌ Connection failed: {e}")
exit(1)
# Test 2: List collections
print("\n2️⃣ Testing collections access...")
try:
collections = list(catalog.get_collections())
print(f"✅ Found {len(collections)} collections")
sentinel_2 = [c for c in collections if 'sentinel-2' in c.id.lower()]
print(f" Sentinel-2 collections: {[c.id for c in sentinel_2]}")
except Exception as e:
print(f"❌ Collections access failed: {e}")
# Test 3: Small search query (very conservative)
print("\n3️⃣ Testing small search query...")
try:
# Tiny bbox in Vietnam
bbox = [105.8, 10.0, 105.9, 10.1] # ~10km x 10km area
end_date = datetime.now()
start_date = end_date - timedelta(days=7) # Last 7 days only
time_range = f"{start_date.strftime('%Y-%m-%d')}/{end_date.strftime('%Y-%m-%d')}"
print(f" Bbox: {bbox}")
print(f" Time: {time_range}")
print(f" Searching...")
search = catalog.search(
collections=["sentinel-2-l2a"],
bbox=bbox,
datetime=time_range,
limit=5 # Only 5 items
)
items = []
for i, item in enumerate(search.items()):
items.append(item)
if i >= 4: # Stop at 5
break
print(f"✅ Search successful! Found {len(items)} items")
if items:
first_item = items[0]
print(f" First item: {first_item.id}")
print(f" Date: {first_item.datetime}")
# Test token signing
signed_item = planetary_computer.sign(first_item)
print(f"✅ SAS token signing works")
print(f" Asset keys: {list(signed_item.assets.keys())[:5]}")
except Exception as e:
print(f"❌ Search failed: {e}")
import traceback
traceback.print_exc()
print("\n" + "=" * 70)
print("🏁 Test completed!")
print("=" * 70)
print("\n💡 Nếu test này PASS:")
print(" → Planetary Computer hoạt động bình thường")
print(" → Vấn đề là query quá lớn (bbox/time range/max_scenes)")
print("\n💡 Nếu test này FAIL:")
print(" → Kiểm tra internet connection")
print(" → Thử lại sau (server có thể bị quá tải)")
print(" → Xem xét dùng dữ liệu local")
.temp/test_raw.py
+17
View File
@@ -0,0 +1,17 @@
import geopandas as gpd
import planetary_computer
import pystac_client
import odc.stac
import numpy as np
bbox = [105.5, 9.2, 106.3, 10.0]
time_range = "2023-01-01/2023-04-30"
catalog = pystac_client.Client.open("https://planetarycomputer.microsoft.com/api/stac/v1", modifier=planetary_computer.sign_inplace)
items = list(catalog.search(collections=["sentinel-2-l2a"], bbox=bbox, datetime=time_range, query={"eo:cloud_cover": {"lt": 30}}).items())[:4]
items = [planetary_computer.sign(item) for item in items]
x = 561609
y = 1024183
ds = odc.stac.load(items, bands=["B02", "B03", "B04", "B08"], x=(x-80, x+80), y=(y-80, y+80), crs="EPSG:32648", resolution=10, patch_url=planetary_computer.sign).compute()
print("B04 nanmean:", np.nanmean(ds["B04"].values))
print("B04 nanmax:", np.nanmax(ds["B04"].values))
.temp/test_shape.py
+41
View File
@@ -0,0 +1,41 @@
import new_import_ODC
importlib = __import__('importlib')
importlib.reload(new_import_ODC)
from new_import_ODC import *
import numpy as np
date_range = ("2022-09-01", "2022-10-01")
longtitude_range = (105.86, 105.94)
latitude_range = (9.65, 9.69)
coordinates = (longtitude_range, latitude_range)
print("Loading S2...")
data = load_data(None, date_range, longtitude_range, latitude_range)
result = mask_clean(data)
ds1 = calculate_indices(result, index="NDVI", satellite_mission="s2")
ndvi = ds1["NDVI"]
time_split = [
slice("2022-09-01", "2023-01-01"),
slice("2023-01-01", "2023-05-01"),
slice("2023-05-01", "2023-07-01"),
slice("2023-07-01", "2022-10-01"),
]
fill_nan_ndvi = fill_nan(ndvi, time_split)
average_ndvi = fill_nan_ndvi.resample(time="1M").mean().compute()
print("Loading S1...")
dsvh, dsvv = load_data_sen1(None, date_range, coordinates)
average_vv = calculate_average(dsvv, time_pattern='1M')
average_vh = calculate_average(dsvh, time_pattern='1M')
train = load_train_data("train/ST_training_data_updated_1130points_new.shp")
point = train.iloc[0]
ndvi_val = average_ndvi.sel(x=point.geometry.x, y=point.geometry.y, method='nearest').values
vh_val = average_vh.sel(x=point.geometry.x, y=point.geometry.y, method='nearest').values
vv_val = average_vv.sel(x=point.geometry.x, y=point.geometry.y, method='nearest').values
print("NDVI shape:", ndvi_val.shape, "ndim:", ndvi_val.ndim)
print("VH shape:", vh_val.shape, "ndim:", vh_val.ndim)
print("VV shape:", vv_val.shape, "ndim:", vv_val.ndim)
.temp/test_spatial_filter.py
+42
View File
@@ -0,0 +1,42 @@
import geopandas as gpd
import planetary_computer
import pystac_client
import odc.stac
import numpy as np
import time
from shapely.geometry import Point, box, shape
bbox = [105.5, 9.2, 106.3, 10.0]
time_range = "2023-01-01/2023-04-30"
catalog = pystac_client.Client.open("https://planetarycomputer.microsoft.com/api/stac/v1", modifier=planetary_computer.sign_inplace)
items = list(catalog.search(collections=["sentinel-2-l2a"], bbox=bbox, datetime=time_range, query={"eo:cloud_cover": {"lt": 30}}).items())
x = 561609
y = 1024183
start = time.time()
# Filter items by spatial intersection
from pyproj import Transformer
# The items geometry are in EPSG:4326 (lon, lat)
# Our x, y are in EPSG:32648
transformer = Transformer.from_crs("epsg:32648", "epsg:4326", always_xy=True)
lon, lat = transformer.transform(x, y)
point = Point(lon, lat)
filtered_items = []
for item in items:
geom = shape(item.geometry)
if geom.contains(point):
filtered_items.append(item)
filtered_items = sorted(filtered_items, key=lambda x: x.properties["eo:cloud_cover"])
print("Original items:", len(items))
print("Filtered items:", len(filtered_items))
print("Time to filter:", time.time() - start)
start = time.time()
filtered_items = [planetary_computer.sign(item) for item in filtered_items]
ds = odc.stac.load(filtered_items[:4], bands=["B02"], x=(x-80, x+80), y=(y-80, y+80), crs="EPSG:32648", resolution=10, patch_url=planetary_computer.sign, fail_on_error=False).compute()
print("Time to load 4 items:", time.time() - start)
print(ds["B02"].shape)
.temp/test_training_api.py
+113
View File
@@ -0,0 +1,113 @@
#!/usr/bin/env python3
"""
Test script to verify training API endpoints
"""
import requests
import json
API_BASE = "http://localhost:8000/api"
def test_training_labels():
"""Test /api/training/labels endpoint"""
print("=" * 70)
print("TEST 1: Getting training labels")
print("=" * 70)
response = requests.get(f"{API_BASE}/training/labels")
if response.ok:
data = response.json()
print(f"✅ Success! Found {data['count']} labels:")
for label in data['labels']:
print(f" {label['code']}: {label['name']}")
else:
print(f"❌ Error: {response.status_code}")
print()
def test_training_files():
"""Test /api/training/files endpoint"""
print("=" * 70)
print("TEST 2: Getting training files")
print("=" * 70)
response = requests.get(f"{API_BASE}/training/files")
if response.ok:
data = response.json()
print(f"✅ Success! Found {data['count']} training files:")
for file in data['files']:
print(f"\n 📄 {file['filename']}")
print(f" Size: {file['size_mb']} MB")
if 'point_count' in file:
print(f" Points: {file['point_count']}")
print(f" Label column: {file.get('label_column', 'N/A')}")
print(f" Unique labels: {file.get('label_count', 0)}")
else:
print(f"❌ Error: {response.status_code}")
print()
def test_shapefile_labels(filename="ST_training data_updated_1130points_new.shp"):
"""Test /api/training/shapefile/{filename}/labels endpoint"""
print("=" * 70)
print(f"TEST 3: Getting labels from shapefile: {filename}")
print("=" * 70)
response = requests.get(f"{API_BASE}/training/shapefile/{filename}/labels")
if response.ok:
data = response.json()
print(f"✅ Success!")
print(f" Filename: {data['filename']}")
print(f" Points: {data['point_count']}")
print(f" Label column: {data['label_column']}")
print(f" Unique labels: {data['label_count']}")
print(f" Bbox: {data['bbox']}")
print(f"\n Labels distribution:")
for label in data['labels']:
mapped = "✅" if label['mapped'] else "⚠️"
print(f" {mapped} {label['name']}: {label['count']} points (code: {label['code']})")
else:
print(f"❌ Error: {response.status_code}")
print(response.text)
print()
def test_config_presets():
"""Test /api/config/presets endpoint"""
print("=" * 70)
print("TEST 4: Getting config presets")
print("=" * 70)
response = requests.get(f"{API_BASE}/config/presets")
if response.ok:
data = response.json()
print(f"✅ Success! Found {len(data['presets'])} presets:")
for preset in data['presets']:
print(f"\n 📋 {preset['name']}")
config = preset['config']
print(f" Bbox: [{config['min_lon']}, {config['min_lat']}, {config['max_lon']}, {config['max_lat']}]")
print(f" Time: {config['start_date']} → {config['end_date']}")
print(f" Resolution: {config['resolution']}m")
else:
print(f"❌ Error: {response.status_code}")
print()
if __name__ == "__main__":
print("\n" + "=" * 70)
print("🧪 TESTING TRAINING API ENDPOINTS")
print("=" * 70 + "\n")
try:
test_training_labels()
test_training_files()
test_shapefile_labels()
test_config_presets()
print("=" * 70)
print("✅ ALL TESTS COMPLETED!")
print("=" * 70)
except requests.exceptions.ConnectionError:
print("\n❌ Error: Cannot connect to API server")
print("Make sure the server is running: python api_server.py")
except Exception as e:
print(f"\n❌ Error: {e}")
import traceback
traceback.print_exc()
.temp/test_unet.py
+5
View File
@@ -0,0 +1,5 @@
from train_cloud_removal import UNet
model = UNet(in_channels=6, out_channels=4)
print(hasattr(model, 'inc'))
print(hasattr(model, 'conv1'))
print(list(model.parameters())[0].shape)
.temp/test_y_mapped.py
+13
View File
@@ -0,0 +1,13 @@
import numpy as np
y = []
# simulate appending 1130 labels
for i in range(1130):
y.append(i % 5)
y = np.array(y)
unique_labels = sorted(list(np.unique(y)))
label_map = {lbl: i for i, lbl in enumerate(unique_labels)}
y_mapped = np.array([label_map[l] for l in y])
print(len(y), len(y_mapped))