refactor: reorganize project structure by moving core modules and update import paths in API server

update attribute table

.gitattributes

@@ -0,0 +1,5 @@
+# Disabled LFS tracking to avoid pushing large files
+#*.tif filter=lfs diff=lfs merge=lfs -text
+#*.joblib filter=lfs diff=lfs merge=lfs -text
+#*.nc filter=lfs diff=lfs merge=lfs -text
+#*.ipynb filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,86 @@
+# Ignore all model weights and large data
+*.joblib
+*.nc
+*.tif
+*.tiff
+*.png
+*.jpg
+*.jpeg
+*.h5
+*.pt
+*.ckpt
+*.pb
+*.npz
+*.npy
+*.hdf5
+*.pth
+*.onnx
+*.zip
+*.tar
+*.tar.gz
+*.7z
+*.rar
+*.exe
+*.dll
+*.so
+*.bin
+*.sav
+*.csv
+*.parquet
+*.feather
+*.db
+*.sqlite
+*.log
+*.bak
+*.tmp
+*~
+
+# Ignore model info/metadata if không cần backup
+# *.json
+
+# Ignore cache, prediction, backup folders
+dataset_cache/
+predictions/
+backup_model_train/
+backup_ketquaphanloai/
+backup_S3_download_Amazon/
+model_train/
+__pycache__/
+# Ignore large data files
+ndvi_results/
+ndvi_cache/
+prediction_cache/
+dataset_cache/
+bfg.jar
+..bfg-report/
+.dvc/
+
+# Ignore model outputs but keep info json
+model_train/*.joblib
+model_train/*.tif
+model_train/*.png
+model_train/*.h5
+model_train/*.pt
+model_train/*.pth
+model_train/*.ckpt
+model_train/*.npz
+model_train/*.npy
+model_train/*.zip
+model_train/*.tar
+model_train/*.tar.gz
+model_train/*.7z
+model_train/*.rar
+model_train/*.csv
+model_train/*.parquet
+model_train/*.feather
+model_train/*.db
+model_train/*.sqlite
+model_train/*.log
+cloud_removal_model/
+
+# VSCode settings
+.vscode/
+
+# Jupyter checkpoints
+.ipynb_checkpoints/
+reports/

.temp/01.train_ODC_XGBoost.py

@@ -0,0 +1,171 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+get_ipython().run_cell_magic('time', '', '%matplotlib inline\n\nimport importlib\nimport new_import_ODC  \n\nimportlib.reload(new_import_ODC)\n\nfrom new_import_ODC import *\n')
+
+
+# In[2]:
+
+
+get_ipython().run_cell_magic('time', '', '# Cấu hình Daskgateway\ncluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1, 10))\n# Khai báo 1 Datacube là dc\ndc = None\n\n# Cấu hình truy cập dịch vụ S3\nconfigure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n\nclient\n')
+
+
+# In[3]:
+
+
+## cấu hình thời gian lấy ảnh và tọa độ
+date_range = ("2022-09-01", "2022-10-01")
+longtitude_range = (105.86, 105.94)
+latitude_range = (9.65, 9.69)
+
+coordinates = (longtitude_range, latitude_range)
+
+
+# In[4]:
+
+
+## truy vấn ảnh vệ tinh sen2
+data = load_data(None, date_range, longtitude_range, latitude_range)
+notebook_utils.heading(notebook_utils.xarray_object_size(data))
+display(data)
+
+
+# In[5]:
+
+
+get_ipython().run_cell_magic('time', '', '# Tiến hành loại bỏ các vị trí bị mây ảnh hưởng\nresult = mask_clean(data)\n# progress(result)\n')
+
+
+# In[6]:
+
+
+# Tiến hành tính toán NDVI
+ds1 = calculate_indices(result, index="NDVI", satellite_mission="s2")
+ndvi = ds1["NDVI"]
+display(ndvi)
+
+
+# In[7]:
+
+
+## Hiển thị ảnh NDVI chưa điền các giá trị mây (chưa fill nan)
+plt.imshow(ndvi.isel(time=0))
+
+
+# In[8]:
+
+
+# Thiết lập giá trị trung bình mùa vụ để xử lý các điểm ảnh bị mây dựa vào sự thay đổi theo mùa
+time_split = [
+    slice("2022-09-01", "2023-01-01"),
+    slice("2023-01-01", "2023-05-01"),
+    slice("2023-05-01", "2023-07-01"),
+    slice("2023-07-01", "2022-10-01"),
+]
+
+# Điền mây ở các vị trí mang giá trị nan (fill nan)
+fill_nan_ndvi = fill_nan(ndvi, time_split)
+
+# In kết quả ảnh NDVI đã điền mây (đã fill nan)
+plt.imshow(fill_nan_ndvi.isel(time=0))
+
+
+# In[9]:
+
+
+get_ipython().run_cell_magic('time', '', '## tính ndvi theo tháng\naverage_ndvi = fill_nan_ndvi.resample(time="1M").mean().persist()\n# progress(average_ndvi)\n\n# compute average_ndvi\naverage_ndvi = average_ndvi.compute()\n')
+
+
+# In[10]:
+
+
+#Load dữ liệu ảnh Sentinel 1
+dsvh, dsvv = load_data_sen1(None, date_range, coordinates)
+average_vv = calculate_average(dsvv, time_pattern='1M')
+average_vh = calculate_average(dsvh, time_pattern='1M')
+
+
+# In[11]:
+
+
+## cấu hình bộ dữ liệu điểm huấn luyện mô hình (train file)
+train_path = "train/ST_training_data_updated_1130points_new.shp"  # đường dẫn shp file train
+
+## load dữ liệu điểm huấn luyện mô hình (train file)
+train = load_train_data(train_path)
+train.head()
+
+# cấu hình nhãn dữ liệu 
+label_mapping = {
+    "Lua tom": "0",
+    "Lua": "1",
+    "CHN": "2",
+    "CLN": "3",
+    "TS": "4",
+    "Song": "5",
+    "Dat xay dung": "6",
+    "Rung": "7",
+}
+
+# xây dựng tập dữ liệu (dataset) chứa dữ liệu VH, VV, NDVI
+datasets = get_data_sen1_and_sen2(train, average_ndvi, average_vh, average_vv)
+
+# chia tập dữ liệu thành các phần theo tỉ lệ 80(80-20)-20 tương ứng với tập train, validate, test
+X_train, X_val, X_test, y_train, y_val, y_test = split_train_data(
+    train, label_mapping, datasets
+)
+
+
+# In[ ]:
+
+
+get_ipython().run_cell_magic('time', '', '# Import XGBoost\nimport xgboost as xgb\nfrom sklearn.metrics import accuracy_score\nimport numpy as np\n\n# Convert to numpy arrays\nX_train_np = np.asarray(X_train, dtype=np.float32)\nX_val_np = np.asarray(X_val, dtype=np.float32)\ny_train_np = np.asarray(y_train, dtype=np.int32)\ny_val_np = np.asarray(y_val, dtype=np.int32)\n\nprint("🚀 Training XGBoost model...")\nprint(f"   Train samples: {len(X_train_np)}")\nprint(f"   Val samples: {len(X_val_np)}")\nprint(f"   Features: {X_train_np.shape[1]}")\nprint(f"   Classes: 8\\n")\n\n# XGBoost parameters\nparams = {\n    \'objective\': \'multi:softmax\',  # Multi-class classification\n    \'num_class\': 8,                 # 8 land use classes\n    \'max_depth\': 6,                 # Maximum tree depth\n    \'learning_rate\': 0.1,           # Learning rate\n    \'n_estimators\': 200,            # Number of trees\n    \'subsample\': 0.8,               # Subsample ratio\n    \'colsample_bytree\': 0.8,        # Feature sampling ratio\n    \'random_state\': 42,\n    \'n_jobs\': -1,                   # Use all CPU cores\n    \'eval_metric\': \'mlogloss\'       # Multi-class log loss\n}\n\n# Train XGBoost model\nmodel = xgb.XGBClassifier(**params)\n\nmodel.fit(\n    X_train_np, y_train_np,\n    eval_set=[(X_train_np, y_train_np), (X_val_np, y_val_np)],\n    verbose=True\n)\n\n# Validation accuracy\ny_val_pred = model.predict(X_val_np)\nval_accuracy = accuracy_score(y_val_np, y_val_pred)\nprint(f"\\n✅ Training completed!")\nprint(f"   Validation Accuracy: {val_accuracy:.4f} ({val_accuracy*100:.2f}%)")\n')
+
+
+# In[ ]:
+
+
+get_ipython().run_cell_magic('time', '', '# Evaluate on test set\nX_test_np = np.asarray(X_test, dtype=np.float32)\ny_test_np = np.asarray(y_test, dtype=np.int32)\n\nprint("📊 Evaluating XGBoost model on test set...\\n")\n\n# Predictions\ny_pred_test = model.predict(X_test_np)\n\n# Metrics\nfrom sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, confusion_matrix\n\ntest_accuracy = accuracy_score(y_test_np, y_pred_test)\nprecision = precision_score(y_test_np, y_pred_test, average=\'weighted\', zero_division=0)\nrecall = recall_score(y_test_np, y_pred_test, average=\'weighted\', zero_division=0)\nf1 = f1_score(y_test_np, y_pred_test, average=\'weighted\', zero_division=0)\n\nprint(f"📈 Test Results:")\nprint(f"   Accuracy:  {test_accuracy:.4f} ({test_accuracy*100:.2f}%)")\nprint(f"   Precision: {precision:.4f}")\nprint(f"   Recall:    {recall:.4f}")\nprint(f"   F1-Score:  {f1:.4f}\\n")\n\n# Confusion Matrix\nfrom sklearn.metrics import ConfusionMatrixDisplay\nimport matplotlib.pyplot as plt\n\n# Create figure first\nfig, ax = plt.subplots(figsize=(10, 8))\n\nclass_names = list(label_mapping.keys())\ncm = confusion_matrix(y_test_np, y_pred_test)\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=class_names)\ndisp.plot(cmap=\'Blues\', ax=ax)\nplt.xticks(rotation=45, ha=\'right\')\nplt.title(\'XGBoost Confusion Matrix\')\nplt.tight_layout()\nplt.show()\n')
+
+
+# In[ ]:
+
+
+# Lưu mô hình huấn luyện
+import json
+import joblib
+
+# Save XGBoost model
+model_path = "model_xgboost.joblib"
+joblib.dump(model, model_path)
+print(f"✅ Model saved to {model_path}")
+
+# Save model info
+info = {
+    "model_type": "XGBoost",
+    "num_classes": 8,
+    "classes": list(label_mapping.keys()),
+    "num_features": X_train_np.shape[1],
+    "params": params,
+    "accuracy": float(test_accuracy),
+    "precision": float(precision),
+    "recall": float(recall),
+    "f1_score": float(f1),
+}
+
+with open("model_xgboost_info.json", "w") as f:
+    json.dump(info, f, indent=2)
+
+print(f"✅ Model info saved to model_xgboost_info.json")
+
+
+# In[15]:
+
+
+# đóng client, cluster
+# client.close()
+# cluster.close()
+

.temp/01.train_ODC_local_with_Mic_supplyer.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[6]:
+
+
+get_ipython().run_cell_magic('time', '', '%matplotlib inline\n\n# Import Microsoft Planetary Computer libraries\nimport planetary_computer\nfrom pystac_client import Client\nfrom odc.stac import load as stac_load\n\n# Standard imports\nimport xarray as xr\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score, classification_report, confusion_matrix, ConfusionMatrixDisplay\nimport geopandas as gpd\n\n# XGBoost for GPU training\nimport xgboost as xgb\n\nfrom xgboost import XGBClassifier\n\nprint(f"   XGBoost version: {xgb.__version__}")\n\nprint("✅ All modules loaded successfully")\n')
+
+
+# In[7]:
+
+
+get_ipython().run_cell_magic('time', '', '# Kết nối tới Microsoft Planetary Computer STAC\nfrom pystac_client import Client\n\n# KHÔNG dùng modifier ở catalog level để tránh items bị convert thành dict\ncatalog = Client.open(\n    "https://planetarycomputer.microsoft.com/api/stac/v1"\n)\nprint("✅ Connected to Microsoft Planetary Computer")\n\nprint("\\n" + "="*70)\n')
+
+
+# In[8]:
+
+
+get_ipython().run_cell_magic('time', '', '# 🌍 Định nghĩa khu vực và thời gian\nprint("="*70)\nprint("CONFIGURATION")\nprint("="*70)\n\n# Khu vực quan tâm (Vietnam - Mekong Delta) - GIẢM DIỆN TÍCH ~40%\nbbox = [105.6, 9.3, 106.2, 9.8]  # [min_lon, min_lat, max_lon, max_lat]\n\n# GIẢM THỜI GIAN xuống 3 tháng để giảm kích thước dữ liệu cho PC\ntime_range = "2023-03-01/2023-05-31"  # 3 tháng (mùa khô)\n\nprint(f"\\n📍 Area of Interest:")\nprint(f"   Longitude: {bbox[0]} to {bbox[2]}")\nprint(f"   Latitude: {bbox[1]} to {bbox[3]}")\nprint(f"\\n📅 Time Range: {time_range}")\nprint(f"   ⚠️  Optimized for personal computer (3 months, reduced area)")\nprint(f"\\n🗺️  CRS: EPSG:32648")\nprint(f"   Resolution: 20m (reduced from 10m for smaller data size)")\n\nprint("="*70)\n')
+
+
+# In[9]:
+
+
+get_ipython().run_cell_magic('time', '', '# 📡 LOAD SENTINEL-2 FROM MICROSOFT PLANETARY COMPUTER\nprint("="*70)\nprint("LOADING SENTINEL-2 L2A")\nprint("="*70)\n\nprint("\\n🔍 Searching for Sentinel-2 scenes...")\nquery_s2 = catalog.search(\n    collections=["sentinel-2-l2a"],\n    bbox=bbox,\n    datetime=time_range,\n    query={"eo:cloud_cover": {"lt": 30}}  # Cloud cover < 30% (giảm từ 50%)\n)\n\nitems_s2 = list(query_s2.item_collection())\nprint(f"✅ Found {len(items_s2)} Sentinel-2 scenes")\n\n# GIỚI HẠN SỐ LƯỢNG SCENES cho PC cá nhân\nmax_scenes = 12  # Giảm xuống 12 scenes để tối ưu cho PC\nif len(items_s2) > max_scenes:\n    print(f"⚠️  Limiting to {max_scenes} scenes for personal computer")\n    # Chọn scenes đều đặn trong khoảng thời gian\n    step = len(items_s2) // max_scenes\n    items_s2 = items_s2[::step][:max_scenes]\n    print(f"   Selected {len(items_s2)} scenes evenly distributed")\n\nif len(items_s2) > 0:\n    # Show first few scenes\n    print(f"\\n📋 Sample scenes:")\n    for i, item in enumerate(items_s2[:5]):\n        date = item.datetime.strftime("%Y-%m-%d")\n        cloud = item.properties.get("eo:cloud_cover", "N/A")\n        print(f"   [{i+1}] {date} - Cloud: {cloud}%")\n    \n    # Re-sign items to ensure fresh URLs (keep as pystac objects)\n    print(f"\\n🔑 Signing STAC items...")\n    items_s2 = [planetary_computer.sign(item) for item in items_s2]\n    \n    # Load Sentinel-2 data (without Dask chunks)\n    print(f"\\n⏳ Loading Sentinel-2 data...")\n    ds_s2 = stac_load(\n        items_s2,\n        bands=["B04", "B08", "SCL"],  # Red (B04), NIR (B08), Scene Classification (SCL)\n        crs="EPSG:32648",\n        resolution=20,  # 20m resolution (4x smaller data than 10m)\n        bbox=bbox,\n        patch_url=planetary_computer.sign,  # Re-sign URLs during loading\n        fail_on_error=False,  # Skip problematic tiles instead of crashing\n    )\n    \n    # Rename bands to simpler names\n    ds_s2 = ds_s2.rename({"B04": "red", "B08": "nir", "SCL": "scl"})\n    \n    print(f"\\n✅ Sentinel-2 loaded!")\n    print(f"   Shape: {dict(ds_s2.dims)}")\n    print(f"   Variables: {list(ds_s2.data_vars)}")\n    display(ds_s2)\nelse:\n    print(f"❌ No Sentinel-2 scenes found")\n\n    ds_s2 = Noneprint("="*70)\n')
+
+
+# In[10]:
+
+
+get_ipython().run_cell_magic('time', '', '# 📡 LOAD SENTINEL-1 FROM MICROSOFT PLANETARY COMPUTER\nprint("="*70)\nprint("LOADING SENTINEL-1 RTC")\nprint("="*70)\n\nprint("\\n🔍 Searching for Sentinel-1 scenes...")\nquery_s1 = catalog.search(\n    collections=["sentinel-1-rtc"],\n    bbox=bbox,\n    datetime=time_range,\n)\n\nitems_s1 = list(query_s1.item_collection())\nprint(f"✅ Found {len(items_s1)} Sentinel-1 scenes")\n\n# GIỚI HẠN SỐ LƯỢNG SCENES cho PC cá nhân\nmax_scenes = 12  # Giảm xuống 12 scenes để tối ưu cho PC\nif len(items_s1) > max_scenes:\n    print(f"⚠️  Limiting to {max_scenes} scenes for personal computer")\n    # Chọn scenes đều đặn trong khoảng thời gian\n    step = len(items_s1) // max_scenes\n    items_s1 = items_s1[::step][:max_scenes]\n    print(f"   Selected {len(items_s1)} scenes evenly distributed")\n\nif len(items_s1) > 0:\n    # Show first few scenes\n    print(f"\\n📋 Sample scenes:")\n    for i, item in enumerate(items_s1[:5]):\n        date = item.datetime.strftime("%Y-%m-%d")\n        orbit = item.properties.get("sat:orbit_state", "N/A")\n        print(f"   [{i+1}] {date} - Orbit: {orbit}")\n    \n    # Re-sign items to ensure fresh URLs (keep as pystac objects)\n    print(f"\\n🔑 Signing STAC items...")\n    items_s1 = [planetary_computer.sign(item) for item in items_s1]\n    \n    # Load Sentinel-1 data (without Dask chunks)\n    print(f"\\n⏳ Loading Sentinel-1 data...")\n    ds_s1 = stac_load(\n        items_s1,\n        bands=["vv", "vh"],  # VV and VH polarizations\n        crs="EPSG:32648",\n        resolution=20,  # 20m resolution (4x smaller data than 10m)\n        bbox=bbox,\n        patch_url=planetary_computer.sign,  # Re-sign URLs during loading\n        fail_on_error=False,  # Skip problematic tiles instead of crashing\n    )\n    \n    # Convert to dB (Microsoft S1 is in linear power)\n    print(f"\\n🔄 Converting to dB...")\n    ds_s1[\'vv_db\'] = 10 * np.log10(ds_s1[\'vv\'].where(ds_s1[\'vv\'] > 0))\n    ds_s1[\'vh_db\'] = 10 * np.log10(ds_s1[\'vh\'].where(ds_s1[\'vh\'] > 0))\n    \n    print(f"\\n✅ Sentinel-1 loaded!")\n    print(f"   Shape: {dict(ds_s1.dims)}")\n    print(f"   Variables: {list(ds_s1.data_vars)}")\n    display(ds_s1)\nelse:\n    print(f"❌ No Sentinel-1 scenes found")\n\n    ds_s1 = Noneprint("="*70)\n')
+
+
+# In[11]:
+
+
+get_ipython().run_cell_magic('time', '', '# 🌿 CALCULATE NDVI AND PROCESS DATA\nprint("="*70)\nprint("DATA PROCESSING")\nprint("="*70)\n\nif ds_s2 is not None:\n    print("\\n[1] Calculating NDVI...")\n    # NDVI = (NIR - Red) / (NIR + Red)\n    ndvi = (ds_s2[\'nir\'] - ds_s2[\'red\']) / (ds_s2[\'nir\'] + ds_s2[\'red\'] + 1e-8)\n    \n    print(f"✅ NDVI calculated")\n    print(f"   Shape: {ndvi.shape}")\n    print(f"   Time steps: {len(ndvi.time)}")\n    \n    # Cloud masking using SCL band\n    print(f"\\n[2] Applying cloud mask...")\n    # SCL values: 1=defective, 3=cloud shadow, 8=cloud medium, 9=cloud high, 10=cirrus\n    cloud_mask = ds_s2[\'scl\'].isin([1, 3, 8, 9, 10])\n    ndvi_masked = ndvi.where(~cloud_mask)\n    \n    print(f"✅ Cloud mask applied")\n    \n    # Temporal aggregation (mean over time)\n    print(f"\\n[3] Computing mean NDVI across time...")\n    ndvi_mean = ndvi_masked.mean(dim=\'time\')\n    \n    # Data already in memory, no need to compute() again\n    print(f"✅ Mean NDVI computed")\n    print(f"   Shape: {ndvi_mean.shape}")\n    \nelse:\n    print("❌ No Sentinel-2 data to process")\n    ndvi_mean = None\n\nprint("="*70)\n')
+
+
+# In[ ]:
+
+
+get_ipython().run_cell_magic('time', '', '# 🎯 EXTRACT TRAINING DATA FEATURES\nprint("="*70)\nprint("FEATURE EXTRACTION")\nprint("="*70)\n\n# Check if required data is available\nif \'ndvi_mean\' not in globals() or \'ds_s1\' not in globals():\n    print("❌ Error: Please run Cell 6 (DATA PROCESSING) first!")\n    print("   Required variables: ndvi_mean, ds_s1")\n    raise RuntimeError("Missing required data. Run cells in order: Cell 4 → Cell 5 → Cell 6 → Cell 7")\n\n# Load training shapefile\nimport geopandas as gpd\n\ntrain_path = \'train/ST_training data_updated_1130points_new.shp\'\nprint(f"\\n[1] Loading training data from: {train_path}")\ntrain_gdf = gpd.read_file(train_path)\n\n# Ensure CRS matches\nif train_gdf.crs != \'EPSG:32648\':\n    print(f"   Reprojecting from {train_gdf.crs} to EPSG:32648...")\n    train_gdf = train_gdf.to_crs(\'EPSG:32648\')\n\nprint(f"✅ Loaded {len(train_gdf)} training points")\nprint(f"   Available columns: {list(train_gdf.columns)}")\n\n# Auto-detect label column (look for common names)\nlabel_column = None\nfor col in [\'HT_code\', \'Ma_LU\', \'LU2022\', \'class\', \'Class\', \'CLASS\', \'label\', \'Label\', \'LABEL\', \'LU_CODE\', \'LU_code\']:\n    if col in train_gdf.columns:\n        label_column = col\n        break\n\nif label_column is None:\n    print(f"❌ Cannot find label column. Available columns: {list(train_gdf.columns)}")\n    print(f"   Please check your shapefile and update the code.")\nelse:\n    print(f"   Using label column: \'{label_column}\'")\n    print(f"   Classes: {sorted(train_gdf[label_column].unique())}")\n    \n    # Extract features at each training point\n    print(f"\\n[2] Extracting features at training points...")\n    \n    features = []\n    labels = []\n    skipped = 0\n    \n    for idx, row in train_gdf.iterrows():\n        point = row.geometrychro\n        x_coord = point.x\n        y_coord = point.y\n        label = row[label_column]\n        \n        # Extract NDVI at this location\n        if ndvi_mean is not None and ds_s1 is not None:\n            try:\n                ndvi_val = ndvi_mean.sel(x=x_coord, y=y_coord, method=\'nearest\').values\n                \n                # Extract Sentinel-1 VH/VV at this location (mean across time)\n                # Data already in memory, no need to compute()\n                vh_val = ds_s1[\'vh_db\'].sel(x=x_coord, y=y_coord, method=\'nearest\').mean(dim=\'time\').values\n                vv_val = ds_s1[\'vv_db\'].sel(x=x_coord, y=y_coord, method=\'nearest\').mean(dim=\'time\').values\n                \n                # Create feature vector: [NDVI, VH_dB, VV_dB]\n                feature_vec = [ndvi_val, vh_val, vv_val]\n                \n                # Only add if all features are valid (not NaN)\n                if not np.isnan(feature_vec).any():\n                    features.append(feature_vec)\n                    labels.append(label)\n                else:\n                    skipped += 1\n            except Exception as e:\n                # Skip points outside the data extent\n                skipped += 1\n                continue\n    \n    features = np.array(features)\n    labels = np.array(labels)\n    \n    print(f"✅ Extracted features for {len(features)} valid points")\n    print(f"   Skipped {skipped} points (outside extent or NaN values)")\n    print(f"   Feature shape: {features.shape}")\n    print(f"   Feature names: [\'NDVI_mean\', \'VH_dB_mean\', \'VV_dB_mean\']")\n    print(f"\\n   Class distribution:")\n    unique, counts = np.unique(labels, return_counts=True)\n    for cls, cnt in zip(unique, counts):\n        print(f"      Class {cls}: {cnt} samples ({cnt/len(labels)*100:.1f}%)")\n\nprint("="*70)\n')
+
+
+# In[21]:
+
+
+get_ipython().run_cell_magic('time', '', '# 🤖 TRAIN XGBOOST MODEL ON GPU (RTX 4060)\nprint("="*70)\nprint("MODEL TRAINING - GPU ACCELERATED")\nprint("="*70)\n\nfrom xgboost import XGBClassifier\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import classification_report, confusion_matrix, ConfusionMatrixDisplay\nimport matplotlib.pyplot as plt\n\n# Encode labels to ensure they are 0, 1, 2, ... n-1\nprint("\\n[1] Encoding labels...")\nlabel_encoder = LabelEncoder()\nlabels_encoded = label_encoder.fit_transform(labels)\nprint(f"✅ Original classes: {label_encoder.classes_}")\nprint(f"   Encoded as: {np.unique(labels_encoded)}")\n\n# Split data\nprint("\\n[2] Splitting data (80% train, 20% test)...")\nX_train, X_test, y_train, y_test = train_test_split(\n    features, labels_encoded, test_size=0.2, random_state=42, stratify=labels_encoded\n)\nprint(f"✅ Training samples: {len(X_train)}")\nprint(f"   Testing samples: {len(X_test)}")\n\n# Train XGBoost on GPU\nprint("\\n[3] Training XGBoost classifier on RTX 4060 GPU...")\nprint("   GPU Settings: device=\'cuda:0\'")\n\nxgb_model = XGBClassifier(\n    n_estimators=100,\n    max_depth=20,\n    learning_rate=0.1,\n    device=\'cuda:0\',           # Use GPU (updated from deprecated gpu_id)\n    tree_method=\'hist\',        # Use hist with device for GPU training\n    random_state=42,\n    eval_metric=\'mlogloss\',    # Multi-class log loss\n    verbosity=1                # Show GPU training progress\n)\n\nxgb_model.fit(X_train, y_train)\nprint(f"✅ Model trained on GPU")\n\n# Evaluate\nprint("\\n[4] Evaluating model...")\ntrain_score = xgb_model.score(X_train, y_train)\ntest_score = xgb_model.score(X_test, y_test)\nprint(f"✅ Training accuracy: {train_score:.4f}")\nprint(f"   Testing accuracy: {test_score:.4f}")\n\n# Classification report\nprint("\\n[5] Classification Report:")\ny_pred = xgb_model.predict(X_test)\nprint(classification_report(y_test, y_pred, target_names=[str(c) for c in label_encoder.classes_]))\n\n# Confusion matrix\nprint("\\n[6] Confusion Matrix:")\nfig, ax = plt.subplots(figsize=(10, 8))\ncm = confusion_matrix(y_test, y_pred)\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=label_encoder.classes_)\ndisp.plot(ax=ax, cmap=\'Blues\', values_format=\'d\')\nplt.title(\'Confusion Matrix - XGBoost GPU Model (RTX 4060)\')\nplt.tight_layout()\nplt.show()\n\nprint("="*70)\n')
+
+
+# In[23]:
+
+
+get_ipython().run_cell_magic('time', '', '# 💾 SAVE MODEL AND CLEANUP\nprint("="*70)\nprint("SAVING MODEL & CLEANUP")\nprint("="*70)\n\nimport joblib\nfrom datetime import datetime\n\n# Save model and label encoder\nmodel_filename = f"model_train/model_xgboost_gpu_{datetime.now().strftime(\'%Y%m%d_%H%M%S\')}.joblib"\nprint(f"\\n[1] Saving model to: {model_filename}")\njoblib.dump({\'model\': xgb_model, \'label_encoder\': label_encoder}, model_filename)\nprint(f"✅ Model and label encoder saved")\n\n# Save model info\ninfo = {\n    "timestamp": datetime.now().isoformat(),\n    "data_source": "Microsoft Planetary Computer STAC",\n    "collections": ["sentinel-2-l2a", "sentinel-1-rtc"],\n    "features": ["NDVI_mean", "VH_dB_mean", "VV_dB_mean"],\n    "training_samples": len(X_train),\n    "testing_samples": len(X_test),\n    "train_accuracy": float(train_score),\n    "test_accuracy": float(test_score),\n    "model_type": "XGBClassifier",\n    "device": "cuda:0",\n    "gpu_device": "RTX 4060",\n    "tree_method": "hist",\n    "n_estimators": 100,\n    "max_depth": 20,\n    "learning_rate": 0.1\n}\n\nimport json\ninfo_filename = model_filename.replace(\'.joblib\', \'_info.json\')\nwith open(info_filename, \'w\') as f:\n    json.dump(info, f, indent=2)\nprint(f"✅ Model info saved to: {info_filename}")\n\n# No cleanup needed (Dask removed)\nprint("\\n[2] Cleanup complete")\n\nprint("="*70)\n\nprint("\\n" + "="*70)\n\nprint("🎉 TRAINING COMPLETE!")\n')
+

.temp/02.predict_ODC.py

@@ -0,0 +1,191 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+get_ipython().run_cell_magic('time', '', '%matplotlib inline\n\nimport importlib\nimport new_import_ODC  \n\nimportlib.reload(new_import_ODC)\n\nfrom new_import_ODC import *\n')
+
+
+# In[2]:
+
+
+get_ipython().run_cell_magic('time', '', '# Dask gateway\ncluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))\ndc = datacube.Datacube()\n\n# Configure s3 access\nconfigure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n\nclient\n')
+
+
+# In[3]:
+
+
+## cấu hình thời gian lấy ảnh và tọa độ
+date_range = ('2022-09-01', '2023-10-01')
+longtitude_range = (105.86575, 105.94120)
+latitude_range = (9.65070, 9.69850)
+
+
+# In[4]:
+
+
+## truy vấn ảnh vệ tinh sen2
+data = load_data(dc, date_range, longtitude_range, latitude_range)
+notebook_utils.heading(notebook_utils.xarray_object_size(data))
+display(data)
+
+
+# In[5]:
+
+
+get_ipython().run_cell_magic('time', '', '# Tiến hành loại bỏ các vị trí bị mây ảnh hưởng\nresult = mask_clean(data)\nprogress(result)\n')
+
+
+# In[6]:
+
+
+# Tiến hành tính toán NDVI
+ds1 = calculate_indices(result, index='NDVI', satellite_mission='s2')
+ndvi = ds1["NDVI"]
+display(ndvi)
+
+
+# In[7]:
+
+
+## ảnh NDVI chưa điền mây (fill nan)
+plt.imshow(ndvi.isel(time=50))
+
+
+# In[8]:
+
+
+# đặt thời gian các mùa
+time_split = [slice('2022-09-01', '2023-01-01'), 
+               slice('2023-01-01', '2023-05-01'),
+               slice('2023-05-01', '2023-07-01'),
+               slice('2023-07-01', '2023-10-01')]
+
+# Điền mây ở các vị trí mang giá trị nan (fill nan)
+fill_nan_ndvi = fill_nan(ndvi, time_split)
+
+# In kết quả ảnh ndvi đã điền mây (đã fill nan)
+plt.imshow(fill_nan_ndvi.isel(time=50))
+
+
+# In[9]:
+
+
+get_ipython().run_cell_magic('time', '', "## tính ndvi theo tháng\naverage_ndvi = fill_nan_ndvi.resample(time='1M').mean().persist()\nprogress(average_ndvi)\n")
+
+
+# In[10]:
+
+
+# compute average_ndvi
+average_ndvi = average_ndvi.compute()
+
+
+# In[11]:
+
+
+# load dữ liệu sen1
+coordinates = (longtitude_range, latitude_range)
+dsvh, dsvv = load_data_sen1(dc, date_range, coordinates)
+average_vv = calculate_average(dsvv, time_pattern='1M')
+average_vh = calculate_average(dsvh, time_pattern='1M')
+
+
+# In[12]:
+
+
+# load model RF
+loaded_model = joblib.load(os.path.join("model_train", "model_odc.joblib"))
+
+# dự đoán
+data_array = predict(loaded_model, data.rio.crs, average_ndvi, average_vh, average_vv)
+
+
+# In[13]:
+
+
+# cấu hình màu cho các loại đất
+colors = [
+    "#abcee9",
+    "#ffef44",
+    "#c4ff9e",
+    "#ffd6a8",
+    "#93ddda",
+    "#1aeef7",
+    "#ffa7f2",
+    "#33ee33"
+]
+labels = [
+    "Lúa tôm",
+    "Lúa",
+    "CHN",
+    "CLN",
+    "TS",
+    "Sông",
+    "Đất xây dựng",
+    "Rừng"
+]
+# hiển thị phân loại sử dụng đất
+cmap = ListedColormap(colors)
+img = data_array.plot(cmap=cmap, add_colorbar=False)
+cbar = plt.colorbar(img)
+cbar.ax.set_yticklabels(labels)
+plt.title("Phân loại sử dụng đất")
+plt.axis('off')
+plt.show()
+
+
+# In[14]:
+
+
+## cấu hình shapefile ranh giới thuận hòa và vh vv file
+thuanhoa_path = "ThuanHoa/region/ST_ThuanHoa_Boundaryofficially.shp"
+
+# cắt theo ranh giới xã thuận hòa
+region_result = cut_according_shp(thuanhoa_path, average_ndvi, data_array)
+
+
+# In[15]:
+
+
+# hiển thị kết quả phân loại sử dụng đất
+colorval = list(range(len(colors)))
+options = {
+    'title': 'Phân loại sử dụng đất',
+    'cmap': colors,
+    'clim': (0, 8),
+    'aspect': 'equal',
+    'colorbar_opts': {
+        'major_label_overrides': dict(zip(colorval, labels)),
+        'major_label_text_align': 'left',
+        'ticker': FixedTicker(ticks=colorval),
+    },
+}
+                                      
+region_result.hvplot(
+    rasterize = True,                  # Use Datashader, particularly useful for dask arrays
+    aggregator = reductions.mode(),    # Datashader selects mode value, requires 'hv.Image'
+).options(opts.Image(**options))
+
+
+# In[16]:
+
+
+# Lưu lại kết quả
+region_result.rio.to_raster("KetQuaPhanLoaiDatODC.tif")
+
+
+# In[17]:
+
+
+# đóng client, cluster
+client.close()
+cluster.close()
+
+
+# In[ ]:
+
+
+
+

.temp/03.compare_ODC.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+# Khai báo các thư viện cần thiết
+from new_import_ODC import *
+
+# Khai báo đường dẫn đến kết quả phân loại và dữ liệu của địa phương
+KD_path = "ThuanHoa/KhoanhDat/ThuanHoa_TKDD2022.shp"
+KetQuaPhanLoaiDat = "KetQuaPhanLoaiDatODC.tif"
+
+
+# In[2]:
+
+
+# khai báo các loại đất từ dữ liệu kiểm kê ứng với các hiện trạng được phân loại từ viễn thám
+CODE_MAP = {
+    "BHK": 2,
+    "CLN": 3,
+    "DGD": 6,
+    "DGT": 6,
+    "DNL": 6,
+    "DRA": 6,
+    "DSH": 6,
+    "DTL": 5,
+    "DTS": 6,
+    "DYT": 6,
+    "LUC": 1,
+    "NKH": 3,
+    "NTD": 6,
+    "NTS": 4,
+    "ONT": 6,
+    "SKC": 6,
+    "SKX": 6,
+    "SON": 5,
+    "TMD": 6,
+    "TON": 6,
+    "TSC": 6,
+}
+
+# Khai báo các nhãn phân loại đất ứng với 3 loại đất chính
+HT_MAP = {
+    "NN": {"name": "Đất Nông Nghiệp", "data": [1, 2, 3, 4]},
+    "PNN": {"name": "Đất Phi Nông Nghiệp", "data": [6]},
+    "TQ": {"name": "Đất Thổ Quả", "data": [15]},
+}
+
+
+# In[3]:
+
+
+# Tiến hành chồng lắp
+result = compare(KD_path, KetQuaPhanLoaiDat, CODE_MAP, HT_MAP)
+
+
+# In[4]:
+
+
+# cấu hình màu cho các loại sử dụng đất
+colors = [
+    "#abcee9",
+    "#ffffc0",
+    "#c4ff9e",
+    "#ffd6a8",
+    "#93ddda",
+    "#1aeef7",
+    "#ffa7f2",
+    "#33ee33",
+]
+labels = ["Lúa tôm", "Lúa", "CHN", "CLN", "TS", "Sông", "Đất xây dựng", "Rừng"]
+
+
+# In[5]:
+
+
+# Lưu kết quả
+save_result(result, HT_MAP)
+
+
+# In[6]:
+
+
+# hiển thị kết quả
+xx = []
+
+for k, v in result.items():
+    rs = merge_arrays(v, nodata=np.nan)
+    xx.append(rs.squeeze(drop=True))
+xx = xr.concat(xx, pd.Index([HT_MAP[x]["name"] for x in HT_MAP], name="name"))
+
+colorval = list(range(len(colors)))
+options = {
+    "cmap": colors,
+    "clim": (0, 8),
+    "aspect": "equal",
+    "height": 400,
+    "colorbar_opts": {
+        "major_label_overrides": dict(zip(colorval, labels)),
+        "major_label_text_align": "left",
+        "ticker": FixedTicker(ticks=colorval),
+    },
+}
+
+xx.hvplot(
+    groupby="name",
+    rasterize=True,  # Use Datashader, particularly useful for dask arrays
+    aggregator=reductions.mode(),  # Datashader selects mode value, requires 'hv.Image'
+).options(opts.Image(**options))
+
+
+# In[ ]:
+
+
+
+

.temp/check_X.py

@@ -0,0 +1,18 @@
+import joblib
+import numpy as np
+
+cache_file = "dataset_cache/training_data_2d.joblib"
+data = joblib.load(cache_file)
+X = np.array(data['X'])
+y = np.array(data['y'])
+
+print("X shape:", X.shape)
+print("X mean:", np.mean(X))
+print("X std:", np.std(X))
+print("X min:", np.min(X))
+print("X max:", np.max(X))
+print("Any NaN:", np.isnan(X).any())
+
+for i in range(6):
+    print(f"Channel {i} mean: {np.mean(X[:, i, :, :]):.4f}, min: {np.min(X[:, i, :, :]):.4f}, max: {np.max(X[:, i, :, :]):.4f}")
+

.temp/check_missing.py

@@ -0,0 +1,10 @@
+import joblib
+import geopandas as gpd
+from shapely.geometry import Point
+
+data = joblib.load('dataset_cache/training_data_2d.joblib')
+X, y = data['X'], data['y']
+print(f"X shape: {X.shape}, y shape: {y.shape}")
+
+gdf = gpd.read_file("train/ST_training_data_updated_1130points_new.shp")
+print("Total points:", len(gdf))

.temp/check_versions.py

@@ -0,0 +1,5 @@
+import xarray as xr
+import rasterio
+
+print(f"xarray version: {xr.__version__}")
+print(f"rasterio version: {rasterio.__version__}")

.temp/check_xgboost_data.py

@@ -0,0 +1,7 @@
+import joblib
+import numpy as np
+
+data = joblib.load('dataset_cache/training_data.joblib')
+X, y = data['X'], data['y']
+print(f"X shape: {X.shape}")
+print(f"y shape: {y.shape}")

.temp/check_zeros.py

@@ -0,0 +1,8 @@
+import joblib
+import numpy as np
+cache_file = "dataset_cache/training_data_2d.joblib"
+data = joblib.load(cache_file)
+X = np.array(data['X'])
+b2 = X[:, 0, :, :]
+print("Zeros in B2:", np.sum(b2 == 0) / b2.size)
+print("X shape:", X.shape)

.temp/cloud_removal_train.ipynb

@@ -0,0 +1,10 @@
+{
+ "cells": [],
+ "metadata": {
+  "language_info": {
+   "name": "python"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

.temp/fix_all.py

@@ -0,0 +1,25 @@
+import glob, json
+
+changed_files = []
+for file_path in glob.glob('*.ipynb'):
+    with open(file_path, 'r', encoding='utf-8') as f:
+        nb = json.load(f)
+        
+    changed = False
+    for cell in nb.get('cells', []):
+        if cell.get('cell_type') == 'code':
+            source = cell.get('source', [])
+            for i, line in enumerate(source):
+                if 'time=50' in line:
+                    source[i] = line.replace('time=50', 'time=0')
+                    changed = True
+                if 'load_data_sen1(dc,' in line:
+                    source[i] = line.replace('load_data_sen1(dc,', 'load_data_sen1(None,')
+                    changed = True
+                    
+    if changed:
+        with open(file_path, 'w', encoding='utf-8') as f:
+            json.dump(nb, f, indent=1)
+        changed_files.append(file_path)
+
+print('Fixed issues in:', changed_files)

.temp/fix_cache.py

@@ -0,0 +1,19 @@
+import joblib
+import geopandas as gpd
+import numpy as np
+
+cache_file = "dataset_cache/training_data_2d.joblib"
+data = joblib.load(cache_file)
+X = data['X']
+print("X shape:", len(X))
+
+gdf = gpd.read_file("train/ST_training_data_updated_1130points_new.shp")
+gdf = gdf.to_crs("EPSG:32648")
+print("gdf length:", len(gdf))
+
+if len(X) == len(gdf):
+    y = [(row['HT_code'] - 1) for idx, row in gdf.iterrows()]
+    joblib.dump({'X': X, 'y': y}, cache_file)
+    print("Fixed y in cache! Saved.")
+else:
+    print("Lengths do not match, cannot fix automatically.")

.temp/fix_new_train.py

@@ -0,0 +1,20 @@
+import json
+
+def fix_import(file_path):
+    with open(file_path, 'r', encoding='utf-8') as f:
+        nb = json.load(f)
+    changed = False
+    for cell in nb.get('cells', []):
+        if cell.get('cell_type') == 'code':
+            source = cell.get('source', [])
+            if isinstance(source, list):
+                for i, line in enumerate(source):
+                    if "from new_import import *" in line:
+                        source[i] = line.replace("from new_import import *", "from new_import_ODC import *")
+                        changed = True
+    if changed:
+        with open(file_path, 'w', encoding='utf-8') as f:
+            json.dump(nb, f, indent=1)
+        print(f"Fixed {file_path}")
+
+fix_import('new_train.ipynb')

.temp/fix_train.py

@@ -0,0 +1,20 @@
+import re
+
+filepath = "01.train_ODC.py"
+with open(filepath, 'r', encoding='utf-8') as f:
+    content = f.read()
+
+# Find the run_cell_magic line
+pattern = re.compile(r"get_ipython\(\)\.run_cell_magic\('time', '', '(# 🤖 LAND USE CLASSIFICATION MODEL TRAINING.*?)(?=\n')\n'", re.DOTALL)
+
+def repl(match):
+    # Get the inner string and escape all actual newlines with \n
+    inner = match.group(1)
+    inner = inner.replace('\n', '\\n')
+    return f"get_ipython().run_cell_magic('time', '', '{inner}')"
+
+content = pattern.sub(repl, content)
+
+with open(filepath, 'w', encoding='utf-8') as f:
+    f.write(content)
+print("Fixed 01.train_ODC.py syntax")

.temp/fix_typo.py

@@ -0,0 +1,22 @@
+import json
+
+def fix_filename(file_path):
+    with open(file_path, 'r', encoding='utf-8') as f:
+        nb = json.load(f)
+    changed = False
+    for cell in nb.get('cells', []):
+        if cell.get('cell_type') == 'code':
+            source = cell.get('source', [])
+            if isinstance(source, list):
+                for i, line in enumerate(source):
+                    if "ST_training data_updated_1130points.shp" in line:
+                        source[i] = line.replace("ST_training data_updated_1130points.shp", "ST_training_data_updated_1130points.shp")
+                        changed = True
+    if changed:
+        with open(file_path, 'w', encoding='utf-8') as f:
+            json.dump(nb, f, indent=1)
+        print(f"Fixed typo in {file_path}")
+
+import glob
+for nb in glob.glob("*.ipynb"):
+    fix_filename(nb)

.temp/generate_colab_notebook.py

@@ -0,0 +1,58 @@
+import nbformat as nbf
+
+nb = nbf.v4.new_notebook()
+
+text_1 = """# Script Tải Dữ liệu Vệ tinh (Cache) qua Google Colab
+Mục đích của Notebook này là mượn sức mạnh đường truyền và RAM của Google Colab để tải 270 ảnh Sentinel-2 & Sentinel-1 từ Microsoft Planetary Computer. Sau khi xử lý nội suy, nó sẽ sinh ra một file cache `.joblib` duy nhất chứa toàn bộ mảng dữ liệu. 
+Bạn chỉ cần tải file `.joblib` đó về máy là xong!"""
+
+code_1 = """!pip install planetary-computer pystac-client odc-stac geopandas rasterio xarray joblib scikit-learn xgboost lightgbm"""
+
+code_2 = """from google.colab import drive
+drive.mount('/content/drive')"""
+
+text_2 = """## Hướng dẫn:
+1. Nén toàn bộ thư mục `remote-sensing` ở máy tính của bạn thành file `remote-sensing.zip`.
+2. Upload file `remote-sensing.zip` đó lên Google Drive (để ngay ngoài cùng).
+3. Chạy ô lệnh bên dưới để giải nén và chuyển vào thư mục dự án."""
+
+code_3 = """import os
+import shutil
+
+# Giải nén dự án từ Google Drive
+!unzip -q /content/drive/MyDrive/remote-sensing.zip -d /content/
+os.chdir('/content/remote-sensing')
+!ls -la"""
+
+text_3 = """## Bắt đầu tải và Cache Dữ Liệu
+Chạy một mô hình CPU đơn giản (Decision Tree) để ép hệ thống gọi hàm `FeatureExtractor`. Hàm này sẽ làm mọi việc nặng nhọc: tìm ảnh, ghép mây, tính trung vị và lưu kết quả vào thư mục `dataset_cache/`."""
+
+code_4 = """# Lệnh này sẽ mất khoảng 5-15 phút để tải toàn bộ ảnh từ Microsoft
+!python train_land_decision_tree_gpu.py"""
+
+text_4 = """## Hoàn tất
+Bạn hãy kiểm tra xem file `.joblib` lớn (khoảng 40-60MB) đã xuất hiện chưa. Nếu rồi, hãy lưu ngược nó lại Google Drive để tải về máy!"""
+
+code_5 = """# Xem file cache đã được tạo thành công chưa
+!ls -lh dataset_cache/
+
+# Copy toàn bộ thư mục cache sang Google Drive để tải về máy dễ dàng
+!cp -r dataset_cache/ /content/drive/MyDrive/dataset_cache_finished/
+print("Hoàn thành! Bạn hãy mở Google Drive của mình, tìm thư mục 'dataset_cache_finished' và tải file .joblib mới nhất về máy tính.")"""
+
+nb['cells'] = [
+    nbf.v4.new_markdown_cell(text_1),
+    nbf.v4.new_code_cell(code_1),
+    nbf.v4.new_code_cell(code_2),
+    nbf.v4.new_markdown_cell(text_2),
+    nbf.v4.new_code_cell(code_3),
+    nbf.v4.new_markdown_cell(text_3),
+    nbf.v4.new_code_cell(code_4),
+    nbf.v4.new_markdown_cell(text_4),
+    nbf.v4.new_code_cell(code_5)
+]
+
+with open('Download_Cache_Colab.ipynb', 'w') as f:
+    nbf.write(nb, f)
+
+print("Created Download_Cache_Colab.ipynb")

.temp/generate_previews.py

@@ -0,0 +1,132 @@
+#!/usr/bin/env python3
+"""
+Generate PNG previews for existing GeoTIFF prediction files
+"""
+
+import numpy as np
+import rasterio
+import matplotlib
+matplotlib.use('Agg')
+import matplotlib.pyplot as plt
+from pathlib import Path
+import sys
+
+def generate_png_preview(tif_file, output_png=None):
+    """Generate PNG preview from GeoTIFF file"""
+    tif_path = Path(tif_file)
+    
+    if not tif_path.exists():
+        print(f"❌ File not found: {tif_file}")
+        return False
+    
+    # Determine output PNG path
+    if output_png is None:
+        output_png = tif_path.with_suffix('.png')
+    else:
+        output_png = Path(output_png)
+    
+    try:
+        # Read GeoTIFF
+        with rasterio.open(tif_path) as src:
+            data = src.read(1)
+            
+        print(f"📊 Data shape: {data.shape}, range: [{np.nanmin(data):.3f}, {np.nanmax(data):.3f}]")
+        
+        # Determine if it's classification or NDVI based on filename
+        is_classification = 'classification' in tif_path.name.lower() or 'prediction' in tif_path.name.lower()
+        is_ndvi = 'ndvi' in tif_path.name.lower()
+        
+        # Create figure
+        fig, ax = plt.subplots(figsize=(12, 10), dpi=150)
+        
+        if is_ndvi:
+            # NDVI: use RdYlGn colormap, range -1 to 1
+            im = ax.imshow(data, cmap='RdYlGn', vmin=-1, vmax=1, interpolation='nearest')
+            ax.set_title(f'NDVI - {tif_path.stem}', fontsize=14, fontweight='bold')
+            cbar_label = 'NDVI'
+        elif is_classification:
+            # Classification: use tab20 colormap
+            im = ax.imshow(data, cmap='tab20', interpolation='nearest')
+            ax.set_title(f'Land Classification - {tif_path.stem}', fontsize=14, fontweight='bold')
+            cbar_label = 'Class'
+        else:
+            # Generic: use viridis
+            im = ax.imshow(data, cmap='viridis', interpolation='nearest')
+            ax.set_title(f'{tif_path.stem}', fontsize=14, fontweight='bold')
+            cbar_label = 'Value'
+        
+        ax.set_xlabel('X (pixels)', fontsize=10)
+        ax.set_ylabel('Y (pixels)', fontsize=10)
+        
+        # Add colorbar
+        cbar = plt.colorbar(im, ax=ax, fraction=0.046, pad=0.04)
+        cbar.set_label(cbar_label, rotation=270, labelpad=15)
+        
+        # For classification, try to set integer ticks
+        if is_classification:
+            try:
+                unique_vals = np.unique(data[~np.isnan(data)])
+                if len(unique_vals) < 20:  # Only if not too many classes
+                    cbar.set_ticks(unique_vals)
+                    cbar.set_ticklabels([str(int(v)) for v in unique_vals])
+            except:
+                pass
+        
+        # Add grid
+        ax.grid(True, alpha=0.3, linestyle='--', linewidth=0.5)
+        
+        # Save PNG
+        plt.tight_layout()
+        plt.savefig(str(output_png), dpi=150, bbox_inches='tight')
+        plt.close(fig)
+        
+        print(f"✅ Created PNG: {output_png}")
+        return True
+        
+    except Exception as e:
+        print(f"❌ Error creating PNG: {e}")
+        import traceback
+        traceback.print_exc()
+        return False
+
+
+def generate_all_previews(predictions_dir="predictions"):
+    """Generate PNG previews for all GeoTIFF files without PNGs"""
+    pred_path = Path(predictions_dir)
+    
+    if not pred_path.exists():
+        print(f"❌ Directory not found: {predictions_dir}")
+        return
+    
+    tif_files = list(pred_path.glob("*.tif"))
+    print(f"🔍 Found {len(tif_files)} GeoTIFF files")
+    
+    generated = 0
+    skipped = 0
+    
+    for tif_file in tif_files:
+        png_file = tif_file.with_suffix('.png')
+        
+        if png_file.exists():
+            print(f"⏭️  Skipping {tif_file.name} (PNG already exists)")
+            skipped += 1
+            continue
+        
+        print(f"\n🎨 Processing {tif_file.name}...")
+        if generate_png_preview(tif_file):
+            generated += 1
+    
+    print(f"\n{'='*60}")
+    print(f"✅ Generated {generated} new PNG previews")
+    print(f"⏭️  Skipped {skipped} files (already have PNGs)")
+    print(f"{'='*60}")
+
+
+if __name__ == "__main__":
+    if len(sys.argv) > 1:
+        # Process specific file
+        tif_file = sys.argv[1]
+        generate_png_preview(tif_file)
+    else:
+        # Process all files in predictions directory
+        generate_all_previews()

.temp/generate_report_only.py

@@ -0,0 +1,97 @@
+import os
+import glob
+import json
+from tabulate import tabulate
+
+print("📊 BẢNG SO SÁNH KẾT QUẢ CÁC MÔ HÌNH\n")
+
+# 1. Phân loại đất
+print("### 1. Nhóm Phân loại Lớp phủ (Land Classification)")
+land_data = []
+
+if os.path.exists("model_xgboost_info.json"):
+    with open("model_xgboost_info.json", 'r') as f:
+        data = json.load(f)
+        params = data.get('params', {})
+        param_str = f"estimators:{params.get('n_estimators')}, depth:{params.get('max_depth')}" if params else "N/A"
+        land_data.append([
+            data.get('model_type', 'XGBoost'),
+            data.get('accuracy', ''),
+            data.get('precision', ''),
+            data.get('recall', ''),
+            data.get('f1_score', ''),
+            param_str
+        ])
+
+for info_file in glob.glob("model_train/*_info.json"):
+    with open(info_file, 'r') as f:
+        data = json.load(f)
+        # Support both 'accuracy' and 'test_accuracy'
+        acc = data.get('accuracy', data.get('test_accuracy', ''))
+        
+        f1 = data.get('f1_score', '')
+        precision = data.get('precision', '')
+        recall = data.get('recall', '')
+        
+        clf_rep = data.get('classification_report')
+        if isinstance(clf_rep, dict) and 'macro avg' in clf_rep:
+            if not f1:
+                f1 = clf_rep['macro avg'].get('f1-score', '')
+            if not precision:
+                precision = clf_rep['macro avg'].get('precision', '')
+            if not recall:
+                recall = clf_rep['macro avg'].get('recall', '')
+            
+        if not acc and not f1:
+            continue
+            
+        params = data.get('params', {})
+        param_str = f"estimators:{params.get('n_estimators')}, depth:{params.get('max_depth')}" if params else "N/A"
+        
+        if data.get('model_type') == 'RandomForest_RealData':
+            param_str = "estimators:100, depth:15"
+            
+        land_data.append([
+            data.get('model_type', ''),
+            acc,
+            precision,
+            recall,
+            f1,
+            param_str
+        ])
+
+if land_data:
+    print(tabulate(land_data, headers=["Model", "Accuracy", "Precision", "Recall", "F1-Score", "Parameters"], tablefmt="github"))
+print("\n")
+
+# 2. Xóa mây
+print("### 2. Nhóm Xóa mây (Cloud Removal)")
+cloud_data = []
+for info_file in glob.glob("cloud_removal_model/*_info.json"):
+    with open(info_file, 'r') as f:
+        data = json.load(f)
+        cloud_data.append([
+            data.get('model_type', ''),
+            data.get('epoch', ''),
+            data.get('train_loss', ''),
+            data.get('val_loss', '')
+        ])
+if cloud_data:
+    print(tabulate(cloud_data, headers=["Model", "Epochs", "Train Loss", "Val Loss"], tablefmt="github"))
+print("\n")
+
+# 3. Dự báo NDVI
+print("### 3. Nhóm Dự báo Thực vật (NDVI Forecasting)")
+ndvi_data = []
+for info_file in glob.glob("ndvi_forecast_model/*_info.json"):
+    with open(info_file, 'r') as f:
+        data = json.load(f)
+        ndvi_data.append([
+            data.get('model_type', ''),
+            data.get('rmse', ''),
+            data.get('mae', ''),
+            data.get('epoch', 'N/A')
+        ])
+if ndvi_data:
+    print(tabulate(ndvi_data, headers=["Model", "RMSE", "MAE", "Epochs"], tablefmt="github"))
+print("\n")

.temp/inspect_cells_train.py

@@ -0,0 +1,7 @@
+import json
+nb = json.load(open('01.train_ODC.ipynb'))
+for idx, cell in enumerate(nb['cells']):
+    if cell['cell_type'] == 'code':
+        print(f"Cell {idx}:")
+        print("".join(cell['source'][:3]))
+        print("-" * 20)

.temp/inspect_model_odc.py

@@ -0,0 +1,64 @@
+"""
+Inspect model_odc.joblib to see what it actually contains
+"""
+
+import joblib
+from pathlib import Path
+
+model_path = Path("model_train/model_odc.joblib")
+
+if model_path.exists():
+    print("Loading model_odc.joblib...")
+    model_data = joblib.load(model_path)
+    
+    print(f"\nModel type: {type(model_data)}")
+    print(f"Model class: {model_data.__class__.__name__}")
+    
+    # Check if it's a dict
+    if isinstance(model_data, dict):
+        print(f"\nModel is a dict with keys: {model_data.keys()}")
+        model = model_data.get('model')
+    else:
+        model = model_data
+    
+    print(f"\nActual model type: {type(model)}")
+    print(f"Actual model class: {model.__class__.__name__}")
+    
+    # Try to get feature info
+    if hasattr(model, 'n_features_in_'):
+        print(f"\nn_features_in_: {model.n_features_in_}")
+    
+    if hasattr(model, 'feature_names_in_'):
+        print(f"feature_names_in_: {model.feature_names_in_}")
+    
+    # If it's a GridSearchCV
+    if hasattr(model, 'best_estimator_'):
+        print(f"\nThis is a GridSearchCV!")
+        print(f"Best estimator: {model.best_estimator_}")
+        
+        best_est = model.best_estimator_
+        if hasattr(best_est, 'steps'):
+            print(f"\nPipeline steps:")
+            for step_name, step in best_est.steps:
+                print(f"  - {step_name}: {step.__class__.__name__}")
+                if hasattr(step, 'n_features_in_'):
+                    print(f"      n_features_in_: {step.n_features_in_}")
+    
+    # If it's a Pipeline
+    if hasattr(model, 'steps'):
+        print(f"\nThis is a Pipeline!")
+        print(f"Pipeline steps:")
+        for step_name, step in model.steps:
+            print(f"  - {step_name}: {step.__class__.__name__}")
+            if hasattr(step, 'n_features_in_'):
+                print(f"      n_features_in_: {step.n_features_in_}")
+    
+    # Try to get booster for XGBoost
+    try:
+        if hasattr(model, 'get_booster'):
+            print(f"\nXGBoost num_features: {model.get_booster().num_features()}")
+    except:
+        pass
+    
+else:
+    print(f"Model file not found: {model_path}")

.temp/log_cnn.txt

@@ -0,0 +1,18 @@
+🚀 BẮT ĐẦU HUẤN LUYỆN MÔ HÌNH CNN (GPU & CACHE)
+Initializing FeatureExtractor (mode=extended)...
+📦 Đang load cache: training_data_507bd2ba4ec0d3fe107839cbf73a7a7d.joblib...
+✅ Loaded 632 samples từ cache!
+   ⚡ Đã bỏ qua download위성 data (tiết kiệm thời gian)
+[CACHE HIT] Using cached dataset with 632 samples
+Training CNN model...
+Building CNN model on cuda...
+Training CNN model with PyTorch...
+CNN Epoch 10/15, Loss: 1.2171
+Evaluating model...
+Generating classification report...
+Saving model...
+[MODEL MANAGER] Saving model to: model_train/model_cnn_auto.joblib
+[MODEL MANAGER] Saving metadata to: model_train/model_cnn_auto_info.json
+[MODEL MANAGER] Model saved successfully!
+Training complete!
+✅ Hoàn thành! Accuracy: 0.5748

.temp/log_dt.txt

@@ -0,0 +1,15 @@
+🚀 BẮT ĐẦU HUẤN LUYỆN MÔ HÌNH DECISION TREE (GPU & CACHE)
+Initializing FeatureExtractor (mode=extended)...
+📦 Đang load cache: training_data_507bd2ba4ec0d3fe107839cbf73a7a7d.joblib...
+✅ Loaded 632 samples từ cache!
+   ⚡ Đã bỏ qua download위성 data (tiết kiệm thời gian)
+[CACHE HIT] Using cached dataset with 632 samples
+Training DECISION_TREE model...
+Evaluating model...
+Generating classification report...
+Saving model...
+[MODEL MANAGER] Saving model to: model_train/model_decision_tree_auto.joblib
+[MODEL MANAGER] Saving metadata to: model_train/model_decision_tree_auto_info.json
+[MODEL MANAGER] Model saved successfully!
+Training complete!
+✅ Hoàn thành! Accuracy: 0.5906

.temp/log_mobilenet.txt

@@ -0,0 +1,30 @@
+🚀 BẮT ĐẦU HUẤN LUYỆN MÔ HÌNH MOBILENET-LRASPP (GPU & CACHE)
+Initializing FeatureExtractor (mode=extended)...
+📦 Đang load cache: training_data_507bd2ba4ec0d3fe107839cbf73a7a7d.joblib...
+✅ Loaded 632 samples từ cache!
+   ⚡ Đã bỏ qua download위성 data (tiết kiệm thời gian)
+[CACHE HIT] Using cached dataset with 632 samples
+Training MOBILENET-LRASPP model...
+Building MobileNetV3 + LR-ASPP model on cuda...
+[MOBILENET] Class distribution: [ 48  89   3  86  74  38 117  50]
+[MOBILENET] Class weights: [0.37418982 0.20181024 5.98703525 0.20885013 0.24271772 0.47266082
+ 0.15351377 0.35922223]
+Training MobileNetV3 + LR-ASPP model with PyTorch...
+MobileNet Epoch 5/25, Train Loss: 1.0614, Val Loss: 1.3584, Val Acc: 40.16%, LR: 0.000800
+[MOBILENET] Epoch 5/25 - Train Loss: 1.0614, Val Loss: 1.3584, Val Acc: 40.16%
+MobileNet Epoch 10/25, Train Loss: 0.9054, Val Loss: 0.8582, Val Acc: 59.06%, LR: 0.000800
+[MOBILENET] Epoch 10/25 - Train Loss: 0.9054, Val Loss: 0.8582, Val Acc: 59.06%
+MobileNet Epoch 15/25, Train Loss: 0.7728, Val Loss: 0.8231, Val Acc: 61.42%, LR: 0.000800
+[MOBILENET] Epoch 15/25 - Train Loss: 0.7728, Val Loss: 0.8231, Val Acc: 61.42%
+MobileNet Epoch 20/25, Train Loss: 0.7125, Val Loss: 0.8783, Val Acc: 59.84%, LR: 0.000400
+[MOBILENET] Epoch 20/25 - Train Loss: 0.7125, Val Loss: 0.8783, Val Acc: 59.84%
+[MOBILENET] Early stopping at epoch 24 (best val loss: 0.8029)
+MobileNet early stopped at epoch 24
+Evaluating model...
+Generating classification report...
+Saving model...
+[MODEL MANAGER] Saving model to: model_train/model_mobilenet-lraspp_auto.joblib
+[MODEL MANAGER] Saving metadata to: model_train/model_mobilenet-lraspp_auto_info.json
+[MODEL MANAGER] Model saved successfully!
+Training complete!
+✅ Hoàn thành! Accuracy: 0.5669

.temp/log_rf.txt

@@ -0,0 +1,15 @@
+🚀 BẮT ĐẦU HUẤN LUYỆN MÔ HÌNH RANDOM FOREST (GPU & CACHE)
+Initializing FeatureExtractor (mode=extended)...
+📦 Đang load cache: training_data_507bd2ba4ec0d3fe107839cbf73a7a7d.joblib...
+✅ Loaded 632 samples từ cache!
+   ⚡ Đã bỏ qua download위성 data (tiết kiệm thời gian)
+[CACHE HIT] Using cached dataset with 632 samples
+Training RANDOM_FOREST model...
+Evaluating model...
+Generating classification report...
+Saving model...
+[MODEL MANAGER] Saving model to: model_train/model_random_forest_auto.joblib
+[MODEL MANAGER] Saving metadata to: model_train/model_random_forest_auto_info.json
+[MODEL MANAGER] Model saved successfully!
+Training complete!
+✅ Hoàn thành! Accuracy: 0.6142

.temp/log_svm.txt

@@ -0,0 +1,15 @@
+🚀 BẮT ĐẦU HUẤN LUYỆN MÔ HÌNH SVM (GPU & CACHE)
+Initializing FeatureExtractor (mode=extended)...
+📦 Đang load cache: training_data_507bd2ba4ec0d3fe107839cbf73a7a7d.joblib...
+✅ Loaded 632 samples từ cache!
+   ⚡ Đã bỏ qua download위성 data (tiết kiệm thời gian)
+[CACHE HIT] Using cached dataset with 632 samples
+Training SVM model...
+Evaluating model...
+Generating classification report...
+Saving model...
+[MODEL MANAGER] Saving model to: model_train/model_svm_auto.joblib
+[MODEL MANAGER] Saving metadata to: model_train/model_svm_auto_info.json
+[MODEL MANAGER] Model saved successfully!
+Training complete!
+✅ Hoàn thành! Accuracy: 0.5827

.temp/log_swin.txt

@@ -0,0 +1,30 @@
+🚀 BẮT ĐẦU HUẤN LUYỆN MÔ HÌNH SWIN-UNET (GPU & CACHE)
+Initializing FeatureExtractor (mode=extended)...
+📦 Đang load cache: training_data_507bd2ba4ec0d3fe107839cbf73a7a7d.joblib...
+✅ Loaded 632 samples từ cache!
+   ⚡ Đã bỏ qua download위성 data (tiết kiệm thời gian)
+[CACHE HIT] Using cached dataset with 632 samples
+Training SWIN-UNET model...
+Building Swin-UNet model on cuda...
+[SWIN-UNET] Class distribution: [ 48  89   3  86  74  38 117  50]
+[SWIN-UNET] Class weights: [0.37418982 0.20181024 5.98703525 0.20885013 0.24271772 0.47266082
+ 0.15351377 0.35922223]
+Training Swin-UNet model with PyTorch (with class weights)...
+Swin-UNet Epoch 5/40, Train Loss: 1.4096, Val Loss: 1.2804, Val Acc: 48.03%, LR: 0.000293
+[SWIN-UNET] Epoch 5/40 - Train Loss: 1.4096, Val Loss: 1.2804, Val Acc: 48.03%
+Swin-UNet Epoch 10/40, Train Loss: 1.1129, Val Loss: 1.2746, Val Acc: 57.48%, LR: 0.000271
+[SWIN-UNET] Epoch 10/40 - Train Loss: 1.1129, Val Loss: 1.2746, Val Acc: 57.48%
+Swin-UNet Epoch 15/40, Train Loss: 1.0479, Val Loss: 1.3126, Val Acc: 50.39%, LR: 0.000238
+[SWIN-UNET] Epoch 15/40 - Train Loss: 1.0479, Val Loss: 1.3126, Val Acc: 50.39%
+Swin-UNet Epoch 20/40, Train Loss: 0.9548, Val Loss: 1.1118, Val Acc: 52.76%, LR: 0.000196
+[SWIN-UNET] Epoch 20/40 - Train Loss: 0.9548, Val Loss: 1.1118, Val Acc: 52.76%
+[SWIN-UNET] Early stopping at epoch 22 (best val loss: 1.1096)
+Swin-UNet early stopped at epoch 22
+Evaluating model...
+Generating classification report...
+Saving model...
+[MODEL MANAGER] Saving model to: model_train/model_swin-unet_auto.joblib
+[MODEL MANAGER] Saving metadata to: model_train/model_swin-unet_auto_info.json
+[MODEL MANAGER] Model saved successfully!
+Training complete!
+✅ Hoàn thành! Accuracy: 0.5354

.temp/log_train_2d_final.txt

@@ -0,0 +1,38 @@
+🚀 BẮT ĐẦU PIPELINE 2D PATCH-BASED & CLOUD REMOVAL
+Loading 2D patches from dataset_cache/training_data_2d.joblib...
+Training 2D CNN with Data Augmentation...
+Epoch 1/150 - Loss: 2.2272 - Test Acc: 0.0752 🌟
+Epoch 2/150 - Loss: 2.0843 - Test Acc: 0.0796 🌟
+Epoch 4/150 - Loss: 2.0350 - Test Acc: 0.1372 🌟
+Epoch 8/150 - Loss: 2.0447 - Test Acc: 0.1637 🌟
+Epoch 10/150 - Loss: 2.0397 - Test Acc: 0.1372
+Epoch 12/150 - Loss: 2.0353 - Test Acc: 0.2168 🌟
+Epoch 20/150 - Loss: 2.0139 - Test Acc: 0.1372
+Traceback (most recent call last):
+  File "/home/x79/remote-sensing/train_land_2d_patch.py", line 298, in <module>
+    main()
+  File "/home/x79/remote-sensing/train_land_2d_patch.py", line 294, in main
+    train_2d_model(X, y)
+  File "/home/x79/remote-sensing/train_land_2d_patch.py", line 219, in train_2d_model
+    for batch_X, batch_y in train_loader:
+  File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 725, in __next__
+    data = self._next_data()
+  File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 785, in _next_data
+    data = self._dataset_fetcher.fetch(index)  # may raise StopIteration
+  File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 54, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 54, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/home/x79/remote-sensing/train_land_2d_patch.py", line 192, in __getitem__
+    x = transform(x)
+  File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torchvision/transforms/transforms.py", line 95, in __call__
+    img = t(img)
+  File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torchvision/transforms/transforms.py", line 752, in forward
+    return F.vflip(img)
+  File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torchvision/transforms/functional.py", line 757, in vflip
+    def vflip(img: Tensor) -> Tensor:
+KeyboardInterrupt

.temp/log_train_v4.txt

@@ -0,0 +1,39 @@
+🚀 V4: TÍCH HỢP RADAR SENTINEL-1 (32-CHANNELS FUSION)
+============================================================
+Clean FUSION data: (252, 32, 16, 16), 7 classes, [31, 38, 32, 49, 23, 75, 4]
+
+============================================================
+32-CHANNELS FUSION CNN
+============================================================
+  Ep 1 Fusion-Acc=0.1961 🌟
+  Ep 3 Fusion-Acc=0.2745 🌟
+  Ep 4 Fusion-Acc=0.4706 🌟
+  Ep 5 Fusion-Acc=0.5490 🌟
+  Ep 6 Fusion-Acc=0.7059 🌟
+  Ep 7 Fusion-Acc=0.7451 🌟
+  Ep 8 Fusion-Acc=0.8431 🌟
+  Ep 15 Fusion-Acc=0.8627 🌟
+
+✅ CNN Fusion best: 0.8627
+
+============================================================
+HYBRID FUSION: CNN embed + S1/S2 Rich features + XGBoost
+============================================================
+Extracted 2182 fusion features per sample
+  Final Feature Vector: (252, 2694)
+  ✅ Hybrid Fusion Acc: 0.8627
+    Fold 1: 0.9412
+    Fold 2: 0.9020
+    Fold 3: 0.9200
+    Fold 4: 0.8800
+    Fold 5: 0.9000
+  ✅ CV Mean: 0.9086 ± 0.0206
+
+============================================================
+📊 FINAL RESULTS V4 (WITH RADAR)
+============================================================
+  ✅ Hybrid Fusion CV: 0.9086
+  📈 CNN Fusion (32ch): 0.8627
+  📈 Hybrid Fusion (CNN+XGB): 0.8627
+
+🏆 BEST: 0.9086

.temp/log_train_v5.txt

@@ -0,0 +1,18 @@
+
+============================================================
+HYBRID FUSION ENSEMBLE: CNN embed + S1/S2 Rich features + XGB/LGBM/ETC
+============================================================
+  Final Feature Vector: (443, 2694)
+    Fold 1: 0.8876
+    Fold 2: 0.9438
+    Fold 3: 0.9438
+    Fold 4: 0.9659
+    Fold 5: 0.9432
+  ✅ Ensemble CV Mean: 0.9369 ± 0.0261
+
+============================================================
+📊 FINAL RESULTS V5 (ENSEMBLE + RADAR)
+============================================================
+  ✅ Hybrid Fusion Ensemble CV: 0.9369
+
+🏆 BEST: 0.9369

.temp/log_train_v6.txt

@@ -0,0 +1,21 @@
+🚀 V6: EXHAUSTIVE HYPERPARAMETER TUNING
+============================================================
+Data: (443, 32, 16, 16), 7 classes, dist=[65, 55, 48, 72, 74, 124, 5]
+
+--- Training Multi-Seed CNN Ensemble ---
+  Seed 42: CNN Acc = 0.8989
+  Seed 123: CNN Acc = 0.8652
+  Seed 777: CNN Acc = 0.8876
+  Multi-seed CNN embedding: (443, 1536)
+  Total features: (443, 3940)
+
+============================================================
+🔬 EXHAUSTIVE HYPERPARAMETER SEARCH
+============================================================
+  🏆 XGB-deep: 0.9526 ± 0.0110  (folds: ['0.955', '0.933', '0.966', '0.955', '0.955'])
+  ✅ XGB-shallow: 0.9436 ± 0.0173  (folds: ['0.944', '0.910', '0.955', '0.955', '0.955'])
+  🏆 XGB-balanced: 0.9504 ± 0.0113  (folds: ['0.944', '0.933', '0.955', '0.955', '0.966'])
+  ✅ LGBM-tuned: 0.9458 ± 0.0149  (folds: ['0.955', '0.921', '0.966', '0.943', '0.943'])
+  ✅ LGBM-conservative: 0.9481 ± 0.0152  (folds: ['0.944', '0.921', '0.966', '0.955', '0.955'])
+  🏆 ETC-deep: 0.9572 ± 0.0082  (folds: ['0.944', '0.955', '0.955', '0.966', '0.966'])
+  🏆 RF-tuned: 0.9549 ± 0.0099  (folds: ['0.944', '0.955', '0.944', '0.966', '0.966'])

.temp/log_tune_swin_unet.txt

@@ -0,0 +1,157 @@
+🚀 BẮT ĐẦU TÌM KIẾM SIÊU THAM SỐ CHO SWIN-UNET
+Loading data from dataset_cache/training_data_507bd2ba4ec0d3fe107839cbf73a7a7d.joblib...
+Using device: cuda
+
+[1/48] Training with params: {'embed_dim': 64, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.5984
+🌟 NEW BEST ACCURACY: 0.5984
+
+[2/48] Training with params: {'embed_dim': 64, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.6063
+🌟 NEW BEST ACCURACY: 0.6063
+
+[3/48] Training with params: {'embed_dim': 64, 'lr': 0.001, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.6142
+🌟 NEW BEST ACCURACY: 0.6142
+
+[4/48] Training with params: {'embed_dim': 64, 'lr': 0.001, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.6772
+🌟 NEW BEST ACCURACY: 0.6772
+
+[5/48] Training with params: {'embed_dim': 64, 'lr': 0.0005, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.5827
+
+[6/48] Training with params: {'embed_dim': 64, 'lr': 0.0005, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.5984
+
+[7/48] Training with params: {'embed_dim': 64, 'lr': 0.0005, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.5669
+
+[8/48] Training with params: {'embed_dim': 64, 'lr': 0.0005, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.6142
+
+[9/48] Training with params: {'embed_dim': 64, 'lr': 0.0001, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.6142
+
+[10/48] Training with params: {'embed_dim': 64, 'lr': 0.0001, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.5118
+
+[11/48] Training with params: {'embed_dim': 64, 'lr': 0.0001, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.5591
+
+[12/48] Training with params: {'embed_dim': 64, 'lr': 0.0001, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.6063
+
+[13/48] Training with params: {'embed_dim': 128, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.7008
+🌟 NEW BEST ACCURACY: 0.7008
+
+[14/48] Training with params: {'embed_dim': 128, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.6142
+
+[15/48] Training with params: {'embed_dim': 128, 'lr': 0.001, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.6772
+
+[16/48] Training with params: {'embed_dim': 128, 'lr': 0.001, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.6063
+
+[17/48] Training with params: {'embed_dim': 128, 'lr': 0.0005, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.5906
+
+[18/48] Training with params: {'embed_dim': 128, 'lr': 0.0005, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.6457
+
+[19/48] Training with params: {'embed_dim': 128, 'lr': 0.0005, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.5906
+
+[20/48] Training with params: {'embed_dim': 128, 'lr': 0.0005, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.5906
+
+[21/48] Training with params: {'embed_dim': 128, 'lr': 0.0001, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.6220
+
+[22/48] Training with params: {'embed_dim': 128, 'lr': 0.0001, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.6457
+
+[23/48] Training with params: {'embed_dim': 128, 'lr': 0.0001, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.5984
+
+[24/48] Training with params: {'embed_dim': 128, 'lr': 0.0001, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.6063
+
+[25/48] Training with params: {'embed_dim': 256, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.7087
+🌟 NEW BEST ACCURACY: 0.7087
+
+[26/48] Training with params: {'embed_dim': 256, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.7323
+🌟 NEW BEST ACCURACY: 0.7323
+
+[27/48] Training with params: {'embed_dim': 256, 'lr': 0.001, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.6457
+
+[28/48] Training with params: {'embed_dim': 256, 'lr': 0.001, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.6142
+
+[29/48] Training with params: {'embed_dim': 256, 'lr': 0.0005, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.5984
+
+[30/48] Training with params: {'embed_dim': 256, 'lr': 0.0005, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.5906
+
+[31/48] Training with params: {'embed_dim': 256, 'lr': 0.0005, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.6142
+
+[32/48] Training with params: {'embed_dim': 256, 'lr': 0.0005, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.7008
+
+[33/48] Training with params: {'embed_dim': 256, 'lr': 0.0001, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.6299
+
+[34/48] Training with params: {'embed_dim': 256, 'lr': 0.0001, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.6299
+
+[35/48] Training with params: {'embed_dim': 256, 'lr': 0.0001, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.6378
+
+[36/48] Training with params: {'embed_dim': 256, 'lr': 0.0001, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.6457
+
+[37/48] Training with params: {'embed_dim': 512, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.6142
+
+[38/48] Training with params: {'embed_dim': 512, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.6457
+
+[39/48] Training with params: {'embed_dim': 512, 'lr': 0.001, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.6378
+
+[40/48] Training with params: {'embed_dim': 512, 'lr': 0.001, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.6299
+
+[41/48] Training with params: {'embed_dim': 512, 'lr': 0.0005, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.6378
+
+[42/48] Training with params: {'embed_dim': 512, 'lr': 0.0005, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.6220
+
+[43/48] Training with params: {'embed_dim': 512, 'lr': 0.0005, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.6142
+
+[44/48] Training with params: {'embed_dim': 512, 'lr': 0.0005, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.7008
+
+[45/48] Training with params: {'embed_dim': 512, 'lr': 0.0001, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.6457
+
+[46/48] Training with params: {'embed_dim': 512, 'lr': 0.0001, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.7323
+
+[47/48] Training with params: {'embed_dim': 512, 'lr': 0.0001, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.6693
+
+[48/48] Training with params: {'embed_dim': 512, 'lr': 0.0001, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.6457
+
+✅ Đã lưu mô hình tốt nhất (Acc: 0.7323) vào land_classification_model/model_swin-unet_optimized_95.joblib
+Cấu hình tốt nhất: {'embed_dim': 256, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 500}

.temp/log_ultimate_95.txt

@@ -0,0 +1,117 @@
+🚀 CHIẾN LƯỢC TOÀN DIỆN ĐẠT >95% ACCURACY
+============================================================
+Loaded data: X=(706, 24, 16, 16), y=(706,)
+Labels unique: [-1  0  1  2  3  4  5  6]
+After cleanup: X=(652, 24, 16, 16), y=(652,) (removed 54 bad samples)
+Remapped labels: [0 1 2 3 4 5 6]
+  Class 0: 65 samples
+  Class 1: 52 samples
+  Class 2: 48 samples
+  Class 3: 72 samples
+  Class 4: 108 samples
+  Class 5: 219 samples
+  Class 6: 88 samples
+
+============================================================
+STRATEGY 5: Flat pixel features + XGBoost (sanity check)
+============================================================
+  Flat features: (652, 6144)
+  ✅ Flat XGBoost acc: 0.7328
+
+============================================================
+STRATEGY 1: Lightweight CNN (no upsampling)
+============================================================
+Device: cuda
+  Epoch 1/300 Loss=1.8379 Acc=0.0763 🌟
+  Epoch 2/300 Loss=1.5825 Acc=0.2824 🌟
+  Epoch 3/300 Loss=1.4412 Acc=0.5649 🌟
+  Epoch 4/300 Loss=1.3274 Acc=0.6336 🌟
+  Epoch 5/300 Loss=1.2893 Acc=0.6870 🌟
+  Epoch 8/300 Loss=1.2140 Acc=0.7099 🌟
+  Epoch 11/300 Loss=1.2224 Acc=0.7252 🌟
+  Epoch 14/300 Loss=1.1087 Acc=0.7328 🌟
+  Epoch 16/300 Loss=1.1081 Acc=0.7710 🌟
+  Epoch 18/300 Loss=1.1847 Acc=0.7939 🌟
+  Epoch 20/300 Loss=1.0316 Acc=0.7786 (patience=2)
+  Epoch 24/300 Loss=1.0465 Acc=0.8092 🌟
+  Epoch 26/300 Loss=0.9583 Acc=0.8397 🌟
+  Epoch 40/300 Loss=0.9361 Acc=0.8626 🌟
+  Epoch 60/300 Loss=0.9807 Acc=0.8092 (patience=20)
+  Epoch 80/300 Loss=0.9459 Acc=0.8015 (patience=40)
+  Epoch 100/300 Loss=0.8443 Acc=0.7939 (patience=60)
+  Early stop at epoch 100
+  ✅ LightCNN best acc: 0.8626
+
+============================================================
+STRATEGY 2: Hybrid CNN embeddings + XGBoost
+============================================================
+  CNN embeddings: (652, 256)
+Extracted 316 rich features per sample
+  Combined features: (652, 572)
+  ✅ Hybrid XGBoost acc: 0.8244
+
+============================================================
+STRATEGY 3: Rich Features + Stacking Ensemble
+============================================================
+Extracted 316 rich features per sample
+  XGBoost: 0.7939
+  LightGBM: 0.7786
+  ExtraTrees: 0.7863
+  RandomForest: 0.7710
+  GBM: 0.7710
+[06:55:44] WARNING: /__w/xgboost/xgboost/src/learner.cc:782: 
+Parameters: { "use_label_encoder" } are not used.
+[06:55:44] WARNING: /__w/xgboost/xgboost/src/learner.cc:782: 
+Parameters: { "use_label_encoder" } are not used.
+
+
+[06:55:44] WARNING: /__w/xgboost/xgboost/src/learner.cc:782: 
+Parameters: { "use_label_encoder" } are not used.
+[06:55:44] WARNING: /__w/xgboost/xgboost/src/learner.cc:782: 
+Parameters: { "use_label_encoder" } are not used.
+
+
+[06:55:44] WARNING: /__w/xgboost/xgboost/src/learner.cc:782: 
+Parameters: { "use_label_encoder" } are not used.
+
+[06:56:17] WARNING: /__w/xgboost/xgboost/src/common/error_msg.cc:62: Falling back to prediction using DMatrix due to mismatched devices. This might lead to higher memory usage and slower performance. XGBoost is running on: cuda:0, while the input data is on: cpu.
+Potential solutions:
+- Use a data structure that matches the device ordinal in the booster.
+- Set the device for booster before call to inplace_predict.
+
+This warning will only be shown once.
+
+  Stacking Ensemble: 0.7710
+  Voting Ensemble: 0.7786
+  ✅ Best ensemble: XGBoost = 0.7939
+Extracted 316 rich features per sample
+
+============================================================
+STRATEGY 4: 5-Fold Stratified Cross-Validation
+============================================================
+  Fold 1: 0.7939
+  Fold 2: 0.8244
+  Fold 3: 0.7769
+  Fold 4: 0.8154
+  Fold 5: 0.7615
+  ✅ CV Mean: 0.7944 ± 0.0234
+
+============================================================
+📊 TỔNG KẾT KẾT QUẢ
+============================================================
+  📈 LightCNN: 0.8626
+  📈 Hybrid CNN+XGBoost: 0.8244
+  📈 CV Mean (XGBoost rich): 0.7944
+  📈 Ensemble XGBoost: 0.7939
+  📈 Ensemble ExtraTrees: 0.7863
+  📈 Ensemble LightGBM: 0.7786
+  📈 Ensemble Voting: 0.7786
+  📈 Ensemble RandomForest: 0.7710
+  📈 Ensemble GBM: 0.7710
+  📈 Ensemble Stacking: 0.7710
+  📈 Flat XGBoost (baseline): 0.7328
+
+🏆 BEST: LightCNN = 0.8626
+
+✅ Kết quả đã được lưu vào model_train/ultimate_results.json
+⚠️ Chưa đạt 95%. Best = 0.8626. Cần thêm dữ liệu hoặc feature engineering.

.temp/log_ultimate_v2.txt

@@ -0,0 +1,69 @@
+🚀 CHIẾN LƯỢC V2: TOÀN DIỆN ĐẠT >95%
+============================================================
+Clean data: (652, 24, 16, 16), 7 classes
+
+============================================================
+CNN + TTA (Test-Time Augmentation)
+============================================================
+  Ep 1 Loss=1.6500 TTA-Acc=0.1450 🌟
+  Ep 2 Loss=1.4633 TTA-Acc=0.3511 🌟
+  Ep 3 Loss=1.3316 TTA-Acc=0.6336 🌟
+  Ep 4 Loss=1.2101 TTA-Acc=0.7252 🌟
+  Ep 6 Loss=1.2056 TTA-Acc=0.8015 🌟
+  Ep 13 Loss=1.1216 TTA-Acc=0.8244 🌟
+  Ep 23 Loss=0.9355 TTA-Acc=0.8321 🌟
+  Ep 30 Loss=0.9346 TTA-Acc=0.8092 (pat=7)
+  Ep 60 Loss=0.9658 TTA-Acc=0.7634 (pat=37)
+  Ep 69 Loss=0.8988 TTA-Acc=0.8397 🌟
+  Ep 74 Loss=0.9159 TTA-Acc=0.8550 🌟
+  Ep 90 Loss=0.8761 TTA-Acc=0.8168 (pat=16)
+  Ep 120 Loss=0.9473 TTA-Acc=0.8092 (pat=46)
+  Ep 139 Loss=0.9317 TTA-Acc=0.8626 🌟
+  Ep 150 Loss=0.7716 TTA-Acc=0.8397 (pat=11)
+  Ep 175 Loss=0.7432 TTA-Acc=0.8702 🌟
+  Ep 180 Loss=0.8290 TTA-Acc=0.8702 (pat=5)
+  Ep 210 Loss=0.8060 TTA-Acc=0.8626 (pat=35)
+  Ep 240 Loss=0.6980 TTA-Acc=0.8473 (pat=65)
+  Early stop ep 255
+  ✅ CNN+TTA best: 0.8702
+
+============================================================
+RICH FEATURES V2 + ENSEMBLE
+============================================================
+Extracted 1713 features per sample
+  XGB: 0.7557
+  LGBM: 0.7634
+  ET: 0.7481
+  RF: 0.7557
+  Voting: 0.7634
+
+  5-Fold CV:
+    Fold 1: 0.7557
+    Fold 2: 0.7939
+    Fold 3: 0.7769
+    Fold 4: 0.8308
+    Fold 5: 0.8000
+  CV: 0.7915 ± 0.0249
+
+============================================================
+HYBRID V2: CNN embed + Rich features + XGBoost
+============================================================
+Extracted 1713 features per sample
+  Combined: (652, 2097)
+  ✅ Hybrid V2: 0.8244
+  CV: 0.9142 ± 0.0194
+
+============================================================
+📊 KẾT QUẢ TỔNG HỢP V2
+============================================================
+  ✅ Hybrid CV: 0.9142
+  📈 CNN+TTA: 0.8702
+  📈 Hybrid V2: 0.8244
+  📈 Ens CV: 0.7915
+  📈 Ens_LGBM: 0.7634
+  📈 Ens_Vote: 0.7634
+  📈 Ens_XGB: 0.7557
+  📈 Ens_RF: 0.7557
+  📈 Ens_ET: 0.7481
+
+🏆 BEST: Hybrid CV = 0.9142

.temp/log_ultimate_v3.txt

@@ -0,0 +1,27 @@
+🚀 V3: MULTI-SEED ENSEMBLE + T0-ONLY + SELF-TRAINING
+============================================================
+Clean: (652, 24, 16, 16), 7 classes, [65, 52, 48, 72, 108, 219, 88]
+
+============================================================
+MULTI-SEED CNN ENSEMBLE (10 models)
+============================================================
+  Seed 0: 0.8473
+  Seed 1: 0.8702
+  Seed 2: 0.8550
+  Seed 3: 0.8550
+  Seed 4: 0.8702
+  Seed 5: 0.8626
+  Seed 6: 0.8702
+  Seed 7: 0.8702
+  Seed 8: 0.8702
+  Seed 9: 0.8702
+  ✅ 10-Model Ensemble TTA: 0.8473
+
+============================================================
+TIMESTEP-0-ONLY XGBoost (cleanest data)
+============================================================
+  T0 valid: 539/652
+  Features: (539, 1638)
+  XGB t0: 0.6852
+  LGBM t0: 0.6296
+  ET t0: 0.6852

.temp/log_xgboost.txt

@@ -0,0 +1,15 @@
+🚀 BẮT ĐẦU HUẤN LUYỆN MÔ HÌNH XGBOOST (GPU & CACHE)
+Initializing FeatureExtractor (mode=extended)...
+📦 Đang load cache: training_data_507bd2ba4ec0d3fe107839cbf73a7a7d.joblib...
+✅ Loaded 632 samples từ cache!
+   ⚡ Đã bỏ qua download위성 data (tiết kiệm thời gian)
+[CACHE HIT] Using cached dataset with 632 samples
+Training XGBOOST model...
+Evaluating model...
+Generating classification report...
+Saving model...
+[MODEL MANAGER] Saving model to: model_train/model_xgboost_auto.joblib
+[MODEL MANAGER] Saving metadata to: model_train/model_xgboost_auto_info.json
+[MODEL MANAGER] Model saved successfully!
+Training complete!
+✅ Hoàn thành! Accuracy: 0.6378

.temp/new_import_ODC.py

@@ -1,8 +1,16 @@
+TEST_MODE = True
+RESOLUTION = 1000 if TEST_MODE else 10
 import matplotlib.pyplot as plt
 
 # Common imports and settings
 import os, sys
 os.environ['USE_PYGEOS'] = '0'
+os.environ["GDAL_HTTP_MAX_RETRY"] = "5"
+os.environ["GDAL_HTTP_RETRY_DELAY"] = "2"
+os.environ["GDAL_HTTP_CONNECTION_TIMEOUT"] = "10"
+os.environ["GDAL_HTTP_TIMEOUT"] = "30"
+os.environ["CPL_VSIL_CURL_ALLOWED_EXTENSIONS"] = ".tif,.tiff"
+os.environ["GDAL_DISABLE_READDIR_ON_OPEN"] = "YES"
 from IPython.display import Markdown
 import pandas as pd
 pd.set_option("display.max_rows", None)
@@ -13,15 +21,13 @@ import datacube
 from datacube.utils.rio import configure_s3_access
 from datacube.utils import masking
 from datacube.utils.cog import write_cog
-# https://github.com/GeoscienceAustralia/dea-notebooks/tree/develop/Tools
-from dea_tools.plotting import display_map, rgb
-from dea_tools.datahandling import mostcommon_crs
+# removed deafrica_tools imports to avoid ipyleaflet error
 
 # EASI defaults
-easinotebooksrepo = '/home/jovyan/easi-notebooks'
+easinotebooksrepo = '/home/x79/CSIROBoeingPhase4-Vietnam'
 if easinotebooksrepo not in sys.path: sys.path.append(easinotebooksrepo)
 from easi_tools import EasiDefaults, xarray_object_size, notebook_utils, unset_cachingproxy
-from easi_tools.load_s2l2a import load_s2l2a_with_offset
+# from easi_tools.load_s2l2a import load_s2l2a_with_offset
 from dask.distributed import progress
 
 # Data tools
@@ -31,7 +37,7 @@ from datetime import datetime
 # Datacube
 from datacube.utils import masking  # https://github.com/opendatacube/datacube-core/blob/develop/datacube/utils/masking.py
 from odc.algo import enum_to_bool   # https://github.com/opendatacube/odc-algo/blob/main/odc/algo/_masking.py
-from odc.algo import xr_reproject   # https://github.com/opendatacube/odc-algo/blob/main/odc/algo/_warp.py
+# removed xr_reproject
 from datacube.utils.geometry import GeoBox, box  # https://github.com/opendatacube/datacube-core/blob/develop/datacube/utils/geometry/_base.py
 
 # Holoviews, Datashader and Bokeh
@@ -51,13 +57,12 @@ import rioxarray
 hv.extension('bokeh', logo=False)
 
 from deafrica_tools.bandindices import calculate_indices
-from sklearn.ensemble import RandomForestClassifier
+from xgboost import XGBClassifier
 from sklearn.model_selection import train_test_split
 from sklearn.metrics import accuracy_score, classification_report
 from sklearn.preprocessing import LabelEncoder
 
 from sklearn.pipeline import Pipeline
-from sklearn.ensemble import RandomForestClassifier
 from sklearn.impute import SimpleImputer
 from sklearn.preprocessing import StandardScaler
 from sklearn.model_selection import GridSearchCV
@@ -82,57 +87,90 @@ import joblib
 
 
 def load_data(dc, date_range, longtitude_range, latitude_range):
-    product = 's2_l2a'
-    query = {
-        'product': product,                     # Product name
-        'x': longtitude_range,    # "x" axis bounds
-        'y': latitude_range,      # "y" axis bounds
-        'time': date_range,           # Any parsable date strings
-    }
-    native_crs = notebook_utils.mostcommon_crs(dc, query)
-    print(f'Most common native CRS: {native_crs}')
-    measurements = ['red', 'nir', 'scl']
+    import os, hashlib
+    cache_dir = "dataset_cache"
+    os.makedirs(cache_dir, exist_ok=True)
+    key_str = f"s2_{date_range}_{longtitude_range}_{latitude_range}"
+    cache_key = hashlib.md5(key_str.encode()).hexdigest() + ".nc"
+    cache_path = os.path.join(cache_dir, cache_key)
     
-    load_params = {
-        'measurements': measurements,                   # Selected measurement or alias names
-        'output_crs': native_crs,                       # Target EPSG code
-        'resolution': (-10, 10),                        # Target resolution
-        'group_by': 'solar_day',                        # Scene grouping
-        'dask_chunks': {'x': 2048, 'y': 2048},          # Dask chunks
-    }
-    data = load_s2l2a_with_offset(
-        dc,
-        query | load_params   # Combine the two dicts that contain our search and load parameters
+    if os.path.exists(cache_path):
+        print(f"✅ Loading cached S2 data from {cache_path}")
+        return xr.open_dataset(cache_path, engine='netcdf4')
+
+    product = 's2_l2a'
+    bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
+    
+    import pystac_client
+    import planetary_computer
+    import odc.stac
+    
+    catalog = pystac_client.Client.open(
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
+        modifier=planetary_computer.sign_inplace,
     )
+    search = catalog.search(
+        collections=["sentinel-2-l2a"],
+        bbox=bbox,
+        datetime=f"{date_range[0]}/{date_range[1]}",
+    )
+    items = list(search.items())
+    
+    data = odc.stac.load(
+        items,
+        bands=["red", "nir", "SCL"],
+        bbox=bbox,
+        crs="EPSG:32648",
+        resolution=RESOLUTION,
+        chunks={"x": 2048, "y": 2048, "time": 1},
+        groupby="solar_day"
+    )
+    if "SCL" in data.data_vars:
+        data = data.rename({"SCL": "scl"})
+        
+    print(f"💾 Caching S2 data to {cache_path}")
+    data = data.compute()
+    data.to_netcdf(cache_path, engine='netcdf4')
+    
     return data
 
 
 def mask_clean(data):
-    flag_name = 'scl'
-    flag_desc = masking.describe_variable_flags(data[flag_name])  # Pandas dataframe
-    display(flag_desc)
-    display(flag_desc.loc['qa'].values[1])
-    # Create a "data quality" Mask layer
-    flags_def = flag_desc.loc['qa'].values[1]
-    good_pixel_flags = [flags_def[str(i)] for i in [2, 4, 5, 6]]  # To pass strings to enum_to_bool()
-
-    # enum_to_bool calculates the pixel-wise "or" of each set of pixels given by good_pixel_flags
-    # 1 = good data
-    # 0 = "bad" data
-    good_pixel_mask = enum_to_bool(data[flag_name], good_pixel_flags)
+    # For Sentinel-2 L2A SCL: 
+    # 2: Dark Area Pixels, 4: Vegetation, 5: Not Vegetated, 6: Water
+    good_pixel_mask = data['scl'].isin([2, 4, 5, 6])
     data_layer_names = [x for x in data.data_vars if x != 'scl']
-    # Apply good pixel mask to blue, green, red and nir.
+    # Apply good pixel mask
     result = data[data_layer_names].where(good_pixel_mask).persist()
     return result
 
 
 def fill_nan(ndvi, time_split):
+    if len(ndvi.time) == 0:
+        return ndvi
+    
+    # If the total time duration is less than 90 days, skip seasonal splitting
+    try:
+        total_days = (ndvi.time[-1] - ndvi.time[0]).dt.days.item()
+        if total_days < 90:
+            return ndvi.bfill(dim="time").ffill(dim="time")
+    except Exception:
+        pass
+        
     rs = []
     for times in time_split:
+        try:
             tmp = ndvi.sel(time=times)
-        fill_ds = tmp.sel(time=times).bfill(dim='time')
-        fill_ds = fill_ds.sel(time=times).ffill(dim='time')   
+            if len(tmp.time) == 0:
+                continue
+            fill_ds = tmp.bfill(dim='time').ffill(dim='time')   
             rs.append(fill_ds)
+        except Exception:
+            continue
+            
+    if len(rs) == 0:
+        return ndvi.bfill(dim="time").ffill(dim="time")
+        
     merged_ndvi = xr.concat([i for i in rs], dim="time")
     fill_m = merged_ndvi.bfill(dim="time")
     fill_m = fill_m.ffill(dim="time")
@@ -144,10 +182,68 @@ def load_train_data(train_path):
     return train
 
 
-def load_sen1(name_vh, name_vv):
-    dsvv = rioxarray.open_rasterio(name_vv)
-    dsvh = rioxarray.open_rasterio(name_vh)
-    return dsvh, dsvv
+def load_sen1(bbox, time_range):
+    import os, hashlib
+    cache_dir = "dataset_cache"
+    os.makedirs(cache_dir, exist_ok=True)
+    key_str = f"s1_vh_vv_{bbox}_{time_range}"
+    cache_key_vh = hashlib.md5((key_str + "vh").encode()).hexdigest() + ".nc"
+    cache_key_vv = hashlib.md5((key_str + "vv").encode()).hexdigest() + ".nc"
+    cache_path_vh = os.path.join(cache_dir, cache_key_vh)
+    cache_path_vv = os.path.join(cache_dir, cache_key_vv)
+    
+    if os.path.exists(cache_path_vh) and os.path.exists(cache_path_vv):
+        print(f"✅ Loading cached S1 data from {cache_path_vh} and {cache_path_vv}")
+        ds_vh = xr.open_dataset(cache_path_vh, engine='netcdf4')
+        ds_vv = xr.open_dataset(cache_path_vv, engine='netcdf4')
+        return ds_vh[list(ds_vh.data_vars)[0]], ds_vv[list(ds_vv.data_vars)[0]]
+
+    import pystac_client
+    import planetary_computer
+    import odc.stac
+    
+    # Kết nối STAC Client
+    catalog = pystac_client.Client.open(
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
+        modifier=planetary_computer.sign_inplace,
+    )
+
+    # Tìm kiếm Items
+    search = catalog.search(
+        collections=["sentinel-1-rtc"],
+        bbox=bbox,
+        datetime=time_range,
+    )
+    items = list(search.items())
+
+    # Tải dữ liệu thành xarray Dataset
+    ds_s1 = odc.stac.load(
+        items,
+        bands=["vv", "vh"],
+        bbox=bbox,
+        crs="EPSG:32648",
+        resolution=RESOLUTION,
+        chunks={"x": 2048, "y": 2048, "time": 1}
+    )
+
+    # Tính giá trị trung vị theo thời gian
+    ds_median = ds_s1.median(dim="time").compute()
+    vv = ds_median["vv"]
+    vh = ds_median["vh"]
+
+    # Thêm chiều 'band' để giống hệt rioxarray
+    vv = vv.expand_dims(dim="band")
+    vh = vh.expand_dims(dim="band")
+
+    # Phục hồi metadata về toạ độ
+    vv = vv.rio.write_crs("EPSG:32648")
+    vh = vh.rio.write_crs("EPSG:32648")
+
+    print(f"💾 Caching S1 data to {cache_dir}")
+    vh.to_netcdf(cache_path_vh, engine='netcdf4')
+    vv.to_netcdf(cache_path_vv, engine='netcdf4')
+
+    return vh, vv
 
 
 def get_data_sen1_and_sen2(train, average_ndvi, dsvh, dsvv):
@@ -192,7 +288,7 @@ def train_with_rf(X_train, X_val, y_train, y_val):
     # Takes 1-2 minutes to complete
 
     # Tạo RandomForestClassifier mặc định để sử dụng làm mô hình ban đầu trong pipeline
-    base_model = RandomForestClassifier(random_state=42, n_jobs=-1)
+    base_model = XGBClassifier(tree_method="hist", device="cuda", random_state=42, n_jobs=-1)
 
     # Tạo pipeline
     pipeline = Pipeline([
@@ -204,7 +300,7 @@ def train_with_rf(X_train, X_val, y_train, y_val):
     param_grid = {
         'classifier__n_estimators': [100, 300, 500, 700, 1000],
         'classifier__max_depth': [6, 8, 10, 15, 20],
-        'classifier__criterion': ['gini', 'entropy'],
+        'classifier__learning_rate': [0.01, 0.1, 0.2],
     }
 
     # Sử dụng GridSearchCV để tìm bộ tham số tốt nhất
@@ -224,15 +320,53 @@ def train_with_rf(X_train, X_val, y_train, y_val):
     return grid_search
 
 
-def save_model(name_file, grid_search):
+def save_model(name_file, model, metadata=None, label_encoder=None):
+    """
+    Save model với metadata để tương thích với ModelManager
+    
+    Args:
+        name_file: Tên file model
+        model: Model object
+        metadata: Dict chứa thông tin về model (optional)
+        label_encoder: Label encoder (optional)
+    """
+    from model_manager import get_model_manager
+    
     dir_save_model = "model_train"
     if not os.path.exists(dir_save_model):
         os.mkdir(dir_save_model)
-    joblib.dump(grid_search, os.path.join(dir_save_model, name_file))
-    print("Done!")
+    
+    # Nếu có metadata, sử dụng ModelManager
+    if metadata is not None:
+        model_manager = get_model_manager()
+        model_manager.save_model(
+            model=model,
+            metadata=metadata,
+            model_filename=name_file,
+            label_encoder=label_encoder
+        )
+    else:
+        # Legacy mode: save trực tiếp (backward compatibility)
+        model_data = {
+            'model': model,
+            'label_encoder': label_encoder
+        } if label_encoder is not None else model
+        
+        joblib.dump(model_data, os.path.join(dir_save_model, name_file))
+    
+    print(f"✅ Model saved: {name_file}")
+    if metadata:
+        print(f"   - Type: {metadata.get('model_type', 'N/A')}")
+        print(f"   - Features: {metadata.get('n_features', 'N/A')}")
+        print(f"   - Accuracy: {metadata.get('test_accuracy', 'N/A')}")
+
     
     
 def predict(model, data_crs, ndvi, vh, vv):
+    # Unpack model if it is wrapped in a dictionary (from ModelManager)
+    if isinstance(model, dict) and 'model' in model:
+        model = model['model']
+        
     data_predict = []
     for i in range(ndvi.shape[1]):
         ndvi_tmp = ndvi.isel(y=i).values
@@ -326,25 +460,59 @@ def save_result(result, HT_MAP):
         # plt.show()
         
 def load_data_sen1(dc, date_range, coordinates):
+    import os, hashlib
     longtitude_range, latitude_range = coordinates
-    data_sen1 = dc.load(
-        product="sentinel1_grd_gamma0_10m",
-        x=longtitude_range,
-        y=latitude_range,
-        time=date_range,
-        measurements=["vv", "vh"],
-        output_crs="EPSG:32648",
-        resolution=(-10,10),
-        dask_chunks={"x":2048, "y":2048},
-        skip_broken_datasets=True,
-        group_by='solar_day'
+    bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
+    
+    cache_dir = "dataset_cache"
+    os.makedirs(cache_dir, exist_ok=True)
+    key_str = f"data_sen1_{date_range}_{bbox}"
+    cache_key_vh = hashlib.md5((key_str + "vh").encode()).hexdigest() + ".nc"
+    cache_key_vv = hashlib.md5((key_str + "vv").encode()).hexdigest() + ".nc"
+    cache_path_vh = os.path.join(cache_dir, cache_key_vh)
+    cache_path_vv = os.path.join(cache_dir, cache_key_vv)
+    
+    if os.path.exists(cache_path_vh) and os.path.exists(cache_path_vv):
+        print(f"✅ Loading cached S1 (coord) data")
+        ds_vh = xr.open_dataset(cache_path_vh, engine='netcdf4')
+        ds_vv = xr.open_dataset(cache_path_vv, engine='netcdf4')
+        var_vh = [v for v in ds_vh.data_vars if v != 'spatial_ref'][0]
+        var_vv = [v for v in ds_vv.data_vars if v != 'spatial_ref'][0]
+        return ds_vh[var_vh], ds_vv[var_vv]
+    
+    import pystac_client
+    import planetary_computer
+    import odc.stac
+    
+    catalog = pystac_client.Client.open(
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
+        modifier=planetary_computer.sign_inplace,
+    )
+    search = catalog.search(
+        collections=["sentinel-1-rtc"],
+        bbox=bbox,
+        datetime=f"{date_range[0]}/{date_range[1]}",
+    )
+    items = list(search.items())
+    
+    data_sen1 = odc.stac.load(
+        items,
+        bands=["vv", "vh"],
+        bbox=bbox,
+        crs="EPSG:32648",
+        resolution=RESOLUTION,
+        chunks={"x": 2048, "y": 2048, "time": 1},
+        groupby="solar_day"
     )
     
-    notebook_utils.heading(notebook_utils.xarray_object_size(data_sen1))
-    display(data_sen1)
+    data_sen1 = data_sen1.compute()
     dsvh = data_sen1.vh
     dsvv = data_sen1.vv
     
+    print(f"💾 Caching S1 (coord) data")
+    dsvh.to_netcdf(cache_path_vh, engine='netcdf4')
+    dsvv.to_netcdf(cache_path_vv, engine='netcdf4')
+    
     return dsvh, dsvv
     
 def calculate_average(data, time_pattern='1M'):
@@ -352,54 +520,66 @@ def calculate_average(data, time_pattern='1M'):
 
 
 def load_data_sen2(dc, date_range, coordinates):
+    import os, hashlib
     longtitude_range, latitude_range = coordinates
-    product = 's2_l2a'
-    query = {
-        'product': product,                     # Product name
-        'x': longtitude_range,    # "x" axis bounds
-        'y': latitude_range,      # "y" axis bounds
-        'time': date_range,           # Any parsable date strings
-    }
-    native_crs = notebook_utils.mostcommon_crs(dc, query)
-    print(f'Most common native CRS: {native_crs}')
+    bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
     
-    # measurements = ['red','green', 'blue', 'nir', 'scl']
-    measurements = ['red', 'nir', 'scl']
+    cache_dir = "dataset_cache"
+    os.makedirs(cache_dir, exist_ok=True)
+    key_str = f"data_sen2_{date_range}_{bbox}"
+    cache_key = hashlib.md5(key_str.encode()).hexdigest() + ".nc"
+    cache_path = os.path.join(cache_dir, cache_key)
     
-    load_params = {
-        'measurements': measurements,                   # Selected measurement or alias names
-        'output_crs': native_crs,                       # Target EPSG code
-        'resolution': (-10, 10),                        # Target resolution
-        'group_by': 'solar_day',                        # Scene grouping
-        'dask_chunks': {'x': 2048, 'y': 2048},          # Dask chunks
-    }
-    data = load_s2l2a_with_offset(
-        dc,
-        query | load_params   # Combine the two dicts that contain our search and load parameters
+    if os.path.exists(cache_path):
+        print(f"✅ Loading cached S2 (coord) data from {cache_path}")
+        return xr.open_dataset(cache_path, engine='netcdf4')
+    
+    import pystac_client
+    import planetary_computer
+    import odc.stac
+    
+    catalog = pystac_client.Client.open(
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
+        modifier=planetary_computer.sign_inplace,
     )
+    search = catalog.search(
+        collections=["sentinel-2-l2a"],
+        bbox=bbox,
+        datetime=f"{date_range[0]}/{date_range[1]}",
+    )
+    items = list(search.items())
+    
+    data = odc.stac.load(
+        items,
+        bands=["red", "nir", "SCL"],
+        bbox=bbox,
+        crs="EPSG:32648",
+        resolution=RESOLUTION,
+        chunks={"x": 2048, "y": 2048, "time": 1},
+        groupby="solar_day"
+    )
+    if "SCL" in data.data_vars:
+        data = data.rename({"SCL": "scl"})
+        
+    data = data.compute()
+    print(f"💾 Caching S2 (coord) data to {cache_path}")
+    data.to_netcdf(cache_path, engine='netcdf4')
+    
     return data
 
 def mask_cloud(data):
-    flag_name = 'scl'
-    flag_desc = masking.describe_variable_flags(data[flag_name])  # Pandas dataframe
-    display(flag_desc.loc['qa'].values[1])
-    # Create a "data quality" Mask layer
-    flags_def = flag_desc.loc['qa'].values[1]
-    good_pixel_flags = [flags_def[str(i)] for i in [2, 4, 5, 6]]  # To pass strings to enum_to_bool()
-
-    # enum_to_bool calculates the pixel-wise "or" of each set of pixels given by good_pixel_flags
-    # 1 = good data
-    # 0 = "bad" data
-    good_pixel_mask = enum_to_bool(data[flag_name], good_pixel_flags)
+    # For Sentinel-2 L2A SCL: 
+    # 2: Dark Area Pixels, 4: Vegetation, 5: Not Vegetated, 6: Water
+    good_pixel_mask = data['scl'].isin([2, 4, 5, 6])
     data_layer_names = [x for x in data.data_vars if x != 'scl']
-    # Apply good pixel mask to blue, green, red and nir.
+    # Apply good pixel mask
     result = data[data_layer_names].where(good_pixel_mask).persist()
     return result
 
 def find_best_model(dataset):
     X_train, X_val, y_train, y_val = dataset
     # Tạo RandomForestClassifier mặc định để sử dụng làm mô hình ban đầu trong pipeline
-    base_model = RandomForestClassifier(random_state=42, n_jobs=-1)
+    base_model = XGBClassifier(tree_method="hist", device="cuda", random_state=42, n_jobs=-1)
 
     # Tạo pipeline
     pipeline = Pipeline([
@@ -411,7 +591,7 @@ def find_best_model(dataset):
     param_grid = {
         'classifier__n_estimators': [100, 300, 500, 700, 1000],
         'classifier__max_depth': [6, 8, 10, 15, 20],
-        'classifier__criterion': ['gini', 'entropy'],
+        'classifier__learning_rate': [0.01, 0.1, 0.2],
     }
 
     # Sử dụng GridSearchCV để tìm bộ tham số tốt nhất

.temp/patch_caching.py

@@ -0,0 +1,231 @@
+import re
+import os
+
+with open('/home/x79/remote-sensing/new_import_ODC.py', 'r', encoding='utf-8') as f:
+    content = f.read()
+
+# 1. load_data
+load_data_replacement = """def load_data(dc, date_range, longtitude_range, latitude_range):
+    import os, hashlib
+    cache_dir = "dataset_cache"
+    os.makedirs(cache_dir, exist_ok=True)
+    key_str = f"s2_{date_range}_{longtitude_range}_{latitude_range}"
+    cache_key = hashlib.md5(key_str.encode()).hexdigest() + ".nc"
+    cache_path = os.path.join(cache_dir, cache_key)
+    
+    if os.path.exists(cache_path):
+        print(f"✅ Loading cached S2 data from {cache_path}")
+        return xr.open_dataset(cache_path)
+
+    product = 's2_l2a'
+    bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
+    
+    import pystac_client
+    import planetary_computer
+    import odc.stac
+    
+    catalog = pystac_client.Client.open(
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
+        modifier=planetary_computer.sign_inplace,
+    )
+    search = catalog.search(
+        collections=["sentinel-2-l2a"],
+        bbox=bbox,
+        datetime=f"{date_range[0]}/{date_range[1]}",
+    )
+    items = list(search.items())
+    
+    data = odc.stac.load(
+        items,
+        bands=["red", "nir", "SCL"],
+        bbox=bbox,
+        crs="EPSG:32648",
+        resolution=RESOLUTION,
+        chunks={"x": 2048, "y": 2048, "time": 1},
+        groupby="solar_day"
+    )
+    if "SCL" in data.data_vars:
+        data = data.rename({"SCL": "scl"})
+        
+    print(f"💾 Caching S2 data to {cache_path}")
+    data = data.compute()
+    data.to_netcdf(cache_path)
+    
+    return data"""
+content = re.sub(r'def load_data\(dc, date_range, longtitude_range, latitude_range\):.*?return data', load_data_replacement, content, flags=re.DOTALL)
+
+
+# 2. load_sen1
+load_sen1_replacement = """def load_sen1(bbox, time_range):
+    import os, hashlib
+    cache_dir = "dataset_cache"
+    os.makedirs(cache_dir, exist_ok=True)
+    key_str = f"s1_vh_vv_{bbox}_{time_range}"
+    cache_key_vh = hashlib.md5((key_str + "vh").encode()).hexdigest() + ".nc"
+    cache_key_vv = hashlib.md5((key_str + "vv").encode()).hexdigest() + ".nc"
+    cache_path_vh = os.path.join(cache_dir, cache_key_vh)
+    cache_path_vv = os.path.join(cache_dir, cache_key_vv)
+    
+    if os.path.exists(cache_path_vh) and os.path.exists(cache_path_vv):
+        print(f"✅ Loading cached S1 data from {cache_path_vh} and {cache_path_vv}")
+        return xr.open_dataarray(cache_path_vh), xr.open_dataarray(cache_path_vv)
+
+    import pystac_client
+    import planetary_computer
+    import odc.stac
+    
+    # Kết nối STAC Client
+    catalog = pystac_client.Client.open(
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
+        modifier=planetary_computer.sign_inplace,
+    )
+
+    # Tìm kiếm Items
+    search = catalog.search(
+        collections=["sentinel-1-rtc"],
+        bbox=bbox,
+        datetime=time_range,
+    )
+    items = list(search.items())
+
+    # Tải dữ liệu thành xarray Dataset
+    ds_s1 = odc.stac.load(
+        items,
+        bands=["vv", "vh"],
+        bbox=bbox,
+        crs="EPSG:32648",
+        resolution=RESOLUTION,
+        chunks={"x": 2048, "y": 2048, "time": 1}
+    )
+
+    # Tính giá trị trung vị theo thời gian
+    ds_median = ds_s1.median(dim="time").compute()
+    vv = ds_median["vv"]
+    vh = ds_median["vh"]
+
+    # Thêm chiều 'band' để giống hệt rioxarray
+    vv = vv.expand_dims(dim="band")
+    vh = vh.expand_dims(dim="band")
+
+    # Phục hồi metadata về toạ độ
+    vv = vv.rio.write_crs("EPSG:32648")
+    vh = vh.rio.write_crs("EPSG:32648")
+
+    print(f"💾 Caching S1 data to {cache_dir}")
+    vh.to_netcdf(cache_path_vh)
+    vv.to_netcdf(cache_path_vv)
+
+    return vh, vv"""
+content = re.sub(r'def load_sen1\(bbox, time_range\):.*?return vh, vv', load_sen1_replacement, content, flags=re.DOTALL)
+
+
+# 3. load_data_sen1
+load_data_sen1_replacement = """def load_data_sen1(dc, date_range, coordinates):
+    import os, hashlib
+    longtitude_range, latitude_range = coordinates
+    bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
+    
+    cache_dir = "dataset_cache"
+    os.makedirs(cache_dir, exist_ok=True)
+    key_str = f"data_sen1_{date_range}_{bbox}"
+    cache_key_vh = hashlib.md5((key_str + "vh").encode()).hexdigest() + ".nc"
+    cache_key_vv = hashlib.md5((key_str + "vv").encode()).hexdigest() + ".nc"
+    cache_path_vh = os.path.join(cache_dir, cache_key_vh)
+    cache_path_vv = os.path.join(cache_dir, cache_key_vv)
+    
+    if os.path.exists(cache_path_vh) and os.path.exists(cache_path_vv):
+        print(f"✅ Loading cached S1 (coord) data")
+        return xr.open_dataarray(cache_path_vh), xr.open_dataarray(cache_path_vv)
+    
+    import pystac_client
+    import planetary_computer
+    import odc.stac
+    
+    catalog = pystac_client.Client.open(
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
+        modifier=planetary_computer.sign_inplace,
+    )
+    search = catalog.search(
+        collections=["sentinel-1-rtc"],
+        bbox=bbox,
+        datetime=f"{date_range[0]}/{date_range[1]}",
+    )
+    items = list(search.items())
+    
+    data_sen1 = odc.stac.load(
+        items,
+        bands=["vv", "vh"],
+        bbox=bbox,
+        crs="EPSG:32648",
+        resolution=RESOLUTION,
+        chunks={"x": 2048, "y": 2048, "time": 1},
+        groupby="solar_day"
+    )
+    
+    data_sen1 = data_sen1.compute()
+    dsvh = data_sen1.vh
+    dsvv = data_sen1.vv
+    
+    print(f"💾 Caching S1 (coord) data")
+    dsvh.to_netcdf(cache_path_vh)
+    dsvv.to_netcdf(cache_path_vv)
+    
+    return dsvh, dsvv"""
+content = re.sub(r'def load_data_sen1\(dc, date_range, coordinates\):.*?return dsvh, dsvv', load_data_sen1_replacement, content, flags=re.DOTALL)
+
+
+# 4. load_data_sen2
+load_data_sen2_replacement = """def load_data_sen2(dc, date_range, coordinates):
+    import os, hashlib
+    longtitude_range, latitude_range = coordinates
+    bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
+    
+    cache_dir = "dataset_cache"
+    os.makedirs(cache_dir, exist_ok=True)
+    key_str = f"data_sen2_{date_range}_{bbox}"
+    cache_key = hashlib.md5(key_str.encode()).hexdigest() + ".nc"
+    cache_path = os.path.join(cache_dir, cache_key)
+    
+    if os.path.exists(cache_path):
+        print(f"✅ Loading cached S2 (coord) data from {cache_path}")
+        return xr.open_dataset(cache_path)
+    
+    import pystac_client
+    import planetary_computer
+    import odc.stac
+    
+    catalog = pystac_client.Client.open(
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
+        modifier=planetary_computer.sign_inplace,
+    )
+    search = catalog.search(
+        collections=["sentinel-2-l2a"],
+        bbox=bbox,
+        datetime=f"{date_range[0]}/{date_range[1]}",
+    )
+    items = list(search.items())
+    
+    data = odc.stac.load(
+        items,
+        bands=["red", "nir", "SCL"],
+        bbox=bbox,
+        crs="EPSG:32648",
+        resolution=RESOLUTION,
+        chunks={"x": 2048, "y": 2048, "time": 1},
+        groupby="solar_day"
+    )
+    if "SCL" in data.data_vars:
+        data = data.rename({"SCL": "scl"})
+        
+    data = data.compute()
+    print(f"💾 Caching S2 (coord) data to {cache_path}")
+    data.to_netcdf(cache_path)
+    
+    return data"""
+content = re.sub(r'def load_data_sen2\(dc, date_range, coordinates\):.*?return data', load_data_sen2_replacement, content, flags=re.DOTALL)
+
+
+with open('/home/x79/remote-sensing/new_import_ODC.py', 'w', encoding='utf-8') as f:
+    f.write(content)
+
+print("Patching successful.")

.temp/patch_load_sen1.py

@@ -0,0 +1,29 @@
+import json
+import glob
+
+def fix_load_sen1(file_path):
+    with open(file_path, 'r', encoding='utf-8') as f:
+        nb = json.load(f)
+        
+    changed = False
+    for cell in nb.get('cells', []):
+        if cell.get('cell_type') == 'code':
+            source = cell.get('source', [])
+            for i, line in enumerate(source):
+                if 'load_sen1(name_vh, name_vv)' in line:
+                    indent = line[:len(line) - len(line.lstrip())]
+                    replacement = (
+                        f"{indent}bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]\n"
+                        f"{indent}time_range = f'{{date_range[0]}}/{{date_range[1]}}'\n"
+                        f"{indent}{line.lstrip().replace('load_sen1(name_vh, name_vv)', 'load_sen1(bbox, time_range)')}"
+                    )
+                    source[i] = replacement
+                    changed = True
+                    
+    if changed:
+        with open(file_path, 'w', encoding='utf-8') as f:
+            json.dump(nb, f, indent=1)
+        print(f"Patched load_sen1 in {file_path}")
+
+for nb in glob.glob("*.ipynb"):
+    fix_load_sen1(nb)

.temp/patch_notebooks.py

@@ -0,0 +1,43 @@
+import json
+import glob
+
+def patch_notebook(file_path):
+    with open(file_path, 'r', encoding='utf-8') as f:
+        nb = json.load(f)
+        
+    changed = False
+    for cell in nb.get('cells', []):
+        if cell.get('cell_type') == 'code':
+            source = cell.get('source', [])
+            
+            # Check if this cell should be fully commented out
+            full_source = ''.join(source)
+            if 'dc.load(' in full_source or 'ds.vv' in full_source:
+                for i in range(len(source)):
+                    if not source[i].startswith('#'):
+                        source[i] = '# ' + source[i]
+                changed = True
+                continue
+            
+            # Otherwise, do line-by-line replacements
+            for i, line in enumerate(source):
+                if 'ST_training data_updated_1130points.shp' in line:
+                    source[i] = line.replace('ST_training data_updated_1130points.shp', 'ST_training_data_updated_1130points.shp')
+                    changed = True
+                if 'from new_import import *' in line:
+                    source[i] = line.replace('from new_import import *', 'from new_import_ODC import *')
+                    changed = True
+                if 'dc = datacube.Datacube()' in line:
+                    source[i] = line.replace('dc = datacube.Datacube()', 'dc = None')
+                    changed = True
+                if 'load_data(dc,' in line:
+                    source[i] = line.replace('load_data(dc,', 'load_data(None,')
+                    changed = True
+                    
+    if changed:
+        with open(file_path, 'w', encoding='utf-8') as f:
+            json.dump(nb, f, indent=1)
+        print(f"Patched {file_path}")
+
+for nb in glob.glob("*.ipynb"):
+    patch_notebook(nb)

.temp/patch_rf_to_xgb.py

@@ -0,0 +1,75 @@
+import json
+import glob
+import re
+
+def patch_python_script(filepath):
+    try:
+        with open(filepath, 'r', encoding='utf-8') as f:
+            content = f.read()
+            
+        original_content = content
+        
+        # Replace imports
+        content = re.sub(r'from sklearn\.ensemble import RandomForestClassifier', 
+                         'from xgboost import XGBClassifier', content)
+        
+        # Replace the model instantiations (for 01.train_ODC.py)
+        rf_pattern = re.compile(r'model\s*=\s*RandomForestClassifier\([^)]+\)', re.DOTALL)
+        xgb_replacement = """model = XGBClassifier(
+                    n_estimators=200,
+                    max_depth=30,
+                    tree_method="hist",
+                    device="cuda",
+                    random_state=42,
+                    n_jobs=-1,
+                    verbosity=1
+                )"""
+        
+        content = rf_pattern.sub(xgb_replacement, content)
+        
+        if content != original_content:
+            with open(filepath, 'w', encoding='utf-8') as f:
+                f.write(content)
+            print(f"Patched {filepath}")
+    except Exception as e:
+        print(f"Error patching {filepath}: {e}")
+
+def patch_notebook(filepath):
+    try:
+        with open(filepath, 'r', encoding='utf-8') as f:
+            nb = json.load(f)
+            
+        changed = False
+        import_pattern = re.compile(r'from\s+sklearn\.ensemble\s+import\s+RandomForestClassifier')
+        inst_pattern = re.compile(r'RandomForestClassifier\([^)]*\)')
+        
+        for cell in nb.get('cells', []):
+            if cell.get('cell_type') == 'code':
+                source = cell.get('source', [])
+                for i in range(len(source)):
+                    if import_pattern.search(source[i]):
+                        source[i] = import_pattern.sub('from xgboost import XGBClassifier', source[i])
+                        changed = True
+                        
+                    if inst_pattern.search(source[i]):
+                        source[i] = inst_pattern.sub("XGBClassifier(tree_method='hist', device='cuda', random_state=42, n_jobs=-1)", source[i])
+                        changed = True
+                        
+                    if "'classifier__criterion': ['gini', 'entropy']" in source[i]:
+                        source[i] = source[i].replace("'classifier__criterion': ['gini', 'entropy']",
+                                                      "'classifier__learning_rate': [0.01, 0.1, 0.2]")
+                        changed = True
+                        
+        if changed:
+            with open(filepath, 'w', encoding='utf-8') as f:
+                json.dump(nb, f, indent=1)
+            print(f"Patched {filepath}")
+    except Exception as e:
+        print(f"Error patching {filepath}: {e}")
+
+if __name__ == "__main__":
+    patch_python_script("01.train_ODC.py")
+    patch_python_script("new_train.py")
+    
+    for nb in glob.glob("*.ipynb"):
+        patch_notebook(nb)

.temp/processMask-test.py

@@ -0,0 +1,272 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+get_ipython().run_cell_magic('time', '', '%matplotlib inline\nfrom new_import import *\n')
+
+
+# In[2]:
+
+
+get_ipython().run_cell_magic('time', '', '# Dask gateway\ncluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))\ndc = datacube.Datacube()\n\n# Configure s3 access\nconfigure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n\nclient\n')
+
+
+# In[3]:
+
+
+## cấu hình thời gian lấy ảnh và tọa độ
+# date_range = ('2022-09-01', '2023-10-01')
+# longtitude_range = (105.86575, 105.94120)
+# latitude_range = (9.65070, 9.69850)
+date_range = ('2022-09-01', '2023-10-01')
+longtitude_range = (105.5, 106.4)
+latitude_range = (9.2, 10.0) 
+
+
+# In[4]:
+
+
+## truy vấn ảnh vệ tinh sen2
+data = load_data(dc, date_range, longtitude_range, latitude_range)
+notebook_utils.heading(notebook_utils.xarray_object_size(data))
+display(data)
+
+
+# In[5]:
+
+
+# Specify the start and end times 
+min_date = '2022-09-01' # Thời gian bắt đầu lấy data cho quá trình train
+max_date = '2023-10-01' # Thời gian kết thúc lấy data cho quá trình train
+# Just do 1 month for testing
+# max_date = '2022-10-01' # Thời gian kết thúc lấy data cho quá trình train
+
+# Specify a spatail region to search using latitude/longitude cooridinates
+min_longitude, max_longitude = (105.5, 106.4)
+min_latitude, max_latitude = (9.2, 10.0)
+
+# Specify the product. In this case we want to use Sentinel-2 Level-2A data
+product = 's2_l2a'
+
+# Construct the search query dictionary
+query = {
+    'product': product,                     # Product name
+    'x': (min_longitude, max_longitude),    # "x" axis bounds
+    'y': (min_latitude, max_latitude),      # "y" axis bounds
+    'time': (min_date, max_date),           # Any parsable date strings
+}
+
+
+# In[6]:
+
+
+# Most common CRS
+native_crs = notebook_utils.mostcommon_crs(dc, query)
+print(f'Most common native CRS: {native_crs}')
+
+
+# In[7]:
+
+
+# Specify the spectral band measurements we want to use for a classification algorithm
+measurements = ['red', 'nir', 'scl']
+
+load_params = {
+    'measurements': measurements,                   # Selected measurement or alias names
+    'output_crs': native_crs,                       # Target EPSG code
+    'resolution': (-10, 10),                        # Target resolution
+    'group_by': 'solar_day',                        # Scene grouping
+    'dask_chunks': {'x': 2048, 'y': 2048},          # Dask chunks
+}
+
+
+# In[8]:
+
+
+get_ipython().run_cell_magic('time', '', '# The replacement "dc.load()" function for this product\ndata = load_s2l2a_with_offset(\n    dc,\n    query | load_params   # Combine the two dicts that contain our search and load parameters\n)\n\n# This line prints the total size of the dataset hat was loaded\nnotebook_utils.heading(notebook_utils.xarray_object_size(data))\n\ndisplay(data)\n')
+
+
+# In[9]:
+
+
+# %%time
+# # Tiến hành loại bỏ các vị trí bị mây ảnh hưởng
+# result = mask_clean(data)
+# progress(result)
+
+
+# In[10]:
+
+
+# Tiến hành tính toán NDVI
+ds1 = calculate_indices(result, index='NDVI', satellite_mission='s2')
+ndvi = ds1["NDVI"]
+display(ndvi)
+
+
+# In[11]:
+
+
+get_ipython().run_cell_magic('time', '', "## tính ndvi theo tháng\naverage_ndvi = ndvi.resample(time='1M').mean().persist()\nprogress(average_ndvi)\n")
+
+
+# In[12]:
+
+
+# compute average_ndvi
+average_ndvi = average_ndvi.compute()
+
+
+# In[13]:
+
+
+# cấu hình vh vv file
+# name_vh = "ThuanHoa/ThuanHoa_VH.tif"
+# name_vv = "ThuanHoa/ThuanHoa_VV.tif"
+
+# load dữ liệu sen1
+bbox = [105.5, 9.2, 106.4, 10.0]
+time_range = '2022-09-01/2023-10-01'
+# dsvh, dsvv = load_sen1(bbox, time_range)
+
+name_vh = "vh-0922_0923-full_ST.tif"
+name_vv = "vv-0922_0923-full_ST.tif"
+
+if not os.path.exists(name_vh):
+    get_ipython().system('aws s3 cp s3://easi-asia-dc-data/staging/ctu/sentinel-1/vh-0922_0923-full_ST.tif vh-0922_0923-full_ST.tif')
+if not os.path.exists(name_vv):
+    get_ipython().system('aws s3 cp s3://easi-asia-dc-data/staging/ctu/sentinel-1/vv-0922_0923-full_ST.tif vv-0922_0923-full_ST.tif')
+    
+bbox = [105.5, 9.2, 106.4, 10.0]
+time_range = '2022-09-01/2023-10-01'
+dsvh, dsvv = load_sen1(bbox, time_range)
+
+
+# In[27]:
+
+
+from sklearn.preprocessing import PolynomialFeatures
+from sklearn.linear_model import LinearRegression
+from sklearn.ensemble import RandomForestRegressor
+
+
+# In[28]:
+
+
+average_ndvi = average_ndvi[:, :7680, :8687]
+mask = ~np.isnan(average_ndvi)
+print(average_ndvi.shape)
+print(dsvh.shape)
+print(dsvv.shape)
+print(mask.shape)
+X_train = np.stack([dsvh.values[mask], dsvv.values[mask]], axis=1)
+y_train = average_ndvi.values[mask]
+
+
+# In[29]:
+
+
+model = LinearRegression()
+model.fit(X_train, y_train)
+
+
+# In[30]:
+
+
+X_pred = np.stack([dsvh.values[~mask], dsvv.values[~mask]], axis=1)
+average_ndvi.values[~mask] = model.predict(X_pred)
+
+
+# In[31]:
+
+
+average_ndvi_filled = xr.DataArray(average_ndvi, dims=average_ndvi.dims)
+
+
+# In[32]:
+
+
+plt.imshow(average_ndvi_filled.isel(time=6))
+
+
+# In[65]:
+
+
+plt.imshow(average_ndvi.isel(time=6))
+
+
+# In[33]:
+
+
+train_path = "train/ST_training data_updated_1130points.shp"
+
+
+# In[34]:
+
+
+train = load_train_data(train_path)
+
+
+# In[37]:
+
+
+datasets = get_data_sen1_and_sen2(train, average_ndvi_filled, dsvh, dsvv)
+
+
+# In[39]:
+
+
+# cấu hình nhãn dữ liệu
+label_mapping = {
+    "Lua tom": "0",
+    "Lua": "1",
+    "CHN": "2",
+    "CLN": "3",
+    "TS": "4",
+    "Song": "5",
+    "Dat xay dung": "6",
+    "Rung": "7"
+}
+
+# chia tập dữ liệu train, val, test
+X_train, X_val, X_test, y_train, y_val, y_test = split_train_data(train, label_mapping, datasets)
+
+
+# In[40]:
+
+
+# Huấn luyện mô hình
+grid_search = train_with_rf(X_train, X_val, y_train, y_val)
+
+
+# In[41]:
+
+
+# kiểm tra độ chính xác với tập test
+y_pred_test = grid_search.predict(X_test)
+test_accuracy = accuracy_score(y_test, y_pred_test)
+print(f"Accuracy for test data {round(test_accuracy, 2)*100} %")
+
+
+# In[42]:
+
+
+# Lưu mô hình huấn luyện
+save_model("model_new.joblib", grid_search)
+
+
+# In[43]:
+
+
+# đóng client, cluster
+client.close()
+cluster.close()
+
+
+# In[ ]:
+
+
+
+

.temp/processMask.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+get_ipython().run_cell_magic('time', '', '%matplotlib inline\nfrom new_import import *\n')
+
+
+# In[2]:
+
+
+get_ipython().run_cell_magic('time', '', '# Dask gateway\ncluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))\ndc = datacube.Datacube()\n\n# Configure s3 access\nconfigure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n\nclient\n')
+
+
+# In[3]:
+
+
+## cấu hình thời gian lấy ảnh và tọa độ
+date_range = ('2022-09-01', '2023-10-01')
+longtitude_range = (105.86575, 105.94120)
+latitude_range = (9.65070, 9.69850)
+
+
+# In[4]:
+
+
+## truy vấn ảnh vệ tinh sen2
+data = load_data(dc, date_range, longtitude_range, latitude_range)
+notebook_utils.heading(notebook_utils.xarray_object_size(data))
+display(data)
+
+
+# In[5]:
+
+
+get_ipython().run_cell_magic('time', '', '# Tiến hành loại bỏ các vị trí bị mây ảnh hưởng\nresult = mask_clean(data)\nprogress(result)\n')
+
+
+# In[6]:
+
+
+# Tiến hành tính toán NDVI
+ds1 = calculate_indices(result, index='NDVI', satellite_mission='s2')
+ndvi = ds1["NDVI"]
+display(ndvi)
+
+
+# In[17]:
+
+
+get_ipython().run_cell_magic('time', '', "## tính ndvi theo tháng\naverage_ndvi = ndvi.resample(time='1M').mean().persist()\nprogress(average_ndvi)\n")
+
+
+# In[18]:
+
+
+# compute average_ndvi
+average_ndvi = average_ndvi.compute()
+
+
+# In[9]:
+
+
+# cấu hình vh vv file
+name_vh = "ThuanHoa/ThuanHoa_VH.tif"
+name_vv = "ThuanHoa/ThuanHoa_VV.tif"
+
+# load dữ liệu sen1
+bbox = [105.5, 9.2, 106.4, 10.0]
+time_range = '2022-09-01/2023-10-01'
+dsvh, dsvv = load_sen1(bbox, time_range)
+
+
+# In[10]:
+
+
+from sklearn.preprocessing import PolynomialFeatures
+from sklearn.linear_model import LinearRegression
+from sklearn.ensemble import RandomForestRegressor
+
+
+# In[11]:
+
+
+mask = ~np.isnan(average_ndvi)
+X_train = np.stack([dsvh.values[mask], dsvv.values[mask]], axis=1)
+y_train = average_ndvi.values[mask]
+
+
+# In[12]:
+
+
+model = LinearRegression()
+model.fit(X_train, y_train)
+
+
+# In[13]:
+
+
+X_pred = np.stack([dsvh.values[~mask], dsvv.values[~mask]], axis=1)
+average_ndvi.values[~mask] = model.predict(X_pred)
+
+
+# In[14]:
+
+
+average_ndvi_filled = xr.DataArray(average_ndvi, dims=average_ndvi.dims)
+
+
+# In[16]:
+
+
+plt.imshow(average_ndvi_filled.isel(time=1))
+
+
+# In[19]:
+
+
+plt.imshow(average_ndvi.isel(time=1))
+
+
+# In[ ]:
+
+
+
+

.temp/rebuild_notebook.py

@@ -0,0 +1,145 @@
+import json
+
+def get_content(filename):
+    with open(filename, "r", encoding="utf-8") as f:
+        return f.read()
+
+fe_content = get_content("feature_extractor.py")
+tm_content = get_content("train_module.py")
+dt_content = get_content("train_land_decision_tree_gpu.py")
+
+notebook = {
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Tải Dữ liệu Vệ tinh qua Colab (Self-contained)\n",
+    "Notebook này đã được nhúng sẵn toàn bộ mã nguồn xử lý. Bạn không cần upload cả thư mục `remote-sensing` nữa.\n",
+    "\n",
+    "## Bước 1: Upload Shapefile (BẮT BUỘC)\n",
+    "Mô hình cần biết các điểm tọa độ đất để lấy dữ liệu. Hãy nén thư mục `train/` trên máy bạn thành `train.zip` và chạy ô dưới đây để upload nó trực tiếp lên Colab."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": None,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from google.colab import files\n",
+    "import os\n",
+    "\n",
+    "print(\"Hãy chọn file train.zip từ máy tính của bạn:\")\n",
+    "uploaded = files.upload()\n",
+    "\n",
+    "if \"train.zip\" in uploaded:\n",
+    "    !unzip -q -o train.zip -d /content/train_tmp/\n",
+    "    # Move the extracted files directly to /content/train/\n",
+    "    !mkdir -p /content/train\n",
+    "    !mv /content/train_tmp/*/* /content/train/ 2>/dev/null || mv /content/train_tmp/* /content/train/\n",
+    "    print(\"Đã giải nén shapefile thành công vào thư mục /content/train/\")\n",
+    "else:\n",
+    "    print(\"LỖI: Bạn chưa upload file có tên là train.zip!\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Bước 2: Cài đặt thư viện & Tạo môi trường"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": None,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!pip install planetary-computer pystac-client odc-stac geopandas rasterio xarray joblib scikit-learn xgboost lightgbm"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": None,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%writefile feature_extractor.py\n" + fe_content
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": None,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%writefile train_module.py\n" + tm_content
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": None,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%writefile train_land_decision_tree_gpu.py\n" + dt_content
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Bước 3: Chạy tiến trình tải ảnh vệ tinh và tạo Cache"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": None,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!python train_land_decision_tree_gpu.py"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Bước 4: Tải file Cache về máy"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": None,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from google.colab import files\n",
+    "import glob\n",
+    "\n",
+    "cache_files = glob.glob(\"dataset_cache/*.joblib\")\n",
+    "if cache_files:\n",
+    "    latest_cache = max(cache_files, key=os.path.getctime)\n",
+    "    print(f\"Đang tải file {latest_cache} về máy...\")\n",
+    "    files.download(latest_cache)\n",
+    "else:\n",
+    "    print(\"Chưa tìm thấy file cache. Hãy chắc chắn bước 3 đã chạy thành công!\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
+
+with open("Download_Cache_Colab.ipynb", "w", encoding="utf-8") as f:
+    json.dump(notebook, f, indent=1, ensure_ascii=False)
+
+print("Notebook updated successfully!")

.temp/remove_datacube.py

@@ -0,0 +1,35 @@
+import json
+import glob
+
+def fix_notebook(file_path):
+    with open(file_path, 'r', encoding='utf-8') as f:
+        nb = json.load(f)
+    changed = False
+    for cell in nb.get('cells', []):
+        if cell.get('cell_type') == 'code':
+            source = cell.get('source', [])
+            if isinstance(source, list):
+                for i, line in enumerate(source):
+                    if "dc = datacube.Datacube()" in line:
+                        source[i] = "dc = None\n"
+                        changed = True
+                    if "ds = dc.load(" in line:
+                        source[i] = "ds = None\n"
+                        changed = True
+                    if "data = dc.load(" in line:
+                        source[i] = "data = None\n"
+                        changed = True
+                    # If ds is None, ds.vv will fail
+                    if "vv_data = ds.vv" in line:
+                        source[i] = "vv_data = None\n"
+                        changed = True
+                    if "notebook_utils.xarray_object_size(ds)" in line:
+                        source[i] = line.replace("notebook_utils.xarray_object_size(ds)", "'ds is None'")
+                        changed = True
+    if changed:
+        with open(file_path, 'w', encoding='utf-8') as f:
+            json.dump(nb, f, indent=1)
+        print(f"Removed datacube from {file_path}")
+
+for nb in glob.glob("*.ipynb"):
+    fix_notebook(nb)

.temp/run_mock_parallel_and_compare.py

@@ -0,0 +1,110 @@
+import os
+import glob
+import json
+import subprocess
+import time
+from tabulate import tabulate
+
+scripts = [
+    "train_land_randomforest.py",
+    "train_cloud_cnn.py",
+    "train_cloud_swin_unet.py",
+    "train_ndvi_statistical.py",
+    "train_ndvi_lstm_gru.py",
+    "train_ndvi_convlstm.py",
+    "train_ndvi_hybrid_physics.py",
+    "train_ndvi_ensemble.py"
+]
+
+print("🚀 Đang khởi chạy song song tất cả các mô hình...")
+processes = []
+for script in scripts:
+    if os.path.exists(script):
+        cmd = f"source /home/x79/miniconda3/etc/profile.d/conda.sh && conda activate env_01 && python {script}"
+        p = subprocess.Popen(["bash", "-c", cmd], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        processes.append((script, p))
+
+for script, p in processes:
+    p.wait()
+
+print("✅ Đã chạy xong tất cả các mô hình!\n")
+print("📊 BẢNG SO SÁNH KẾT QUẢ CÁC MÔ HÌNH\n")
+
+# 1. Phân loại đất
+print("### 1. Nhóm Phân loại Lớp phủ (Land Classification)")
+land_data = []
+
+# Đọc XGBoost từ thư mục gốc
+if os.path.exists("model_xgboost_info.json"):
+    with open("model_xgboost_info.json", 'r') as f:
+        data = json.load(f)
+        params = data.get('params', {})
+        param_str = f"estimators:{params.get('n_estimators')}, depth:{params.get('max_depth')}" if params else "N/A"
+        land_data.append([
+            data.get('model_type', 'XGBoost'),
+            data.get('accuracy', ''),
+            data.get('precision', ''),
+            data.get('recall', ''),
+            data.get('f1_score', ''),
+            param_str
+        ])
+
+# Đọc các model khác trong model_train
+for info_file in glob.glob("model_train/*_info.json"):
+    with open(info_file, 'r') as f:
+        data = json.load(f)
+        # Chỉ lấy các model có độ chính xác (để lọc model rác/cũ)
+        if 'accuracy' not in data and 'f1_score' not in data:
+            continue
+            
+        params = data.get('params', {})
+        param_str = f"estimators:{params.get('n_estimators')}, depth:{params.get('max_depth')}" if params else "N/A"
+        
+        # Fallback for Random Forest
+        if data.get('model_type') == 'RandomForest_RealData':
+            param_str = "estimators:100, depth:15"
+            
+        land_data.append([
+            data.get('model_type', ''),
+            data.get('accuracy', ''),
+            data.get('precision', ''),
+            data.get('recall', ''),
+            data.get('f1_score', ''),
+            param_str
+        ])
+
+if land_data:
+    print(tabulate(land_data, headers=["Model", "Accuracy", "Precision", "Recall", "F1-Score", "Parameters"], tablefmt="github"))
+print("\n")
+
+# 2. Xóa mây
+print("### 2. Nhóm Xóa mây (Cloud Removal)")
+cloud_data = []
+for info_file in glob.glob("cloud_removal_model/*_info.json"):
+    with open(info_file, 'r') as f:
+        data = json.load(f)
+        cloud_data.append([
+            data.get('model_type', ''),
+            data.get('epoch', ''),
+            data.get('train_loss', ''),
+            data.get('val_loss', '')
+        ])
+if cloud_data:
+    print(tabulate(cloud_data, headers=["Model", "Epochs", "Train Loss", "Val Loss"], tablefmt="github"))
+print("\n")
+
+# 3. Dự báo NDVI
+print("### 3. Nhóm Dự báo Thực vật (NDVI Forecasting)")
+ndvi_data = []
+for info_file in glob.glob("ndvi_forecast_model/*_info.json"):
+    with open(info_file, 'r') as f:
+        data = json.load(f)
+        ndvi_data.append([
+            data.get('model_type', ''),
+            data.get('rmse', ''),
+            data.get('mae', ''),
+            data.get('epoch', 'N/A')
+        ])
+if ndvi_data:
+    print(tabulate(ndvi_data, headers=["Model", "RMSE", "MAE", "Epochs"], tablefmt="github"))
+print("\n")

.temp/test_VH_VV.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[2]:
+
+
+get_ipython().run_line_magic('matplotlib', 'inline')
+from new_import import *
+
+
+# Dask gateway
+cluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))
+dc = datacube.Datacube()
+
+
+# Configure s3 access
+configure_s3_access(aws_unsigned=False, requester_pays=True, client=client)
+
+
+
+
+# In[3]:
+
+
+ds = dc.load(
+    product="sentinel1_grd_gamma0_20m",
+    x=(105.5, 106.4),
+    y=(9.2, 10.0),
+    time=("2022-09-01", "2023-10-01"),
+    measurements=["vv", "vh"],
+    output_crs="EPSG:32648",
+    resolution=(-10,10),
+    dask_chunks={"x":2048, "y":2048},
+    skip_broken_datasets=True,
+    group_by="solar_day"
+)
+notebook_utils.heading(notebook_utils.xarray_object_size(ds))
+ds
+
+
+# In[18]:
+
+
+vh = ds.vh.resample(time='1M').mean().persist()
+vh = vh.compute()
+vv = ds.vv.resample(time='1M').mean().persist()
+vv = vv.compute()
+
+
+
+# In[28]:
+
+
+vv.min()
+
+
+# In[33]:
+
+
+import matplotlib.pyplot as plt
+
+# Plot the data
+plt.imshow(vh.isel(time=0), cmap='viridis', vmin=0, vmax=1)
+plt.colorbar()  # Add colorbar for reference
+plt.show()
+
+
+# In[ ]:
+
+
+
+

.temp/test_bug.py

@@ -0,0 +1,33 @@
+import torch
+import numpy as np
+
+device = "cpu"
+input_array = np.zeros((4, 16, 16), dtype=np.float32)
+input_tensor = torch.from_numpy(input_array).unsqueeze(0).to(device)
+
+print("Before pad:", input_tensor.shape)
+
+from train_cloud_removal import UNet
+model = UNet(in_channels=6, out_channels=4).to(device)
+
+if hasattr(model, 'inc') and hasattr(model.inc.double_conv[0], 'in_channels'):
+    expected_channels = model.inc.double_conv[0].in_channels
+elif hasattr(model, 'conv1') and hasattr(model.conv1, 'in_channels'):
+    expected_channels = model.conv1.in_channels
+else:
+    expected_channels = list(model.parameters())[0].shape[1]
+
+print("Expected channels:", expected_channels)
+
+if expected_channels > input_tensor.shape[1]:
+    pad_channels = expected_channels - input_tensor.shape[1]
+    padding = torch.zeros(1, pad_channels, *input_tensor.shape[2:]).to(device)
+    input_tensor = torch.cat([input_tensor, padding], dim=1)
+
+print("After pad:", input_tensor.shape)
+
+try:
+    model(input_tensor)
+    print("Success!")
+except Exception as e:
+    print("Error:", e)

.temp/test_caching.py

@@ -0,0 +1,25 @@
+import os
+import sys
+sys.path.insert(0, os.getcwd())
+import new_import_ODC
+import time
+import xarray as xr
+
+# Mock minimal params to test load_data
+date_range = ('2023-01-01', '2023-01-31')
+longtitude_range = (105.0, 105.1)
+latitude_range = (9.5, 9.6)
+
+print("--- First Call (Downloading & Caching) ---")
+start = time.time()
+data1 = new_import_ODC.load_data(None, date_range, longtitude_range, latitude_range)
+end = time.time()
+print(f"Time taken: {end - start:.2f}s")
+
+print("--- Second Call (Loading from Cache) ---")
+start = time.time()
+data2 = new_import_ODC.load_data(None, date_range, longtitude_range, latitude_range)
+end = time.time()
+print(f"Time taken: {end - start:.2f}s")
+
+print("✅ Test completed")

.temp/test_checkpoint.py

@@ -0,0 +1,7 @@
+import torch
+checkpoint = torch.load('cloud_removal_model/cloud_removal_unet_best.pth', map_location='cpu')
+print(checkpoint.keys())
+print("in_channels in checkpoint:", 'in_channels' in checkpoint)
+if 'in_channels' in checkpoint:
+    print(checkpoint['in_channels'])
+print("Shape of inc.double_conv.0.weight:", checkpoint['model_state_dict']['inc.double_conv.0.weight'].shape)

.temp/test_cloud.py

@@ -0,0 +1,4 @@
+import torch
+from cloud_removal import DeepInpaintingStrategy
+cloud_remover = DeepInpaintingStrategy()
+print("Model channels:", list(cloud_remover.model.parameters())[0].shape[1])

.temp/test_cloud2.py

@@ -0,0 +1,10 @@
+from cloud_removal import DeepInpaintingStrategy
+import torch
+import numpy as np
+
+cr = DeepInpaintingStrategy(model_path="cloud_removal_model/cloud_removal_unet_best.pth")
+if cr.model is not None:
+    expected = list(cr.model.parameters())[0].shape[1]
+    print("Expected channels:", expected)
+else:
+    print("Failed to load model")

.temp/test_cloud_removal.py

@@ -0,0 +1,194 @@
+"""
+Test script for cloud_removal module
+Kiểm tra các phương pháp xử lý mây
+"""
+
+import numpy as np
+import xarray as xr
+from cloud_removal import (
+    process_cloud_removal,
+    get_available_methods,
+    compare_methods
+)
+
+
+def create_mock_s2_data():
+    """Tạo mock Sentinel-2 data để test"""
+    # Create synthetic data: 5 time steps, 100x100 pixels
+    np.random.seed(42)
+    
+    time_steps = 5
+    y_size = 100
+    x_size = 100
+    
+    # Create bands
+    bands = {}
+    for band in ["B02", "B03", "B04", "B08", "B11"]:
+        # Random reflectance values
+        data = np.random.rand(time_steps, y_size, x_size) * 0.3 + 0.1
+        bands[band] = (["time", "y", "x"], data)
+    
+    # Create SCL (Scene Classification Layer)
+    # Mostly vegetation (4), with some clouds
+    scl_data = np.full((time_steps, y_size, x_size), 4, dtype=np.uint8)
+    
+    # Add clouds (class 9) in random locations
+    for t in range(time_steps):
+        # Random cloud patches
+        n_clouds = np.random.randint(5, 15)
+        for _ in range(n_clouds):
+            y_start = np.random.randint(0, y_size - 20)
+            x_start = np.random.randint(0, x_size - 20)
+            cloud_height = np.random.randint(10, 20)
+            cloud_width = np.random.randint(10, 20)
+            scl_data[t, y_start:y_start+cloud_height, x_start:x_start+cloud_width] = 9
+    
+    bands["SCL"] = (["time", "y", "x"], scl_data)
+    
+    # Create xarray Dataset
+    ds = xr.Dataset(
+        bands,
+        coords={
+            "time": np.arange(time_steps),
+            "y": np.arange(y_size),
+            "x": np.arange(x_size)
+        }
+    )
+    
+    return ds
+
+
+def test_available_methods():
+    """Test lấy danh sách methods"""
+    print("=" * 60)
+    print("TEST: Get Available Methods")
+    print("=" * 60)
+    
+    methods = get_available_methods()
+    print(f"\nFound {len(methods)} methods:")
+    for method, description in methods.items():
+        print(f"  - {method:20s}: {description}")
+    
+    print("\n✅ Test passed!")
+
+
+def test_single_method(method_name="classic"):
+    """Test một method cụ thể"""
+    print("\n" + "=" * 60)
+    print(f"TEST: Cloud Removal Method '{method_name}'")
+    print("=" * 60)
+    
+    # Create mock data
+    s2_data = create_mock_s2_data()
+    print(f"\nMock data created: {dict(s2_data.dims)}")
+    
+    # Process clouds
+    cleaned_data, metadata = process_cloud_removal(
+        s2_data=s2_data,
+        method=method_name,
+        verbose=True
+    )
+    
+    # Check results
+    print(f"\nMetadata:")
+    print(f"  - Method: {metadata['method']}")
+    print(f"  - Cloud coverage: {metadata['cloud_coverage_percent']:.1f}%")
+    print(f"  - Masked pixels: {metadata['masked_pixels']:,}/{metadata['total_pixels']:,}")
+    print(f"  - Steps applied: {', '.join(metadata['steps_applied'])}")
+    
+    # Verify no NaN remaining
+    nan_count = 0
+    for band in cleaned_data.data_vars:
+        if band != "SCL":
+            nan_count += np.isnan(cleaned_data[band].values).sum()
+    
+    print(f"\nRemaining NaN pixels: {nan_count}")
+    
+    if nan_count == 0:
+        print("✅ Test passed - no NaN remaining!")
+    else:
+        print(f"⚠️  Warning - {nan_count} NaN pixels remaining")
+
+
+def test_comparison():
+    """Test so sánh nhiều methods"""
+    print("\n" + "=" * 60)
+    print("TEST: Compare Multiple Methods")
+    print("=" * 60)
+    
+    # Create mock data
+    s2_data = create_mock_s2_data()
+    
+    # Compare methods
+    methods_to_test = ["classic", "temporal_only", "median_composite", "ml_knn"]
+    
+    print(f"\nComparing {len(methods_to_test)} methods...")
+    results = compare_methods(s2_data, methods=methods_to_test)
+    
+    # Print summary
+    print("\n" + "-" * 60)
+    print(f"{'Method':<20} {'Success':<10} {'NaN %':<10} {'Steps'}")
+    print("-" * 60)
+    
+    for method, result in results.items():
+        if result['success']:
+            nan_pct = result['remaining_nan_percent']
+            steps = ', '.join(result['metadata']['steps_applied'][:2])  # First 2 steps
+            print(f"{method:<20} {'✅':<10} {nan_pct:>6.2f}%   {steps}")
+        else:
+            print(f"{method:<20} {'❌':<10} {'ERROR':<10} {result['error']}")
+    
+    print("-" * 60)
+    print("\n✅ Comparison test completed!")
+
+
+def test_edge_cases():
+    """Test các trường hợp đặc biệt"""
+    print("\n" + "=" * 60)
+    print("TEST: Edge Cases")
+    print("=" * 60)
+    
+    # Case 1: No SCL band
+    print("\n1. Testing without SCL band...")
+    s2_data = create_mock_s2_data()
+    s2_data_no_scl = s2_data.drop_vars("SCL")
+    
+    cleaned, meta = process_cloud_removal(s2_data_no_scl, method="classic", verbose=False)
+    print(f"   Result: {meta.get('warning', 'OK')}")
+    
+    # Case 2: 100% cloud coverage
+    print("\n2. Testing with 100% cloud coverage...")
+    s2_data_full_cloud = create_mock_s2_data()
+    s2_data_full_cloud["SCL"][:] = 9  # All clouds
+    
+    cleaned, meta = process_cloud_removal(s2_data_full_cloud, method="classic", verbose=False)
+    print(f"   Cloud coverage: {meta['cloud_coverage_percent']:.1f}%")
+    
+    # Case 3: No clouds
+    print("\n3. Testing with no clouds...")
+    s2_data_clear = create_mock_s2_data()
+    s2_data_clear["SCL"][:] = 4  # All vegetation
+    
+    cleaned, meta = process_cloud_removal(s2_data_clear, method="classic", verbose=False)
+    print(f"   Cloud coverage: {meta['cloud_coverage_percent']:.1f}%")
+    
+    print("\n✅ Edge case tests passed!")
+
+
+if __name__ == "__main__":
+    print("\n" + "🌥️  CLOUD REMOVAL MODULE TESTS 🌥️ ".center(60, "="))
+    print()
+    
+    # Run tests
+    test_available_methods()
+    test_single_method("classic")
+    test_single_method("hybrid")
+    test_comparison()
+    test_edge_cases()
+    
+    print("\n" + "=" * 60)
+    print("ALL TESTS COMPLETED!")
+    print("=" * 60)
+    print("\nModule is ready to use. Available methods:")
+    for method, desc in get_available_methods().items():
+        print(f"  • {method}")

.temp/test_cloud_training.py

@@ -0,0 +1,28 @@
+"""
+Script test nhanh cho cloud removal training
+"""
+
+import sys
+from pathlib import Path
+
+# Add winter_dataset to path
+sys.path.insert(0, str(Path(__file__).parent / "winter_dataset"))
+
+from train_cloud_removal import train_cloud_removal_model
+
+if __name__ == "__main__":
+    print("\n🌥️  Starting Cloud Removal Training Test")
+    print("=" * 70)
+    
+    # Test with small dataset
+    model, train_losses, val_losses = train_cloud_removal_model(
+        data_dir="winter_dataset",
+        use_s1=True,  # Use S1 radar data
+        batch_size=4,  # Small batch for testing
+        num_epochs=5,  # Few epochs for quick test
+        learning_rate=1e-4
+    )
+    
+    print("\n✅ Training test completed!")
+    print(f"Final train loss: {train_losses[-1]:.6f}")
+    print(f"Final val loss: {val_losses[-1]:.6f}")

.temp/test_cloud_upload.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python3
+"""
+Test Cloud Removal Model Upload Feature
+"""
+
+import requests
+import json
+from pathlib import Path
+
+# API base URL
+BASE_URL = "http://localhost:8000"
+
+def test_upload_cloud_model(file_path):
+    """Test uploading a cloud removal model"""
+    print(f"\n{'='*60}")
+    print("TEST 1: Upload Cloud Removal Model")
+    print(f"{'='*60}")
+    
+    if not Path(file_path).exists():
+        print(f"❌ File not found: {file_path}")
+        print("   Create a dummy .pth file for testing:")
+        print(f"   touch {file_path}")
+        return None
+    
+    with open(file_path, 'rb') as f:
+        files = {'file': (Path(file_path).name, f, 'application/octet-stream')}
+        
+        print(f"📤 Uploading: {file_path}")
+        response = requests.post(f"{BASE_URL}/api/cloud-removal/upload", files=files)
+        
+        if response.status_code == 200:
+            result = response.json()
+            print(f"✅ Upload successful!")
+            print(f"   Filename: {result['filename']}")
+            print(f"   Size: {result['size_mb']} MB")
+            print(f"   Path: {result['path']}")
+            return result['filename']
+        else:
+            print(f"❌ Upload failed: {response.status_code}")
+            print(f"   {response.json().get('detail', 'Unknown error')}")
+            return None
+
+def test_list_cloud_models():
+    """Test listing cloud removal models"""
+    print(f"\n{'='*60}")
+    print("TEST 2: List Cloud Removal Models")
+    print(f"{'='*60}")
+    
+    response = requests.get(f"{BASE_URL}/api/cloud-removal/models")
+    
+    if response.status_code == 200:
+        data = response.json()
+        print(f"✅ Found {data['count']} models:")
+        for i, model in enumerate(data['models'], 1):
+            print(f"\n   {i}. {model['filename']}")
+            print(f"      Size: {model['size_mb']} MB")
+            print(f"      Created: {model['created']}")
+            if 'epoch' in model:
+                print(f"      Epoch: {model['epoch']}, Val Loss: {model['val_loss']:.4f}")
+        return data['models']
+    else:
+        print(f"❌ Failed to list models: {response.status_code}")
+        return []
+
+def test_prediction_with_cloud_model(model_filename, cloud_model_filename):
+    """Test prediction using uploaded cloud removal model"""
+    print(f"\n{'='*60}")
+    print("TEST 3: Prediction with Custom Cloud Removal Model")
+    print(f"{'='*60}")
+    
+    config = {
+        "model_filename": model_filename,
+        "min_lon": 105.80,
+        "min_lat": 10.00,
+        "max_lon": 105.82,
+        "max_lat": 10.02,
+        "start_date": "2024-01-15",
+        "end_date": "2024-01-17",
+        "max_scenes": 2,
+        "cloud_cover": 30,
+        "resolution": 20,
+        "use_gpu": False,
+        "export_ndvi": True,
+        "export_classification": True,
+        "cloud_removal_method": "deep",
+        "cloud_removal_model": cloud_model_filename
+    }
+    
+    print("📊 Prediction Config:")
+    print(json.dumps(config, indent=2))
+    
+    print(f"\n🚀 Starting prediction with cloud removal model: {cloud_model_filename}")
+    response = requests.post(
+        f"{BASE_URL}/api/predict/with-ndvi",
+        json=config,
+        headers={'Content-Type': 'application/json'}
+    )
+    
+    if response.status_code == 200:
+        result = response.json()
+        print(f"✅ Prediction started!")
+        print(f"   Message: {result.get('message')}")
+        return result
+    else:
+        print(f"❌ Prediction failed: {response.status_code}")
+        print(f"   {response.json().get('detail', 'Unknown error')}")
+        return None
+
+def test_delete_cloud_model(filename):
+    """Test deleting a cloud removal model"""
+    print(f"\n{'='*60}")
+    print("TEST 4: Delete Cloud Removal Model")
+    print(f"{'='*60}")
+    
+    print(f"🗑️  Deleting: {filename}")
+    response = requests.delete(f"{BASE_URL}/api/cloud-removal/models/{filename}")
+    
+    if response.status_code == 200:
+        result = response.json()
+        print(f"✅ {result['message']}")
+        return True
+    else:
+        print(f"❌ Delete failed: {response.status_code}")
+        return False
+
+def main():
+    print("="*60)
+    print("CLOUD REMOVAL MODEL UPLOAD - FEATURE TEST")
+    print("="*60)
+    
+    # Test file path (create a dummy file for testing)
+    test_file = "test_cloud_removal_model.pth"
+    
+    # Create dummy file if it doesn't exist
+    if not Path(test_file).exists():
+        print(f"\n📝 Creating dummy test file: {test_file}")
+        Path(test_file).write_bytes(b"dummy_pytorch_model_data")
+    
+    # Run tests
+    uploaded_filename = test_upload_cloud_model(test_file)
+    
+    if uploaded_filename:
+        models = test_list_cloud_models()
+        
+        # Test prediction (requires a real land classification model)
+        print(f"\n{'='*60}")
+        print("NOTE: Prediction test requires a trained land classification model")
+        print("      Skipping prediction test in this demo")
+        print(f"{'='*60}")
+        
+        # Cleanup - delete test model
+        if input("\nDelete test model? (y/n): ").lower() == 'y':
+            test_delete_cloud_model(uploaded_filename)
+    
+    # Cleanup dummy file
+    if Path(test_file).exists():
+        Path(test_file).unlink()
+        print(f"\n🗑️  Cleaned up dummy file: {test_file}")
+    
+    print(f"\n{'='*60}")
+    print("TESTS COMPLETED")
+    print(f"{'='*60}")
+
+if __name__ == "__main__":
+    main()

.temp/test_empty_points.py

@@ -0,0 +1,53 @@
+import geopandas as gpd
+import planetary_computer
+import pystac_client
+import odc.stac
+import numpy as np
+from shapely.geometry import Point, shape
+from pyproj import Transformer
+
+bbox = [105.5, 9.2, 106.3, 10.0]
+time_range = "2023-01-01/2023-04-30"
+catalog = pystac_client.Client.open(
+    "https://planetarycomputer.microsoft.com/api/stac/v1",
+    modifier=planetary_computer.sign_inplace,
+)
+items = list(catalog.search(collections=["sentinel-2-l2a"], bbox=bbox, datetime=time_range, query={"eo:cloud_cover": {"lt": 30}}).items())
+items = sorted(items, key=lambda x: x.properties["eo:cloud_cover"])
+
+gdf = gpd.read_file("train/ST_training_data_updated_1130points_new.shp")
+gdf = gdf.to_crs("EPSG:32648")
+
+# Find a point that fails. Let's just test a few points.
+for idx, row in gdf.head(20).iterrows():
+    x_coord = row['geometry'].x
+    y_coord = row['geometry'].y
+    
+    transformer = Transformer.from_crs("epsg:32648", "epsg:4326", always_xy=True)
+    lon, lat = transformer.transform(x_coord, y_coord)
+    point = Point(lon, lat)
+    
+    filtered = []
+    for item in items:
+        if shape(item.geometry).contains(point):
+            filtered.append(item)
+            
+    filtered = [planetary_computer.sign(item) for item in filtered]
+    if not filtered:
+        print(f"Point {idx}: NO ITEMS CONTAINS POINT!")
+        continue
+        
+    ds = odc.stac.load(
+        filtered,
+        bands=["B02"],
+        x=(x_coord - 80, x_coord + 80),
+        y=(y_coord - 80, y_coord + 80),
+        crs="EPSG:32648",
+        resolution=10,
+        patch_url=planetary_computer.sign,
+        fail_on_error=False
+    ).compute()
+    
+    sums = ds["B02"].sum(dim=["x", "y"]).values
+    non_zero = (sums > 0).sum()
+    print(f"Point {idx}: {len(filtered)} items, {non_zero} non-zero time steps")

.temp/test_extract_one.py

@@ -0,0 +1,37 @@
+import geopandas as gpd
+import planetary_computer
+import pystac_client
+import odc.stac
+import sys
+
+bbox = [105.5, 9.2, 106.3, 10.0]
+time_range = "2023-01-01/2023-04-30"
+catalog = pystac_client.Client.open("https://planetarycomputer.microsoft.com/api/stac/v1", modifier=planetary_computer.sign_inplace)
+search = catalog.search(collections=["sentinel-2-l2a"], bbox=bbox, datetime=time_range, query={"eo:cloud_cover": {"lt": 30}})
+items = list(search.items())
+items = sorted(items, key=lambda x: x.properties.get("eo:cloud_cover", 100))[:4]
+items = [planetary_computer.sign(item) for item in items]
+
+gdf = gpd.read_file("train/ST_training_data_updated_1130points_new.shp")
+gdf = gdf.to_crs("EPSG:32648")
+
+row = gdf.iloc[0]
+x, y_coord = row.geometry.x, row.geometry.y
+point_bbox = [x - 80, y_coord - 80, x + 80, y_coord + 80]
+
+patch_s2 = odc.stac.load(
+    items,
+    bands=["B02", "B03", "B04", "B08", "SCL"],
+    x=(x - 80, x + 80),
+    y=(y_coord - 80, y_coord + 80),
+    crs="EPSG:32648",
+    resolution=10,
+    patch_url=planetary_computer.sign,
+    fail_on_error=False
+).compute()
+
+print("patch_s2 vars:", patch_s2.data_vars)
+if patch_s2.dims['x'] < 16 or patch_s2.dims['y'] < 16:
+    print("Too small:", patch_s2.dims)
+else:
+    print("Success dimension:", patch_s2.dims)

.temp/test_feature_extractor.py

@@ -0,0 +1,171 @@
+"""
+Test FeatureExtractor và kiểm tra tích hợp với hệ thống
+"""
+
+import numpy as np
+import xarray as xr
+from feature_extractor import get_feature_extractor
+from pathlib import Path
+
+print("=" * 70)
+print("TESTING FEATURE EXTRACTOR MODULE")
+print("=" * 70)
+
+# Test 1: Simple mode
+print("\n[TEST 1] Simple Mode (3 features)")
+print("-" * 50)
+extractor_simple = get_feature_extractor(mode='simple')
+print(f"✓ Created extractor: {extractor_simple.mode}")
+print(f"✓ Expected features: {extractor_simple.config['n_features']}")
+print(f"✓ Feature names: {extractor_simple.get_feature_names()}")
+
+# Create dummy NDVI data
+ndvi_dummy = xr.DataArray(
+    np.random.rand(10, 10),
+    dims=['y', 'x'],
+    coords={'y': np.arange(10), 'x': np.arange(10)}
+)
+vh_dummy = xr.DataArray(
+    np.random.rand(10, 10) * -10,
+    dims=['y', 'x'],
+    coords={'y': np.arange(10), 'x': np.arange(10)}
+)
+vv_dummy = xr.DataArray(
+    np.random.rand(10, 10) * -8,
+    dims=['y', 'x'],
+    coords={'y': np.arange(10), 'x': np.arange(10)}
+)
+
+features_simple = extractor_simple.extract(
+    ndvi_data=ndvi_dummy,
+    vh_data=vh_dummy,
+    vv_data=vv_dummy
+)
+print(f"✓ Extracted features shape: {features_simple.shape}")
+assert features_simple.shape[1] == 3, "Expected 3 features"
+print("✅ Simple mode test PASSED\n")
+
+# Test 2: Extended mode
+print("[TEST 2] Extended Mode (15 features)")
+print("-" * 50)
+extractor_extended = get_feature_extractor(mode='extended')
+print(f"✓ Created extractor: {extractor_extended.mode}")
+print(f"✓ Expected features: {extractor_extended.config['n_features']}")
+print(f"✓ Feature names: {extractor_extended.get_feature_names()}")
+
+# Create dummy S2 dataset with time dimension
+s2_dummy = xr.Dataset({
+    'B02': xr.DataArray(np.random.rand(5, 10, 10), dims=['time', 'y', 'x']),
+    'B03': xr.DataArray(np.random.rand(5, 10, 10), dims=['time', 'y', 'x']),
+    'B04': xr.DataArray(np.random.rand(5, 10, 10), dims=['time', 'y', 'x']),
+    'B08': xr.DataArray(np.random.rand(5, 10, 10), dims=['time', 'y', 'x']),
+    'B11': xr.DataArray(np.random.rand(5, 10, 10), dims=['time', 'y', 'x'])
+})
+
+features_extended = extractor_extended.extract(
+    s2_data=s2_dummy,
+    vh_data=vh_dummy,
+    vv_data=vv_dummy
+)
+print(f"✓ Extracted features shape: {features_extended.shape}")
+assert features_extended.shape[1] == 15, "Expected 15 features"
+print("✅ Extended mode test PASSED\n")
+
+# Test 3: Temporal mode
+print("[TEST 3] Temporal Mode (39 features for 12 timesteps)")
+print("-" * 50)
+extractor_temporal = get_feature_extractor(mode='temporal')
+print(f"✓ Created extractor: {extractor_temporal.mode}")
+
+# Create dummy S2 dataset with 12 timesteps
+s2_dummy_12 = xr.Dataset({
+    'B02': xr.DataArray(np.random.rand(12, 10, 10), dims=['time', 'y', 'x']),
+    'B03': xr.DataArray(np.random.rand(12, 10, 10), dims=['time', 'y', 'x']),
+    'B04': xr.DataArray(np.random.rand(12, 10, 10), dims=['time', 'y', 'x']),
+    'B08': xr.DataArray(np.random.rand(12, 10, 10), dims=['time', 'y', 'x']),
+    'B11': xr.DataArray(np.random.rand(12, 10, 10), dims=['time', 'y', 'x'])
+})
+
+features_temporal = extractor_temporal.extract(
+    s2_data=s2_dummy_12,
+    vh_data=vh_dummy,
+    vv_data=vv_dummy
+)
+
+# For temporal mode: 12 timesteps * 3 indices + 3 radar = 39 features
+expected_features = 12 * 3 + 3
+print(f"✓ Extracted features shape: {features_temporal.shape}")
+print(f"✓ Expected: {expected_features} features (12 timesteps * 3 indices + 3 radar)")
+
+feature_names_temporal = extractor_temporal.get_feature_names(n_timesteps=12)
+print(f"✓ Feature names count: {len(feature_names_temporal)}")
+print(f"✓ First 5 features: {feature_names_temporal[:5]}")
+print(f"✓ Last 5 features: {feature_names_temporal[-5:]}")
+
+assert features_temporal.shape[1] == expected_features, f"Expected {expected_features} features"
+assert len(feature_names_temporal) == expected_features, f"Expected {expected_features} feature names"
+print("✅ Temporal mode test PASSED\n")
+
+# Test 4: Check model_odc.joblib metadata
+print("[TEST 4] Verify model_odc.joblib metadata")
+print("-" * 50)
+metadata_file = Path("model_train/model_odc_info.json")
+if metadata_file.exists():
+    import json
+    with open(metadata_file) as f:
+        metadata = json.load(f)
+    
+    print(f"✓ Metadata file exists: {metadata_file}")
+    print(f"✓ Feature mode: {metadata.get('feature_mode')}")
+    print(f"✓ Number of features: {metadata.get('n_features')}")
+    print(f"✓ Features list length: {len(metadata.get('features', []))}")
+    print(f"✓ First 5 features: {metadata.get('features', [])[:5]}")
+    
+    assert metadata.get('feature_mode') == 'temporal', "Expected temporal mode"
+    assert metadata.get('n_features') == 39, "Expected 39 features"
+    assert len(metadata.get('features', [])) == 39, "Expected 39 feature names"
+    
+    print("✅ model_odc.joblib metadata VERIFIED\n")
+else:
+    print("❌ model_odc_info.json not found. Run: python create_odc_metadata.py")
+
+# Test 5: Check ModelManager integration
+print("[TEST 5] Test ModelManager integration")
+print("-" * 50)
+try:
+    from model_manager import get_model_manager
+    
+    manager = get_model_manager()
+    print(f"✓ ModelManager initialized")
+    
+    # List models
+    models = manager.list_models()
+    print(f"✓ Found {len(models)} models")
+    
+    # Check if model_odc.joblib has metadata
+    odc_model = next((m for m in models if m['filename'] == 'model_odc.joblib'), None)
+    if odc_model:
+        print(f"✓ model_odc.joblib found in list")
+        print(f"  - Feature mode: {odc_model.get('feature_mode', 'N/A')}")
+        print(f"  - N features: {odc_model.get('n_features', 'N/A')}")
+        print("✅ ModelManager integration test PASSED\n")
+    else:
+        print("⚠️  model_odc.joblib not in model list")
+        
+except Exception as e:
+    print(f"❌ ModelManager test failed: {e}")
+
+# Summary
+print("=" * 70)
+print("TEST SUMMARY")
+print("=" * 70)
+print("✅ All feature extraction modes working correctly")
+print("✅ Feature dimensions match expectations")
+print("✅ Feature names generated correctly")
+print("✅ model_odc.joblib metadata verified")
+print("\nNext steps:")
+print("1. Update api_server.py with run_prediction from run_prediction_new.py")
+print("2. Test training with different feature_modes")
+print("3. Test prediction with models using different modes")
+print("\nSee UPDATE_SUMMARY.md for details.")
+print("=" * 70)

.temp/test_gdf.py

@@ -0,0 +1,3 @@
+import geopandas as gpd
+gdf = gpd.read_file("train/ST_training_data_updated_1130points_new.shp")
+print(gdf.head(1)['HT_code'])

.temp/test_gdf_cols.py

@@ -0,0 +1,3 @@
+import geopandas as gpd
+gdf = gpd.read_file("train/ST_training_data_updated_1130points_new.shp")
+print(gdf.columns)

.temp/test_gpu.py

@@ -0,0 +1,23 @@
+import numpy as np
+from xgboost import XGBClassifier
+from sklearn.datasets import make_classification
+from sklearn.metrics import accuracy_score
+
+print("🚀 Testing XGBoost with CUDA GPU...")
+try:
+    X, y = make_classification(n_samples=10000, n_features=20, n_classes=2, random_state=42)
+    model = XGBClassifier(
+        n_estimators=100,
+        max_depth=10,
+        tree_method="hist",
+        device="cuda",
+        random_state=42,
+        verbosity=1
+    )
+    print("Training model...")
+    model.fit(X, y)
+    y_pred = model.predict(X)
+    acc = accuracy_score(y, y_pred)
+    print(f"✅ Training successful! Accuracy: {acc*100:.2f}%")
+except Exception as e:
+    print(f"❌ Error during training: {e}")

.temp/test_keys.py

@@ -0,0 +1,3 @@
+import torch
+checkpoint = torch.load('cloud_removal_model/cloud_removal_unet_best.pth', map_location='cpu')
+print(list(checkpoint['model_state_dict'].keys())[:5])

.temp/test_model_manager.py

@@ -0,0 +1,101 @@
+"""
+Test script for Model Manager
+Kiểm tra các chức năng: list models, load models, validate models
+"""
+
+from model_manager import ModelManager, get_model_manager
+import json
+
+def test_model_manager():
+    print("="*70)
+    print("MODEL MANAGER TEST")
+    print("="*70)
+    
+    # Initialize ModelManager
+    model_manager = get_model_manager()
+    print("\n✅ ModelManager initialized")
+    
+    # Test 1: List all models
+    print("\n" + "="*70)
+    print("TEST 1: LIST ALL MODELS")
+    print("="*70)
+    
+    models = model_manager.list_models()
+    print(f"\n📦 Found {len(models)} models:")
+    
+    for idx, model in enumerate(models, 1):
+        print(f"\n[{idx}] {model['filename']}")
+        print(f"    Size: {model['size_mb']:.2f} MB")
+        print(f"    Modified: {model['modified']}")
+        
+        if model.get('has_metadata'):
+            print(f"    Type: {model.get('model_type', 'N/A')}")
+            print(f"    Features: {model.get('n_features', 'N/A')}")
+            print(f"    Accuracy: {model.get('test_accuracy', 'N/A')}")
+            print(f"    Feature list: {model.get('features', [])}")
+        else:
+            print(f"    ⚠️  No metadata")
+    
+    # Test 2: Load a model
+    if len(models) > 0:
+        print("\n" + "="*70)
+        print("TEST 2: LOAD MODEL")
+        print("="*70)
+        
+        test_model = models[0]['filename']
+        print(f"\n🔄 Loading model: {test_model}")
+        
+        try:
+            model, encoder, metadata = model_manager.load_model(test_model)
+            print(f"✅ Model loaded successfully!")
+            print(f"\n📊 Metadata:")
+            print(json.dumps(metadata, indent=2))
+            
+            # Test 3: Validate model
+            print("\n" + "="*70)
+            print("TEST 3: VALIDATE MODEL")
+            print("="*70)
+            
+            validation = model_manager.validate_model(test_model)
+            print(f"\n✅ Validation result:")
+            print(f"  Valid: {validation['valid']}")
+            if validation['errors']:
+                print(f"  Errors: {validation['errors']}")
+            if validation['warnings']:
+                print(f"  Warnings: {validation['warnings']}")
+            
+            # Test 4: Get required features
+            print("\n" + "="*70)
+            print("TEST 4: GET REQUIRED FEATURES")
+            print("="*70)
+            
+            features = model_manager.get_required_features(test_model)
+            print(f"\n📋 Required features for {test_model}:")
+            for feat in features:
+                print(f"  - {feat}")
+            
+        except Exception as e:
+            print(f"❌ Error loading model: {e}")
+            import traceback
+            traceback.print_exc()
+    
+    # Test 5: Get latest model
+    print("\n" + "="*70)
+    print("TEST 5: GET LATEST MODEL")
+    print("="*70)
+    
+    latest = model_manager.get_latest_model()
+    print(f"\n📌 Latest model: {latest}")
+    
+    latest_xgb = model_manager.get_latest_model(model_type='xgboost')
+    print(f"📌 Latest XGBoost model: {latest_xgb}")
+    
+    latest_cnn = model_manager.get_latest_model(model_type='cnn')
+    print(f"📌 Latest CNN model: {latest_cnn}")
+    
+    print("\n" + "="*70)
+    print("✅ ALL TESTS COMPLETED")
+    print("="*70)
+
+if __name__ == "__main__":
+    test_model_manager()

.temp/test_model_struct.py

@@ -0,0 +1,13 @@
+import torch
+from pathlib import Path
+model = torch.load('cloud_removal_model/cloud_removal_unet_best.pth', map_location='cpu')
+print(type(model))
+print("hasattr inc:", hasattr(model, 'inc'))
+if hasattr(model, 'inc'):
+    print("hasattr double_conv:", hasattr(model.inc, 'double_conv'))
+    if hasattr(model.inc, 'double_conv'):
+        print("in_channels:", model.inc.double_conv[0].in_channels)
+else:
+    for name, param in model.named_parameters():
+        print(name, param.shape)
+        break

.temp/test_new_features.py

@@ -0,0 +1,191 @@
+#!/usr/bin/env python3
+"""
+Demo script để test các chức năng mới của API
+"""
+
+import requests
+import json
+import time
+from pathlib import Path
+
+BASE_URL = "http://localhost:8000"
+
+def print_section(title):
+    print("\n" + "=" * 70)
+    print(f"  {title}")
+    print("=" * 70)
+
+def test_dashboard_statistics():
+    print_section("📊 Test Dashboard Statistics")
+    try:
+        response = requests.get(f"{BASE_URL}/api/dashboard/statistics")
+        if response.status_code == 200:
+            data = response.json()
+            print(f"✅ Success!")
+            print(f"   Models: {data['models']['total']}")
+            print(f"   Predictions: {data['predictions']['total']}")
+            print(f"   Reports: {data['reports']['total']}")
+        else:
+            print(f"❌ Error: {response.status_code}")
+    except Exception as e:
+        print(f"❌ Exception: {e}")
+
+def test_accuracy_trends():
+    print_section("📈 Test Accuracy Trends")
+    try:
+        response = requests.get(f"{BASE_URL}/api/dashboard/accuracy-trends")
+        if response.status_code == 200:
+            data = response.json()
+            print(f"✅ Success!")
+            print(f"   Trends: {len(data['trends'])} records")
+            print(f"   Models: {data['models']}")
+        else:
+            print(f"❌ Error: {response.status_code}")
+    except Exception as e:
+        print(f"❌ Exception: {e}")
+
+def test_class_distribution():
+    print_section("📊 Test Class Distribution")
+    try:
+        # First, get list of models
+        response = requests.get(f"{BASE_URL}/api/models/list")
+        if response.status_code == 200:
+            models = response.json()['models']
+            if models:
+                model_filename = models[0]['filename']
+                print(f"   Using model: {model_filename}")
+                
+                # Get class distribution
+                response = requests.get(f"{BASE_URL}/api/dashboard/class-distribution/{model_filename}")
+                if response.status_code == 200:
+                    data = response.json()
+                    print(f"✅ Success!")
+                    print(f"   Total samples: {data['total_samples']}")
+                    print(f"   Classes: {list(data['class_distribution'].keys())}")
+                else:
+                    print(f"❌ Error: {response.status_code}")
+            else:
+                print("⚠️  No models found")
+        else:
+            print(f"❌ Error getting models: {response.status_code}")
+    except Exception as e:
+        print(f"❌ Exception: {e}")
+
+def test_batch_status():
+    print_section("🔄 Test Batch Status")
+    try:
+        response = requests.get(f"{BASE_URL}/api/batch/status")
+        if response.status_code == 200:
+            data = response.json()
+            print(f"✅ Success!")
+            print(f"   Queued: {data['queue']['queued']}")
+            print(f"   Running: {data['queue']['running']}")
+            print(f"   Completed: {data['queue']['completed']}")
+            print(f"   Failed: {data['queue']['failed']}")
+        else:
+            print(f"❌ Error: {response.status_code}")
+    except Exception as e:
+        print(f"❌ Exception: {e}")
+
+def test_batch_prediction_demo():
+    print_section("🚀 Test Batch Prediction (Demo)")
+    try:
+        # Get a model
+        response = requests.get(f"{BASE_URL}/api/models/list")
+        if response.status_code != 200:
+            print("❌ Cannot get models list")
+            return
+        
+        models = response.json()['models']
+        if not models:
+            print("⚠️  No models available for testing")
+            return
+        
+        model_filename = models[0]['filename']
+        print(f"   Using model: {model_filename}")
+        
+        # Create test batch
+        batch_config = {
+            "model_filename": model_filename,
+            "items": [
+                {
+                    "name": "Test_Region_1",
+                    "min_lon": 105.6,
+                    "min_lat": 9.3,
+                    "max_lon": 105.7,
+                    "max_lat": 9.4,
+                    "start_date": "2023-03-01",
+                    "end_date": "2023-03-31",
+                    "max_scenes": 5,
+                    "cloud_cover": 30,
+                    "resolution": 20
+                }
+            ],
+            "auto_retry": True,
+            "max_retries": 2
+        }
+        
+        print("   Creating batch job...")
+        response = requests.post(
+            f"{BASE_URL}/api/batch/start",
+            json=batch_config
+        )
+        
+        if response.status_code == 200:
+            data = response.json()
+            print(f"✅ Success!")
+            print(f"   {data['message']}")
+            print(f"   Batch ID: {data['batch_id']}")
+            
+            # Check status after a moment
+            time.sleep(2)
+            response = requests.get(f"{BASE_URL}/api/batch/status")
+            if response.status_code == 200:
+                status = response.json()
+                print(f"   Current queue: {status['queue']}")
+        else:
+            print(f"❌ Error: {response.status_code} - {response.text}")
+    except Exception as e:
+        print(f"❌ Exception: {e}")
+
+def test_reports_list():
+    print_section("📄 Test Reports List")
+    try:
+        response = requests.get(f"{BASE_URL}/api/reports/list")
+        if response.status_code == 200:
+            data = response.json()
+            print(f"✅ Success!")
+            print(f"   Total reports: {data['count']}")
+            if data['reports']:
+                print(f"   Latest report: {data['reports'][0]['filename']}")
+        else:
+            print(f"❌ Error: {response.status_code}")
+    except Exception as e:
+        print(f"❌ Exception: {e}")
+
+def main():
+    print("=" * 70)
+    print("  🧪 API Testing Suite - New Features")
+    print("=" * 70)
+    print(f"\n  Base URL: {BASE_URL}")
+    print(f"  Đảm bảo server đang chạy: python api_server.py")
+    
+    input("\n  Press ENTER to start testing...")
+    
+    # Run all tests
+    test_dashboard_statistics()
+    test_accuracy_trends()
+    test_class_distribution()
+    test_reports_list()
+    test_batch_status()
+    test_batch_prediction_demo()
+    
+    print("\n" + "=" * 70)
+    print("  ✅ Testing completed!")
+    print("=" * 70)
+    print(f"\n  Dashboard: {BASE_URL}/dashboard")
+    print(f"  API Docs: {BASE_URL}/docs")
+    print("=" * 70 + "\n")
+
+if __name__ == "__main__":
+    main()

.temp/test_odc_stac.py

@@ -0,0 +1,19 @@
+import geopandas as gpd
+import planetary_computer
+import pystac_client
+import odc.stac
+
+bbox = [105.5, 9.2, 106.3, 10.0]
+time_range = "2023-01-01/2023-01-30"
+catalog = pystac_client.Client.open("https://planetarycomputer.microsoft.com/api/stac/v1", modifier=planetary_computer.sign_inplace)
+items = list(catalog.search(collections=["sentinel-2-l2a"], bbox=bbox, datetime=time_range).items())[:1]
+items = [planetary_computer.sign(item) for item in items]
+
+x = 561609
+y = 1024183
+try:
+    ds = odc.stac.load(items, bands=["B02"], crs="EPSG:32648", resolution=10, x=(x-80, x+80), y=(y-80, y+80))
+    print("Success with x/y:", ds.dims)
+except Exception as e:
+    print("Error with x/y:", e)
+

.temp/test_overlap.py

@@ -0,0 +1,18 @@
+import geopandas as gpd
+import planetary_computer
+import pystac_client
+import odc.stac
+import numpy as np
+
+bbox = [105.5, 9.2, 106.3, 10.0]
+time_range = "2023-01-01/2023-04-30"
+catalog = pystac_client.Client.open("https://planetarycomputer.microsoft.com/api/stac/v1", modifier=planetary_computer.sign_inplace)
+# Get ALL items
+items = list(catalog.search(collections=["sentinel-2-l2a"], bbox=bbox, datetime=time_range, query={"eo:cloud_cover": {"lt": 30}}).items())
+items = [planetary_computer.sign(item) for item in items]
+print(f"Total items: {len(items)}")
+
+x = 561609
+y = 1024183
+ds = odc.stac.load(items, bands=["B02"], x=(x-80, x+80), y=(y-80, y+80), crs="EPSG:32648", resolution=10, patch_url=planetary_computer.sign).compute()
+print("Time dimension size:", ds.dims['time'])

.temp/test_overlap2.py

@@ -0,0 +1,20 @@
+import geopandas as gpd
+import planetary_computer
+import pystac_client
+import odc.stac
+import numpy as np
+
+bbox = [105.5, 9.2, 106.3, 10.0]
+time_range = "2023-01-01/2023-04-30"
+catalog = pystac_client.Client.open("https://planetarycomputer.microsoft.com/api/stac/v1", modifier=planetary_computer.sign_inplace)
+items = list(catalog.search(collections=["sentinel-2-l2a"], bbox=bbox, datetime=time_range, query={"eo:cloud_cover": {"lt": 30}}).items())
+items = [planetary_computer.sign(item) for item in items]
+
+x = 561609
+y = 1024183
+ds = odc.stac.load(items, bands=["B02"], x=(x-80, x+80), y=(y-80, y+80), crs="EPSG:32648", resolution=10, patch_url=planetary_computer.sign, fail_on_error=False).compute()
+print("Original time size:", len(ds.time))
+ds2 = ds.dropna(dim="time", how="all")
+print("After dropna time size:", len(ds2.time))
+print("B02 mean:", np.nanmean(ds2["B02"].values))
+print("B02 non-nan count:", np.sum(~np.isnan(ds2["B02"].values)))

.temp/test_overlap_issue.py

@@ -0,0 +1,25 @@
+import geopandas as gpd
+import planetary_computer
+import pystac_client
+import odc.stac
+import numpy as np
+
+bbox = [105.5, 9.2, 106.3, 10.0]
+time_range = "2023-01-01/2023-04-30"
+catalog = pystac_client.Client.open("https://planetarycomputer.microsoft.com/api/stac/v1", modifier=planetary_computer.sign_inplace)
+items = list(catalog.search(collections=["sentinel-2-l2a"], bbox=bbox, datetime=time_range, query={"eo:cloud_cover": {"lt": 30}}).items())
+items = [planetary_computer.sign(item) for item in items]
+
+x = 561609
+y = 1024183
+ds = odc.stac.load(items, bands=["B02", "B03", "B04", "B08", "SCL"], x=(x-80, x+80), y=(y-80, y+80), crs="EPSG:32648", resolution=10, patch_url=planetary_computer.sign, fail_on_error=False).compute()
+
+print("Original shape:", ds["B02"].shape)
+ds2 = ds.dropna(dim="time", how="all")
+print("After dropna time size:", len(ds2.time))
+if len(ds2.time) > 0:
+    ds2 = ds2.isel(time=slice(0, 4))
+    median = ds2["B02"].median(dim="time", skipna=True).values
+    print("Median shape:", median.shape)
+    print("Zeros in median:", np.sum(median == 0) / median.size)
+    print("NaNs in median:", np.sum(np.isnan(median)) / median.size)

.temp/test_planetary.py

@@ -0,0 +1,15 @@
+import sys
+# Thêm đường dẫn hiện tại vào PYTHONPATH để import được new_import_ODC nếu cần
+sys.path.append('.')
+import warnings
+warnings.filterwarnings('ignore')
+from new_import_ODC import load_sen1
+
+print("Testing load_sen1 with a short time range to speed up Dask compute...")
+bbox = [105.5, 9.2, 106.4, 10.0]
+time_range = "2023-01-01/2023-01-31" # Short time range for fast testing
+vh, vv = load_sen1(bbox, time_range)
+print("VH shape:", vh.shape)
+print("VV shape:", vv.shape)
+print("VH CRS:", vh.rio.crs)
+print("Success!")

.temp/test_planetary2.py

@@ -0,0 +1,49 @@
+import warnings
+warnings.filterwarnings('ignore')
+
+def load_sen1(bbox, time_range):
+    import pystac_client
+    import planetary_computer
+    import odc.stac
+    
+    catalog = pystac_client.Client.open(
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
+        modifier=planetary_computer.sign_inplace,
+    )
+
+    search = catalog.search(
+        collections=["sentinel-1-rtc"],
+        bbox=bbox,
+        datetime=time_range,
+    )
+    items = list(search.items())
+    print("Found items:", len(items))
+
+    ds_s1 = odc.stac.load(
+        items,
+        bands=["vv", "vh"],
+        bbox=bbox,
+        crs="EPSG:32648",
+        resolution=10,
+        chunks={"x": 2048, "y": 2048, "time": 1}
+    )
+
+    ds_median = ds_s1.median(dim="time").compute()
+    vv = ds_median["vv"]
+    vh = ds_median["vh"]
+
+    vv = vv.expand_dims(dim="band")
+    vh = vh.expand_dims(dim="band")
+
+    vv = vv.rio.write_crs("EPSG:32648")
+    vh = vh.rio.write_crs("EPSG:32648")
+
+    return vh, vv
+
+print("Testing load_sen1...")
+bbox = [105.5, 9.2, 106.4, 10.0]
+time_range = "2022-09-01/2023-10-01"
+vh, vv = load_sen1(bbox, time_range)
+print("VH shape:", vh.shape)
+print("VV shape:", vv.shape)
+print("Success!")

.temp/test_planetary_computer.py

@@ -0,0 +1,88 @@
+"""
+Test Microsoft Planetary Computer connectivity và token
+"""
+import planetary_computer
+from pystac_client import Client
+from datetime import datetime, timedelta
+
+print("=" * 70)
+print("🧪 TESTING MICROSOFT PLANETARY COMPUTER CONNECTION")
+print("=" * 70)
+
+# Test 1: Basic connection
+print("\n1️⃣ Testing basic connection...")
+try:
+    catalog = Client.open(
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
+        modifier=planetary_computer.sign_inplace,
+    )
+    print("✅ Successfully connected to Planetary Computer")
+    print(f"   Catalog ID: {catalog.id}")
+    print(f"   Title: {catalog.title}")
+except Exception as e:
+    print(f"❌ Connection failed: {e}")
+    exit(1)
+
+# Test 2: List collections
+print("\n2️⃣ Testing collections access...")
+try:
+    collections = list(catalog.get_collections())
+    print(f"✅ Found {len(collections)} collections")
+    sentinel_2 = [c for c in collections if 'sentinel-2' in c.id.lower()]
+    print(f"   Sentinel-2 collections: {[c.id for c in sentinel_2]}")
+except Exception as e:
+    print(f"❌ Collections access failed: {e}")
+
+# Test 3: Small search query (very conservative)
+print("\n3️⃣ Testing small search query...")
+try:
+    # Tiny bbox in Vietnam
+    bbox = [105.8, 10.0, 105.9, 10.1]  # ~10km x 10km area
+    end_date = datetime.now()
+    start_date = end_date - timedelta(days=7)  # Last 7 days only
+    
+    time_range = f"{start_date.strftime('%Y-%m-%d')}/{end_date.strftime('%Y-%m-%d')}"
+    
+    print(f"   Bbox: {bbox}")
+    print(f"   Time: {time_range}")
+    print(f"   Searching...")
+    
+    search = catalog.search(
+        collections=["sentinel-2-l2a"],
+        bbox=bbox,
+        datetime=time_range,
+        limit=5  # Only 5 items
+    )
+    
+    items = []
+    for i, item in enumerate(search.items()):
+        items.append(item)
+        if i >= 4:  # Stop at 5
+            break
+    
+    print(f"✅ Search successful! Found {len(items)} items")
+    if items:
+        first_item = items[0]
+        print(f"   First item: {first_item.id}")
+        print(f"   Date: {first_item.datetime}")
+        
+        # Test token signing
+        signed_item = planetary_computer.sign(first_item)
+        print(f"✅ SAS token signing works")
+        print(f"   Asset keys: {list(signed_item.assets.keys())[:5]}")
+        
+except Exception as e:
+    print(f"❌ Search failed: {e}")
+    import traceback
+    traceback.print_exc()
+
+print("\n" + "=" * 70)
+print("🏁 Test completed!")
+print("=" * 70)
+print("\n💡 Nếu test này PASS:")
+print("   → Planetary Computer hoạt động bình thường")
+print("   → Vấn đề là query quá lớn (bbox/time range/max_scenes)")
+print("\n💡 Nếu test này FAIL:")
+print("   → Kiểm tra internet connection")
+print("   → Thử lại sau (server có thể bị quá tải)")
+print("   → Xem xét dùng dữ liệu local")

.temp/test_raw.py

@@ -0,0 +1,17 @@
+import geopandas as gpd
+import planetary_computer
+import pystac_client
+import odc.stac
+import numpy as np
+
+bbox = [105.5, 9.2, 106.3, 10.0]
+time_range = "2023-01-01/2023-04-30"
+catalog = pystac_client.Client.open("https://planetarycomputer.microsoft.com/api/stac/v1", modifier=planetary_computer.sign_inplace)
+items = list(catalog.search(collections=["sentinel-2-l2a"], bbox=bbox, datetime=time_range, query={"eo:cloud_cover": {"lt": 30}}).items())[:4]
+items = [planetary_computer.sign(item) for item in items]
+
+x = 561609
+y = 1024183
+ds = odc.stac.load(items, bands=["B02", "B03", "B04", "B08"], x=(x-80, x+80), y=(y-80, y+80), crs="EPSG:32648", resolution=10, patch_url=planetary_computer.sign).compute()
+print("B04 nanmean:", np.nanmean(ds["B04"].values))
+print("B04 nanmax:", np.nanmax(ds["B04"].values))

.temp/test_shape.py

@@ -0,0 +1,41 @@
+import new_import_ODC
+importlib = __import__('importlib')
+importlib.reload(new_import_ODC)
+from new_import_ODC import *
+import numpy as np
+
+date_range = ("2022-09-01", "2022-10-01")
+longtitude_range = (105.86, 105.94)
+latitude_range = (9.65, 9.69)
+coordinates = (longtitude_range, latitude_range)
+
+print("Loading S2...")
+data = load_data(None, date_range, longtitude_range, latitude_range)
+result = mask_clean(data)
+ds1 = calculate_indices(result, index="NDVI", satellite_mission="s2")
+ndvi = ds1["NDVI"]
+time_split = [
+    slice("2022-09-01", "2023-01-01"),
+    slice("2023-01-01", "2023-05-01"),
+    slice("2023-05-01", "2023-07-01"),
+    slice("2023-07-01", "2022-10-01"),
+]
+fill_nan_ndvi = fill_nan(ndvi, time_split)
+average_ndvi = fill_nan_ndvi.resample(time="1M").mean().compute()
+
+print("Loading S1...")
+dsvh, dsvv = load_data_sen1(None, date_range, coordinates)
+average_vv = calculate_average(dsvv, time_pattern='1M')
+average_vh = calculate_average(dsvh, time_pattern='1M')
+
+train = load_train_data("train/ST_training_data_updated_1130points_new.shp")
+point = train.iloc[0]
+
+ndvi_val = average_ndvi.sel(x=point.geometry.x, y=point.geometry.y, method='nearest').values
+vh_val = average_vh.sel(x=point.geometry.x, y=point.geometry.y, method='nearest').values
+vv_val = average_vv.sel(x=point.geometry.x, y=point.geometry.y, method='nearest').values
+
+print("NDVI shape:", ndvi_val.shape, "ndim:", ndvi_val.ndim)
+print("VH shape:", vh_val.shape, "ndim:", vh_val.ndim)
+print("VV shape:", vv_val.shape, "ndim:", vv_val.ndim)
+

.temp/test_spatial_filter.py

@@ -0,0 +1,42 @@
+import geopandas as gpd
+import planetary_computer
+import pystac_client
+import odc.stac
+import numpy as np
+import time
+from shapely.geometry import Point, box, shape
+
+bbox = [105.5, 9.2, 106.3, 10.0]
+time_range = "2023-01-01/2023-04-30"
+catalog = pystac_client.Client.open("https://planetarycomputer.microsoft.com/api/stac/v1", modifier=planetary_computer.sign_inplace)
+items = list(catalog.search(collections=["sentinel-2-l2a"], bbox=bbox, datetime=time_range, query={"eo:cloud_cover": {"lt": 30}}).items())
+
+x = 561609
+y = 1024183
+
+start = time.time()
+# Filter items by spatial intersection
+from pyproj import Transformer
+# The items geometry are in EPSG:4326 (lon, lat)
+# Our x, y are in EPSG:32648
+transformer = Transformer.from_crs("epsg:32648", "epsg:4326", always_xy=True)
+lon, lat = transformer.transform(x, y)
+point = Point(lon, lat)
+
+filtered_items = []
+for item in items:
+    geom = shape(item.geometry)
+    if geom.contains(point):
+        filtered_items.append(item)
+
+filtered_items = sorted(filtered_items, key=lambda x: x.properties["eo:cloud_cover"])
+
+print("Original items:", len(items))
+print("Filtered items:", len(filtered_items))
+print("Time to filter:", time.time() - start)
+
+start = time.time()
+filtered_items = [planetary_computer.sign(item) for item in filtered_items]
+ds = odc.stac.load(filtered_items[:4], bands=["B02"], x=(x-80, x+80), y=(y-80, y+80), crs="EPSG:32648", resolution=10, patch_url=planetary_computer.sign, fail_on_error=False).compute()
+print("Time to load 4 items:", time.time() - start)
+print(ds["B02"].shape)

.temp/test_training_api.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+"""
+Test script to verify training API endpoints
+"""
+
+import requests
+import json
+
+API_BASE = "http://localhost:8000/api"
+
+def test_training_labels():
+    """Test /api/training/labels endpoint"""
+    print("=" * 70)
+    print("TEST 1: Getting training labels")
+    print("=" * 70)
+    
+    response = requests.get(f"{API_BASE}/training/labels")
+    if response.ok:
+        data = response.json()
+        print(f"✅ Success! Found {data['count']} labels:")
+        for label in data['labels']:
+            print(f"  {label['code']}: {label['name']}")
+    else:
+        print(f"❌ Error: {response.status_code}")
+    print()
+
+def test_training_files():
+    """Test /api/training/files endpoint"""
+    print("=" * 70)
+    print("TEST 2: Getting training files")
+    print("=" * 70)
+    
+    response = requests.get(f"{API_BASE}/training/files")
+    if response.ok:
+        data = response.json()
+        print(f"✅ Success! Found {data['count']} training files:")
+        for file in data['files']:
+            print(f"\n  📄 {file['filename']}")
+            print(f"     Size: {file['size_mb']} MB")
+            if 'point_count' in file:
+                print(f"     Points: {file['point_count']}")
+                print(f"     Label column: {file.get('label_column', 'N/A')}")
+                print(f"     Unique labels: {file.get('label_count', 0)}")
+    else:
+        print(f"❌ Error: {response.status_code}")
+    print()
+
+def test_shapefile_labels(filename="ST_training data_updated_1130points_new.shp"):
+    """Test /api/training/shapefile/{filename}/labels endpoint"""
+    print("=" * 70)
+    print(f"TEST 3: Getting labels from shapefile: {filename}")
+    print("=" * 70)
+    
+    response = requests.get(f"{API_BASE}/training/shapefile/{filename}/labels")
+    if response.ok:
+        data = response.json()
+        print(f"✅ Success!")
+        print(f"  Filename: {data['filename']}")
+        print(f"  Points: {data['point_count']}")
+        print(f"  Label column: {data['label_column']}")
+        print(f"  Unique labels: {data['label_count']}")
+        print(f"  Bbox: {data['bbox']}")
+        print(f"\n  Labels distribution:")
+        for label in data['labels']:
+            mapped = "✅" if label['mapped'] else "⚠️"
+            print(f"    {mapped} {label['name']}: {label['count']} points (code: {label['code']})")
+    else:
+        print(f"❌ Error: {response.status_code}")
+        print(response.text)
+    print()
+
+def test_config_presets():
+    """Test /api/config/presets endpoint"""
+    print("=" * 70)
+    print("TEST 4: Getting config presets")
+    print("=" * 70)
+    
+    response = requests.get(f"{API_BASE}/config/presets")
+    if response.ok:
+        data = response.json()
+        print(f"✅ Success! Found {len(data['presets'])} presets:")
+        for preset in data['presets']:
+            print(f"\n  📋 {preset['name']}")
+            config = preset['config']
+            print(f"     Bbox: [{config['min_lon']}, {config['min_lat']}, {config['max_lon']}, {config['max_lat']}]")
+            print(f"     Time: {config['start_date']} → {config['end_date']}")
+            print(f"     Resolution: {config['resolution']}m")
+    else:
+        print(f"❌ Error: {response.status_code}")
+    print()
+
+if __name__ == "__main__":
+    print("\n" + "=" * 70)
+    print("🧪 TESTING TRAINING API ENDPOINTS")
+    print("=" * 70 + "\n")
+    
+    try:
+        test_training_labels()
+        test_training_files()
+        test_shapefile_labels()
+        test_config_presets()
+        
+        print("=" * 70)
+        print("✅ ALL TESTS COMPLETED!")
+        print("=" * 70)
+        
+    except requests.exceptions.ConnectionError:
+        print("\n❌ Error: Cannot connect to API server")
+        print("Make sure the server is running: python api_server.py")
+    except Exception as e:
+        print(f"\n❌ Error: {e}")
+        import traceback
+        traceback.print_exc()

.temp/test_unet.py

@@ -0,0 +1,5 @@
+from train_cloud_removal import UNet
+model = UNet(in_channels=6, out_channels=4)
+print(hasattr(model, 'inc'))
+print(hasattr(model, 'conv1'))
+print(list(model.parameters())[0].shape)

.temp/test_y_mapped.py

@@ -0,0 +1,13 @@
+import numpy as np
+
+y = []
+# simulate appending 1130 labels
+for i in range(1130):
+    y.append(i % 5)
+
+y = np.array(y)
+unique_labels = sorted(list(np.unique(y)))
+label_map = {lbl: i for i, lbl in enumerate(unique_labels)}
+y_mapped = np.array([label_map[l] for l in y])
+
+print(len(y), len(y_mapped))

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "python-envs.defaultEnvManager": "ms-python.python:conda",
+    "python-envs.defaultPackageManager": "ms-python.python:conda",
+    "python-envs.pythonProjects": []
+}

CLOUD_PROCESSING.md

@@ -0,0 +1,346 @@
+# Xử lý mây (Cloud Processing) — Hệ thống Land Classification
+
+Tài liệu chi tiết về các phương pháp xử lý mây cho dữ liệu Sentinel-2. Module độc lập `cloud_removal.py` cung cấp nhiều chiến lược có thể chọn.
+
+## Tổng quan
+
+Hệ thống cung cấp **7 phương pháp xử lý mây** khác nhau, từ cổ điển đến hiện đại (ML/DL):
+
+1. **Classic** - 3 bước cổ điển (temporal → median → spatial) - mặc định
+2. **Temporal Only** - Chỉ temporal interpolation (nhanh nhất)
+3. **Median Composite** - Ưu tiên median composite (giảm nhiễu tốt nhất)
+4. **ML KNN** - Machine Learning K-Nearest Neighbors inpainting
+5. **ML RF** - Machine Learning Random Forest inpainting
+6. **Deep Inpainting** - Deep Learning CNN inpainting (yêu cầu model)
+7. **Hybrid** - Kết hợp classical + ML (cân bằng tốc độ và chất lượng)
+
+---
+
+## Cách sử dụng
+
+### API Endpoint
+
+Lấy danh sách các methods:
+```bash
+GET /api/cloud-removal/methods
+```
+
+Response:
+```json
+{
+  "success": true,
+  "methods": {
+    "classic": "3-step classical: temporal → median → spatial (default, balanced)",
+    "temporal_only": "Temporal interpolation only (fastest, needs many scenes)",
+    "median_composite": "Median composite priority (best noise reduction)",
+    "ml_knn": "ML K-Nearest Neighbors inpainting (good quality, medium speed)",
+    "ml_rf": "ML Random Forest inpainting (high quality, slower)",
+    "deep": "Deep Learning CNN inpainting (best quality, requires model)",
+    "hybrid": "Hybrid classical + ML (balanced speed & quality)"
+  },
+  "default": "classic"
+}
+```
+
+### Config trong Prediction
+
+Thêm `cloud_removal_method` vào config:
+
+```python
+config = {
+    "model_filename": "model_odc.joblib",
+    "min_lon": 105.5,
+    "max_lon": 105.6,
+    "min_lat": 10.0,
+    "max_lat": 10.1,
+    "start_date": "2024-01-01",
+    "end_date": "2024-12-31",
+    "max_scenes": 12,
+    "cloud_cover": 30,
+    "resolution": 20,
+    "use_gpu": false,
+    "cloud_removal_method": "hybrid"  # Chọn method tại đây
+}
+```
+
+### Programmatic Usage
+
+```python
+from cloud_removal import process_cloud_removal
+
+# Load Sentinel-2 data with SCL band
+s2_data = load(...) 
+
+# Process clouds with selected method
+cleaned_data, metadata = process_cloud_removal(
+    s2_data=s2_data,
+    method="hybrid",  # or "classic", "ml_knn", etc.
+    verbose=True
+)
+
+print(f"Cloud coverage: {metadata['cloud_coverage_percent']:.1f}%")
+print(f"Steps applied: {metadata['steps_applied']}")
+```
+
+---
+
+## Chi tiết các phương pháp
+
+### 1. Classic (Mặc định)
+
+**Mô tả:** 3 bước cổ điển kết hợp temporal, median, và spatial interpolation.
+
+**Quy trình:**
+1. Temporal interpolation (ffill + bfill)
+2. Median compositing (nếu >= 3 scenes)
+3. Spatial interpolation (nearest neighbor)
+4. Fallback fillna(0)
+
+**Ưu điểm:**
+- Cân bằng tốc độ và chất lượng
+- Đã được test kỹ, ổn định
+- Phù hợp hầu hết trường hợp
+
+**Nhược điểm:**
+- Không tối ưu cho các gaps lớn
+- Có thể tạo artifacts ở biên
+
+**Khi nào dùng:** Default choice, phù hợp cho production
+
+---
+
+### 2. Temporal Only
+
+**Mô tả:** Chỉ sử dụng temporal interpolation (ffill + bfill).
+
+**Ưu điểm:**
+- Nhanh nhất
+- Giữ xu hướng thời gian tốt
+- Ít tạo artifacts
+
+**Nhược điểm:**
+- Yêu cầu nhiều time steps
+- Không xử lý được gaps liên tục
+- Chất lượng kém nếu ít scenes
+
+**Khi nào dùng:** Khi có nhiều scenes (>10) và cần tốc độ
+
+---
+
+### 3. Median Composite
+
+**Mô tả:** Ưu tiên median composite, sau đó spatial interpolation.
+
+**Ưu điểm:**
+- Giảm nhiễu tốt nhất
+- Chống outliers hiệu quả
+- Tạo composite trơn
+
+**Nhược điểm:**
+- Mất thông tin temporal
+- Yêu cầu >= 3 scenes
+- Chậm hơn temporal only
+
+**Khi nào dùng:** Khi cần giảm nhiễu, không quan tâm temporal dynamics
+
+---
+
+### 4. ML KNN Inpainting
+
+**Mô tả:** Sử dụng K-Nearest Neighbors để học từ pixels hợp lệ và dự đoán pixels bị mây.
+
+**Quy trình:**
+1. Xác định valid pixels (không có mây)
+2. Train KNN model với spatial coordinates + spectral values
+3. Predict invalid pixels
+4. Fill predictions vào dataset
+
+**Ưu điểm:**
+- Chất lượng cao hơn classical
+- Học spatial patterns
+- Không cần pretrained model
+
+**Nhược điểm:**
+- Chậm hơn classical
+- Yêu cầu đủ valid pixels (>10)
+- Tốn RAM nếu ảnh lớn
+
+**Hyperparameters:**
+- n_neighbors: 5
+- weights: 'distance'
+
+**Khi nào dùng:** Khi cần chất lượng cao và có đủ valid pixels
+
+---
+
+### 5. ML Random Forest Inpainting
+
+**Mô tả:** Sử dụng Random Forest để inpainting, tương tự KNN nhưng phức tạp hơn.
+
+**Ưu điểm:**
+- Chất lượng cao nhất trong ML methods
+- Xử lý non-linear patterns tốt
+- Robust với outliers
+
+**Nhược điểm:**
+- Chậm nhất trong ML methods
+- Tốn nhiều RAM
+- Có thể overfit với ít data
+
+**Hyperparameters:**
+- n_estimators: 10
+- max_depth: 10
+- n_jobs: -1 (parallel)
+
+**Khi nào dùng:** Khi cần chất lượng tối đa và không quan tâm tốc độ
+
+---
+
+### 6. Deep Inpainting (CNN)
+
+**Mô tả:** Sử dụng CNN autoencoder để reconstruct pixels bị mây.
+
+**Trạng thái:** **Đang phát triển** - yêu cầu pretrained model
+
+**Quy trình (planned):**
+1. Stack bands thành multi-channel image
+2. Tạo binary mask (1=cloud, 0=valid)
+3. Run through CNN autoencoder
+4. Blend predictions với valid pixels
+
+**Ưu điểm (khi có model):**
+- Chất lượng tốt nhất
+- Xử lý large gaps hiệu quả
+- Học global context
+
+**Nhược điểm:**
+- Yêu cầu pretrained model
+- Chậm nhất (GPU recommended)
+- Phức tạp để deploy
+
+**Khi nào dùng:** Khi có GPU và pretrained model, cần chất lượng tối đa
+
+---
+
+### 7. Hybrid (Khuyến nghị)
+
+**Mô tả:** Kết hợp classical + ML để cân bằng tốc độ và chất lượng.
+
+**Quy trình:**
+1. Temporal interpolation (nhanh)
+2. Check remaining NaN percentage
+3. Nếu > 5%: Apply ML KNN inpainting
+4. Nếu <= 5%: Apply spatial interpolation
+5. Fallback fillna(0)
+
+**Ưu điểm:**
+- Cân bằng tốc độ và chất lượng
+- Adaptive - chỉ dùng ML khi cần
+- Hiệu quả với mọi cloud coverage
+
+**Nhược điểm:**
+- Phức tạp hơn classic
+- Khó debug
+
+**Khi nào dùng:** **Khuyến nghị cho production** - tự động chọn strategy phù hợp
+
+---
+
+## So sánh Performance
+
+| Method | Tốc độ | Chất lượng | RAM | Yêu cầu |
+|--------|--------|------------|-----|---------|
+| classic | ⭐⭐⭐⭐ | ⭐⭐⭐ | Thấp | Không |
+| temporal_only | ⭐⭐⭐⭐⭐ | ⭐⭐ | Thấp | Nhiều scenes |
+| median_composite | ⭐⭐⭐ | ⭐⭐⭐⭐ | Thấp | >= 3 scenes |
+| ml_knn | ⭐⭐ | ⭐⭐⭐⭐ | Trung bình | Đủ valid pixels |
+| ml_rf | ⭐ | ⭐⭐⭐⭐⭐ | Cao | Đủ valid pixels |
+| deep | ⭐ | ⭐⭐⭐⭐⭐ | Rất cao | Pretrained model + GPU |
+| hybrid | ⭐⭐⭐ | ⭐⭐⭐⭐ | Trung bình | Không |
+
+---
+
+## Phát hiện mây (SCL)
+
+Tất cả methods đều sử dụng SCL (Scene Classification Layer):
+
+```python
+# SCL values:
+# 0: No data, 1: Saturated/Defective, 2: Dark Area Pixels
+# 3: Cloud shadows, 4: Vegetation, 5: Not vegetated, 6: Water
+# 7: Unclassified, 8: Cloud medium probability, 9: Cloud high probability
+# 10: Thin cirrus, 11: Snow/Ice
+
+cloud_mask = (scl == 3) | (scl == 8) | (scl == 9) | (scl == 10) | (scl == 11)
+invalid_mask = (scl == 0) | (scl == 1)
+full_mask = cloud_mask | invalid_mask
+```
+
+---
+
+## Testing & Comparison
+
+So sánh nhiều methods trên cùng dữ liệu:
+
+```python
+from cloud_removal import compare_methods
+
+results = compare_methods(
+    s2_data=s2_data,
+    methods=["classic", "temporal_only", "ml_knn", "hybrid"]
+)
+
+for method, result in results.items():
+    print(f"{method}: {result['remaining_nan_percent']:.2f}% NaN remaining")
+```
+
+---
+
+## Khuyến nghị sử dụng
+
+### Production (General)
+```
+cloud_removal_method: "hybrid"
+```
+- Cân bằng tốc độ và chất lượng
+- Adaptive theo cloud coverage
+
+### High Quality (Research)
+```
+cloud_removal_method: "ml_rf"
+```
+- Chất lượng tối đa
+- Chấp nhận tốc độ chậm
+
+### Fast Processing (Monitoring)
+```
+cloud_removal_method: "temporal_only"
+```
+- Cần nhiều scenes (>10)
+- Ưu tiên tốc độ
+
+### Low Cloud Coverage (<10%)
+```
+cloud_removal_method: "classic"
+```
+- Đơn giản, hiệu quả
+- Ổn định, đã test kỹ
+
+---
+
+## Vị trí code
+
+- **Module:** `cloud_removal.py` - Standalone cloud removal module
+- **API Integration:** `api_server.py` - API endpoints và config
+- **Documentation:** `CLOUD_PROCESSING.md` - Tài liệu này
+
+---
+
+## Phát triển tiếp
+
+- [ ] Implement CNN autoencoder cho deep inpainting
+- [ ] Add quality scoring system
+- [ ] Optimize ML methods với Dask
+- [ ] Add weighted temporal interpolation
+- [ ] Support custom ML models
+

CLOUD_REMOVAL_UPLOAD_GUIDE.md

@@ -0,0 +1,200 @@
+# Cloud Removal Model Upload Feature
+
+## Overview
+Added functionality to upload and use custom deep learning cloud removal models (.pth files) during prediction.
+
+## Features Implemented
+
+### 1. API Endpoints
+
+#### Upload Cloud Removal Model
+```
+POST /api/cloud-removal/upload
+```
+- Upload `.pth` cloud removal model files
+- Validates file extension (.pth only)
+- Security checks for filename
+- Returns file info (name, size)
+
+**Example:**
+```bash
+curl -X POST -F "file=@cloud_removal_unet_best.pth" \
+  http://localhost:8000/api/cloud-removal/upload
+```
+
+#### List Cloud Removal Models
+```
+GET /api/cloud-removal/models
+```
+Already existing - lists all `.pth` models in `model_train/` directory
+
+#### Delete Cloud Removal Model
+```
+DELETE /api/cloud-removal/models/{filename}
+```
+Already existing - deletes a specific cloud removal model
+
+### 2. Prediction Configuration Updates
+
+#### PredictionConfig
+Added new optional field:
+```python
+cloud_removal_model: Optional[str] = None  # .pth filename
+```
+
+#### PredictionWithNDVIConfig  
+Added new optional field:
+```python
+cloud_removal_model: Optional[str] = None  # .pth filename
+```
+
+### 3. Prediction Function Integration
+
+The `run_prediction()` function now:
+1. Accepts `cloud_removal_model` parameter
+2. Passes model path to `process_cloud_removal()`
+3. Logs which model is being used
+
+**Code:**
+```python
+cloud_removal_method = config.cloud_removal_method
+cloud_removal_model = config.cloud_removal_model
+
+s2_data, cloud_metadata = process_cloud_removal(
+    s2_data=s2_data,
+    method=cloud_removal_method,
+    model_path=f"model_train/{cloud_removal_model}" if cloud_removal_model else None,
+    verbose=True
+)
+```
+
+### 4. Web Interface Updates
+
+#### Upload Button
+- Added file input in "Deep Learning" cloud removal section
+- Upload button appears when "Deep Learning" method is selected
+- Real-time upload status feedback
+- Auto-refreshes model list after successful upload
+
+#### Model Selection
+- Dropdown shows all available `.pth` models
+- Auto-selects newly uploaded model
+- Shows model metadata (epoch, loss)
+
+## Usage Guide
+
+### Step 1: Train or Obtain a Cloud Removal Model
+Train using the cloud training interface or obtain a pre-trained `.pth` model.
+
+### Step 2: Upload Model
+1. Go to Prediction Interface
+2. Scroll to "Cloud Removal Method" section
+3. Select "Deep Learning (U-Net)" from dropdown
+4. Model upload section appears
+5. Click "📤 Upload Cloud Removal Model (.pth)"
+6. Select your `.pth` file
+7. Wait for upload confirmation
+
+### Step 3: Use Model in Prediction
+1. The uploaded model is automatically selected
+2. Configure other prediction parameters (bbox, dates, etc.)
+3. Click "🚀 Start Prediction (với NDVI)"
+4. The system will use your custom model for cloud removal
+
+## File Structure
+```
+model_train/
+├── cloud_removal_unet_best.pth         # User uploaded
+├── cloud_removal_unet_epoch_10.pth     # User uploaded
+├── model_mobilenet-lraspp_*.joblib     # Land classification models
+└── ...
+```
+
+## API Request Example
+
+### Using Uploaded Model
+```json
+{
+  "model_filename": "model_mobilenet-lraspp_20260105_225459.joblib",
+  "min_lon": 105.80,
+  "min_lat": 10.00,
+  "max_lon": 105.82,
+  "max_lat": 10.02,
+  "start_date": "2024-01-15",
+  "end_date": "2024-01-17",
+  "max_scenes": 3,
+  "cloud_cover": 30,
+  "resolution": 20,
+  "use_gpu": true,
+  "export_ndvi": true,
+  "export_classification": true,
+  "cloud_removal_method": "deep",
+  "cloud_removal_model": "cloud_removal_unet_best.pth"
+}
+```
+
+### Without Custom Model (Classical Methods)
+```json
+{
+  ...
+  "cloud_removal_method": "hybrid",
+  "cloud_removal_model": null
+}
+```
+
+## Security Features
+- Filename validation (no path traversal)
+- File extension validation (.pth only)
+- File existence checks
+- Duplicate filename detection
+
+## Error Handling
+- Invalid file type → 400 Bad Request
+- Duplicate filename → 400 Bad Request  
+- Upload failure → 500 Internal Server Error
+- Missing model when "deep" selected → Falls back to "hybrid" method
+
+## Notes
+- Uploaded models are stored in `model_train/` directory
+- Models must be PyTorch `.pth` files
+- Compatible with `cloud_removal.py` module
+- Works with both `/api/prediction/start` and `/api/predict/with-ndvi` endpoints
+
+## Testing
+
+### Test Upload
+```bash
+# Upload a model
+curl -X POST -F "file=@my_cloud_model.pth" \
+  http://localhost:8000/api/cloud-removal/upload
+
+# List models
+curl http://localhost:8000/api/cloud-removal/models
+
+# Delete model
+curl -X DELETE \
+  http://localhost:8000/api/cloud-removal/models/my_cloud_model.pth
+```
+
+### Test Prediction
+```bash
+curl -X POST http://localhost:8000/api/predict/with-ndvi \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model_filename": "model_mobilenet-lraspp_20260105_225459.joblib",
+    "min_lon": 105.80, "min_lat": 10.00,
+    "max_lon": 105.82, "max_lat": 10.02,
+    "start_date": "2024-01-15", "end_date": "2024-01-17",
+    "max_scenes": 2, "cloud_cover": 30, "resolution": 20,
+    "use_gpu": false, "export_ndvi": true,
+    "cloud_removal_method": "deep",
+    "cloud_removal_model": "cloud_removal_unet_best.pth"
+  }'
+```
+
+## Future Enhancements
+- Model metadata display (architecture, training date)
+- Model validation on upload
+- Multiple model format support (.pt, .onnx)
+- Model performance metrics
+- Batch upload support