refactor: reorganize project structure by moving core modules and update import paths in API server

update attribute table

.gitignore

@@ -76,9 +76,11 @@ model_train/*.feather
 model_train/*.db
 model_train/*.sqlite
 model_train/*.log
+cloud_removal_model/
 
 # VSCode settings
 .vscode/
 
 # Jupyter checkpoints
 .ipynb_checkpoints/
+reports/

.stfolder/syncthing-folder-a35f20.txt

@@ -1,5 +0,0 @@
-# This directory is a Syncthing folder marker.
-# Do not delete.
-
-folderID: rs-data
-created: 2026-04-03T11:35:19+07:00

.temp/01.train_ODC_XGBoost.py

@@ -0,0 +1,171 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+get_ipython().run_cell_magic('time', '', '%matplotlib inline\n\nimport importlib\nimport new_import_ODC  \n\nimportlib.reload(new_import_ODC)\n\nfrom new_import_ODC import *\n')
+
+
+# In[2]:
+
+
+get_ipython().run_cell_magic('time', '', '# Cấu hình Daskgateway\ncluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1, 10))\n# Khai báo 1 Datacube là dc\ndc = None\n\n# Cấu hình truy cập dịch vụ S3\nconfigure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n\nclient\n')
+
+
+# In[3]:
+
+
+## cấu hình thời gian lấy ảnh và tọa độ
+date_range = ("2022-09-01", "2022-10-01")
+longtitude_range = (105.86, 105.94)
+latitude_range = (9.65, 9.69)
+
+coordinates = (longtitude_range, latitude_range)
+
+
+# In[4]:
+
+
+## truy vấn ảnh vệ tinh sen2
+data = load_data(None, date_range, longtitude_range, latitude_range)
+notebook_utils.heading(notebook_utils.xarray_object_size(data))
+display(data)
+
+
+# In[5]:
+
+
+get_ipython().run_cell_magic('time', '', '# Tiến hành loại bỏ các vị trí bị mây ảnh hưởng\nresult = mask_clean(data)\n# progress(result)\n')
+
+
+# In[6]:
+
+
+# Tiến hành tính toán NDVI
+ds1 = calculate_indices(result, index="NDVI", satellite_mission="s2")
+ndvi = ds1["NDVI"]
+display(ndvi)
+
+
+# In[7]:
+
+
+## Hiển thị ảnh NDVI chưa điền các giá trị mây (chưa fill nan)
+plt.imshow(ndvi.isel(time=0))
+
+
+# In[8]:
+
+
+# Thiết lập giá trị trung bình mùa vụ để xử lý các điểm ảnh bị mây dựa vào sự thay đổi theo mùa
+time_split = [
+    slice("2022-09-01", "2023-01-01"),
+    slice("2023-01-01", "2023-05-01"),
+    slice("2023-05-01", "2023-07-01"),
+    slice("2023-07-01", "2022-10-01"),
+]
+
+# Điền mây ở các vị trí mang giá trị nan (fill nan)
+fill_nan_ndvi = fill_nan(ndvi, time_split)
+
+# In kết quả ảnh NDVI đã điền mây (đã fill nan)
+plt.imshow(fill_nan_ndvi.isel(time=0))
+
+
+# In[9]:
+
+
+get_ipython().run_cell_magic('time', '', '## tính ndvi theo tháng\naverage_ndvi = fill_nan_ndvi.resample(time="1M").mean().persist()\n# progress(average_ndvi)\n\n# compute average_ndvi\naverage_ndvi = average_ndvi.compute()\n')
+
+
+# In[10]:
+
+
+#Load dữ liệu ảnh Sentinel 1
+dsvh, dsvv = load_data_sen1(None, date_range, coordinates)
+average_vv = calculate_average(dsvv, time_pattern='1M')
+average_vh = calculate_average(dsvh, time_pattern='1M')
+
+
+# In[11]:
+
+
+## cấu hình bộ dữ liệu điểm huấn luyện mô hình (train file)
+train_path = "train/ST_training_data_updated_1130points_new.shp"  # đường dẫn shp file train
+
+## load dữ liệu điểm huấn luyện mô hình (train file)
+train = load_train_data(train_path)
+train.head()
+
+# cấu hình nhãn dữ liệu 
+label_mapping = {
+    "Lua tom": "0",
+    "Lua": "1",
+    "CHN": "2",
+    "CLN": "3",
+    "TS": "4",
+    "Song": "5",
+    "Dat xay dung": "6",
+    "Rung": "7",
+}
+
+# xây dựng tập dữ liệu (dataset) chứa dữ liệu VH, VV, NDVI
+datasets = get_data_sen1_and_sen2(train, average_ndvi, average_vh, average_vv)
+
+# chia tập dữ liệu thành các phần theo tỉ lệ 80(80-20)-20 tương ứng với tập train, validate, test
+X_train, X_val, X_test, y_train, y_val, y_test = split_train_data(
+    train, label_mapping, datasets
+)
+
+
+# In[ ]:
+
+
+get_ipython().run_cell_magic('time', '', '# Import XGBoost\nimport xgboost as xgb\nfrom sklearn.metrics import accuracy_score\nimport numpy as np\n\n# Convert to numpy arrays\nX_train_np = np.asarray(X_train, dtype=np.float32)\nX_val_np = np.asarray(X_val, dtype=np.float32)\ny_train_np = np.asarray(y_train, dtype=np.int32)\ny_val_np = np.asarray(y_val, dtype=np.int32)\n\nprint("🚀 Training XGBoost model...")\nprint(f"   Train samples: {len(X_train_np)}")\nprint(f"   Val samples: {len(X_val_np)}")\nprint(f"   Features: {X_train_np.shape[1]}")\nprint(f"   Classes: 8\\n")\n\n# XGBoost parameters\nparams = {\n    \'objective\': \'multi:softmax\',  # Multi-class classification\n    \'num_class\': 8,                 # 8 land use classes\n    \'max_depth\': 6,                 # Maximum tree depth\n    \'learning_rate\': 0.1,           # Learning rate\n    \'n_estimators\': 200,            # Number of trees\n    \'subsample\': 0.8,               # Subsample ratio\n    \'colsample_bytree\': 0.8,        # Feature sampling ratio\n    \'random_state\': 42,\n    \'n_jobs\': -1,                   # Use all CPU cores\n    \'eval_metric\': \'mlogloss\'       # Multi-class log loss\n}\n\n# Train XGBoost model\nmodel = xgb.XGBClassifier(**params)\n\nmodel.fit(\n    X_train_np, y_train_np,\n    eval_set=[(X_train_np, y_train_np), (X_val_np, y_val_np)],\n    verbose=True\n)\n\n# Validation accuracy\ny_val_pred = model.predict(X_val_np)\nval_accuracy = accuracy_score(y_val_np, y_val_pred)\nprint(f"\\n✅ Training completed!")\nprint(f"   Validation Accuracy: {val_accuracy:.4f} ({val_accuracy*100:.2f}%)")\n')
+
+
+# In[ ]:
+
+
+get_ipython().run_cell_magic('time', '', '# Evaluate on test set\nX_test_np = np.asarray(X_test, dtype=np.float32)\ny_test_np = np.asarray(y_test, dtype=np.int32)\n\nprint("📊 Evaluating XGBoost model on test set...\\n")\n\n# Predictions\ny_pred_test = model.predict(X_test_np)\n\n# Metrics\nfrom sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, confusion_matrix\n\ntest_accuracy = accuracy_score(y_test_np, y_pred_test)\nprecision = precision_score(y_test_np, y_pred_test, average=\'weighted\', zero_division=0)\nrecall = recall_score(y_test_np, y_pred_test, average=\'weighted\', zero_division=0)\nf1 = f1_score(y_test_np, y_pred_test, average=\'weighted\', zero_division=0)\n\nprint(f"📈 Test Results:")\nprint(f"   Accuracy:  {test_accuracy:.4f} ({test_accuracy*100:.2f}%)")\nprint(f"   Precision: {precision:.4f}")\nprint(f"   Recall:    {recall:.4f}")\nprint(f"   F1-Score:  {f1:.4f}\\n")\n\n# Confusion Matrix\nfrom sklearn.metrics import ConfusionMatrixDisplay\nimport matplotlib.pyplot as plt\n\n# Create figure first\nfig, ax = plt.subplots(figsize=(10, 8))\n\nclass_names = list(label_mapping.keys())\ncm = confusion_matrix(y_test_np, y_pred_test)\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=class_names)\ndisp.plot(cmap=\'Blues\', ax=ax)\nplt.xticks(rotation=45, ha=\'right\')\nplt.title(\'XGBoost Confusion Matrix\')\nplt.tight_layout()\nplt.show()\n')
+
+
+# In[ ]:
+
+
+# Lưu mô hình huấn luyện
+import json
+import joblib
+
+# Save XGBoost model
+model_path = "model_xgboost.joblib"
+joblib.dump(model, model_path)
+print(f"✅ Model saved to {model_path}")
+
+# Save model info
+info = {
+    "model_type": "XGBoost",
+    "num_classes": 8,
+    "classes": list(label_mapping.keys()),
+    "num_features": X_train_np.shape[1],
+    "params": params,
+    "accuracy": float(test_accuracy),
+    "precision": float(precision),
+    "recall": float(recall),
+    "f1_score": float(f1),
+}
+
+with open("model_xgboost_info.json", "w") as f:
+    json.dump(info, f, indent=2)
+
+print(f"✅ Model info saved to model_xgboost_info.json")
+
+
+# In[15]:
+
+
+# đóng client, cluster
+# client.close()
+# cluster.close()
+

.temp/01.train_ODC_local_with_Mic_supplyer.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[6]:
+
+
+get_ipython().run_cell_magic('time', '', '%matplotlib inline\n\n# Import Microsoft Planetary Computer libraries\nimport planetary_computer\nfrom pystac_client import Client\nfrom odc.stac import load as stac_load\n\n# Standard imports\nimport xarray as xr\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score, classification_report, confusion_matrix, ConfusionMatrixDisplay\nimport geopandas as gpd\n\n# XGBoost for GPU training\nimport xgboost as xgb\n\nfrom xgboost import XGBClassifier\n\nprint(f"   XGBoost version: {xgb.__version__}")\n\nprint("✅ All modules loaded successfully")\n')
+
+
+# In[7]:
+
+
+get_ipython().run_cell_magic('time', '', '# Kết nối tới Microsoft Planetary Computer STAC\nfrom pystac_client import Client\n\n# KHÔNG dùng modifier ở catalog level để tránh items bị convert thành dict\ncatalog = Client.open(\n    "https://planetarycomputer.microsoft.com/api/stac/v1"\n)\nprint("✅ Connected to Microsoft Planetary Computer")\n\nprint("\\n" + "="*70)\n')
+
+
+# In[8]:
+
+
+get_ipython().run_cell_magic('time', '', '# 🌍 Định nghĩa khu vực và thời gian\nprint("="*70)\nprint("CONFIGURATION")\nprint("="*70)\n\n# Khu vực quan tâm (Vietnam - Mekong Delta) - GIẢM DIỆN TÍCH ~40%\nbbox = [105.6, 9.3, 106.2, 9.8]  # [min_lon, min_lat, max_lon, max_lat]\n\n# GIẢM THỜI GIAN xuống 3 tháng để giảm kích thước dữ liệu cho PC\ntime_range = "2023-03-01/2023-05-31"  # 3 tháng (mùa khô)\n\nprint(f"\\n📍 Area of Interest:")\nprint(f"   Longitude: {bbox[0]} to {bbox[2]}")\nprint(f"   Latitude: {bbox[1]} to {bbox[3]}")\nprint(f"\\n📅 Time Range: {time_range}")\nprint(f"   ⚠️  Optimized for personal computer (3 months, reduced area)")\nprint(f"\\n🗺️  CRS: EPSG:32648")\nprint(f"   Resolution: 20m (reduced from 10m for smaller data size)")\n\nprint("="*70)\n')
+
+
+# In[9]:
+
+
+get_ipython().run_cell_magic('time', '', '# 📡 LOAD SENTINEL-2 FROM MICROSOFT PLANETARY COMPUTER\nprint("="*70)\nprint("LOADING SENTINEL-2 L2A")\nprint("="*70)\n\nprint("\\n🔍 Searching for Sentinel-2 scenes...")\nquery_s2 = catalog.search(\n    collections=["sentinel-2-l2a"],\n    bbox=bbox,\n    datetime=time_range,\n    query={"eo:cloud_cover": {"lt": 30}}  # Cloud cover < 30% (giảm từ 50%)\n)\n\nitems_s2 = list(query_s2.item_collection())\nprint(f"✅ Found {len(items_s2)} Sentinel-2 scenes")\n\n# GIỚI HẠN SỐ LƯỢNG SCENES cho PC cá nhân\nmax_scenes = 12  # Giảm xuống 12 scenes để tối ưu cho PC\nif len(items_s2) > max_scenes:\n    print(f"⚠️  Limiting to {max_scenes} scenes for personal computer")\n    # Chọn scenes đều đặn trong khoảng thời gian\n    step = len(items_s2) // max_scenes\n    items_s2 = items_s2[::step][:max_scenes]\n    print(f"   Selected {len(items_s2)} scenes evenly distributed")\n\nif len(items_s2) > 0:\n    # Show first few scenes\n    print(f"\\n📋 Sample scenes:")\n    for i, item in enumerate(items_s2[:5]):\n        date = item.datetime.strftime("%Y-%m-%d")\n        cloud = item.properties.get("eo:cloud_cover", "N/A")\n        print(f"   [{i+1}] {date} - Cloud: {cloud}%")\n    \n    # Re-sign items to ensure fresh URLs (keep as pystac objects)\n    print(f"\\n🔑 Signing STAC items...")\n    items_s2 = [planetary_computer.sign(item) for item in items_s2]\n    \n    # Load Sentinel-2 data (without Dask chunks)\n    print(f"\\n⏳ Loading Sentinel-2 data...")\n    ds_s2 = stac_load(\n        items_s2,\n        bands=["B04", "B08", "SCL"],  # Red (B04), NIR (B08), Scene Classification (SCL)\n        crs="EPSG:32648",\n        resolution=20,  # 20m resolution (4x smaller data than 10m)\n        bbox=bbox,\n        patch_url=planetary_computer.sign,  # Re-sign URLs during loading\n        fail_on_error=False,  # Skip problematic tiles instead of crashing\n    )\n    \n    # Rename bands to simpler names\n    ds_s2 = ds_s2.rename({"B04": "red", "B08": "nir", "SCL": "scl"})\n    \n    print(f"\\n✅ Sentinel-2 loaded!")\n    print(f"   Shape: {dict(ds_s2.dims)}")\n    print(f"   Variables: {list(ds_s2.data_vars)}")\n    display(ds_s2)\nelse:\n    print(f"❌ No Sentinel-2 scenes found")\n\n    ds_s2 = Noneprint("="*70)\n')
+
+
+# In[10]:
+
+
+get_ipython().run_cell_magic('time', '', '# 📡 LOAD SENTINEL-1 FROM MICROSOFT PLANETARY COMPUTER\nprint("="*70)\nprint("LOADING SENTINEL-1 RTC")\nprint("="*70)\n\nprint("\\n🔍 Searching for Sentinel-1 scenes...")\nquery_s1 = catalog.search(\n    collections=["sentinel-1-rtc"],\n    bbox=bbox,\n    datetime=time_range,\n)\n\nitems_s1 = list(query_s1.item_collection())\nprint(f"✅ Found {len(items_s1)} Sentinel-1 scenes")\n\n# GIỚI HẠN SỐ LƯỢNG SCENES cho PC cá nhân\nmax_scenes = 12  # Giảm xuống 12 scenes để tối ưu cho PC\nif len(items_s1) > max_scenes:\n    print(f"⚠️  Limiting to {max_scenes} scenes for personal computer")\n    # Chọn scenes đều đặn trong khoảng thời gian\n    step = len(items_s1) // max_scenes\n    items_s1 = items_s1[::step][:max_scenes]\n    print(f"   Selected {len(items_s1)} scenes evenly distributed")\n\nif len(items_s1) > 0:\n    # Show first few scenes\n    print(f"\\n📋 Sample scenes:")\n    for i, item in enumerate(items_s1[:5]):\n        date = item.datetime.strftime("%Y-%m-%d")\n        orbit = item.properties.get("sat:orbit_state", "N/A")\n        print(f"   [{i+1}] {date} - Orbit: {orbit}")\n    \n    # Re-sign items to ensure fresh URLs (keep as pystac objects)\n    print(f"\\n🔑 Signing STAC items...")\n    items_s1 = [planetary_computer.sign(item) for item in items_s1]\n    \n    # Load Sentinel-1 data (without Dask chunks)\n    print(f"\\n⏳ Loading Sentinel-1 data...")\n    ds_s1 = stac_load(\n        items_s1,\n        bands=["vv", "vh"],  # VV and VH polarizations\n        crs="EPSG:32648",\n        resolution=20,  # 20m resolution (4x smaller data than 10m)\n        bbox=bbox,\n        patch_url=planetary_computer.sign,  # Re-sign URLs during loading\n        fail_on_error=False,  # Skip problematic tiles instead of crashing\n    )\n    \n    # Convert to dB (Microsoft S1 is in linear power)\n    print(f"\\n🔄 Converting to dB...")\n    ds_s1[\'vv_db\'] = 10 * np.log10(ds_s1[\'vv\'].where(ds_s1[\'vv\'] > 0))\n    ds_s1[\'vh_db\'] = 10 * np.log10(ds_s1[\'vh\'].where(ds_s1[\'vh\'] > 0))\n    \n    print(f"\\n✅ Sentinel-1 loaded!")\n    print(f"   Shape: {dict(ds_s1.dims)}")\n    print(f"   Variables: {list(ds_s1.data_vars)}")\n    display(ds_s1)\nelse:\n    print(f"❌ No Sentinel-1 scenes found")\n\n    ds_s1 = Noneprint("="*70)\n')
+
+
+# In[11]:
+
+
+get_ipython().run_cell_magic('time', '', '# 🌿 CALCULATE NDVI AND PROCESS DATA\nprint("="*70)\nprint("DATA PROCESSING")\nprint("="*70)\n\nif ds_s2 is not None:\n    print("\\n[1] Calculating NDVI...")\n    # NDVI = (NIR - Red) / (NIR + Red)\n    ndvi = (ds_s2[\'nir\'] - ds_s2[\'red\']) / (ds_s2[\'nir\'] + ds_s2[\'red\'] + 1e-8)\n    \n    print(f"✅ NDVI calculated")\n    print(f"   Shape: {ndvi.shape}")\n    print(f"   Time steps: {len(ndvi.time)}")\n    \n    # Cloud masking using SCL band\n    print(f"\\n[2] Applying cloud mask...")\n    # SCL values: 1=defective, 3=cloud shadow, 8=cloud medium, 9=cloud high, 10=cirrus\n    cloud_mask = ds_s2[\'scl\'].isin([1, 3, 8, 9, 10])\n    ndvi_masked = ndvi.where(~cloud_mask)\n    \n    print(f"✅ Cloud mask applied")\n    \n    # Temporal aggregation (mean over time)\n    print(f"\\n[3] Computing mean NDVI across time...")\n    ndvi_mean = ndvi_masked.mean(dim=\'time\')\n    \n    # Data already in memory, no need to compute() again\n    print(f"✅ Mean NDVI computed")\n    print(f"   Shape: {ndvi_mean.shape}")\n    \nelse:\n    print("❌ No Sentinel-2 data to process")\n    ndvi_mean = None\n\nprint("="*70)\n')
+
+
+# In[ ]:
+
+
+get_ipython().run_cell_magic('time', '', '# 🎯 EXTRACT TRAINING DATA FEATURES\nprint("="*70)\nprint("FEATURE EXTRACTION")\nprint("="*70)\n\n# Check if required data is available\nif \'ndvi_mean\' not in globals() or \'ds_s1\' not in globals():\n    print("❌ Error: Please run Cell 6 (DATA PROCESSING) first!")\n    print("   Required variables: ndvi_mean, ds_s1")\n    raise RuntimeError("Missing required data. Run cells in order: Cell 4 → Cell 5 → Cell 6 → Cell 7")\n\n# Load training shapefile\nimport geopandas as gpd\n\ntrain_path = \'train/ST_training data_updated_1130points_new.shp\'\nprint(f"\\n[1] Loading training data from: {train_path}")\ntrain_gdf = gpd.read_file(train_path)\n\n# Ensure CRS matches\nif train_gdf.crs != \'EPSG:32648\':\n    print(f"   Reprojecting from {train_gdf.crs} to EPSG:32648...")\n    train_gdf = train_gdf.to_crs(\'EPSG:32648\')\n\nprint(f"✅ Loaded {len(train_gdf)} training points")\nprint(f"   Available columns: {list(train_gdf.columns)}")\n\n# Auto-detect label column (look for common names)\nlabel_column = None\nfor col in [\'HT_code\', \'Ma_LU\', \'LU2022\', \'class\', \'Class\', \'CLASS\', \'label\', \'Label\', \'LABEL\', \'LU_CODE\', \'LU_code\']:\n    if col in train_gdf.columns:\n        label_column = col\n        break\n\nif label_column is None:\n    print(f"❌ Cannot find label column. Available columns: {list(train_gdf.columns)}")\n    print(f"   Please check your shapefile and update the code.")\nelse:\n    print(f"   Using label column: \'{label_column}\'")\n    print(f"   Classes: {sorted(train_gdf[label_column].unique())}")\n    \n    # Extract features at each training point\n    print(f"\\n[2] Extracting features at training points...")\n    \n    features = []\n    labels = []\n    skipped = 0\n    \n    for idx, row in train_gdf.iterrows():\n        point = row.geometrychro\n        x_coord = point.x\n        y_coord = point.y\n        label = row[label_column]\n        \n        # Extract NDVI at this location\n        if ndvi_mean is not None and ds_s1 is not None:\n            try:\n                ndvi_val = ndvi_mean.sel(x=x_coord, y=y_coord, method=\'nearest\').values\n                \n                # Extract Sentinel-1 VH/VV at this location (mean across time)\n                # Data already in memory, no need to compute()\n                vh_val = ds_s1[\'vh_db\'].sel(x=x_coord, y=y_coord, method=\'nearest\').mean(dim=\'time\').values\n                vv_val = ds_s1[\'vv_db\'].sel(x=x_coord, y=y_coord, method=\'nearest\').mean(dim=\'time\').values\n                \n                # Create feature vector: [NDVI, VH_dB, VV_dB]\n                feature_vec = [ndvi_val, vh_val, vv_val]\n                \n                # Only add if all features are valid (not NaN)\n                if not np.isnan(feature_vec).any():\n                    features.append(feature_vec)\n                    labels.append(label)\n                else:\n                    skipped += 1\n            except Exception as e:\n                # Skip points outside the data extent\n                skipped += 1\n                continue\n    \n    features = np.array(features)\n    labels = np.array(labels)\n    \n    print(f"✅ Extracted features for {len(features)} valid points")\n    print(f"   Skipped {skipped} points (outside extent or NaN values)")\n    print(f"   Feature shape: {features.shape}")\n    print(f"   Feature names: [\'NDVI_mean\', \'VH_dB_mean\', \'VV_dB_mean\']")\n    print(f"\\n   Class distribution:")\n    unique, counts = np.unique(labels, return_counts=True)\n    for cls, cnt in zip(unique, counts):\n        print(f"      Class {cls}: {cnt} samples ({cnt/len(labels)*100:.1f}%)")\n\nprint("="*70)\n')
+
+
+# In[21]:
+
+
+get_ipython().run_cell_magic('time', '', '# 🤖 TRAIN XGBOOST MODEL ON GPU (RTX 4060)\nprint("="*70)\nprint("MODEL TRAINING - GPU ACCELERATED")\nprint("="*70)\n\nfrom xgboost import XGBClassifier\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import classification_report, confusion_matrix, ConfusionMatrixDisplay\nimport matplotlib.pyplot as plt\n\n# Encode labels to ensure they are 0, 1, 2, ... n-1\nprint("\\n[1] Encoding labels...")\nlabel_encoder = LabelEncoder()\nlabels_encoded = label_encoder.fit_transform(labels)\nprint(f"✅ Original classes: {label_encoder.classes_}")\nprint(f"   Encoded as: {np.unique(labels_encoded)}")\n\n# Split data\nprint("\\n[2] Splitting data (80% train, 20% test)...")\nX_train, X_test, y_train, y_test = train_test_split(\n    features, labels_encoded, test_size=0.2, random_state=42, stratify=labels_encoded\n)\nprint(f"✅ Training samples: {len(X_train)}")\nprint(f"   Testing samples: {len(X_test)}")\n\n# Train XGBoost on GPU\nprint("\\n[3] Training XGBoost classifier on RTX 4060 GPU...")\nprint("   GPU Settings: device=\'cuda:0\'")\n\nxgb_model = XGBClassifier(\n    n_estimators=100,\n    max_depth=20,\n    learning_rate=0.1,\n    device=\'cuda:0\',           # Use GPU (updated from deprecated gpu_id)\n    tree_method=\'hist\',        # Use hist with device for GPU training\n    random_state=42,\n    eval_metric=\'mlogloss\',    # Multi-class log loss\n    verbosity=1                # Show GPU training progress\n)\n\nxgb_model.fit(X_train, y_train)\nprint(f"✅ Model trained on GPU")\n\n# Evaluate\nprint("\\n[4] Evaluating model...")\ntrain_score = xgb_model.score(X_train, y_train)\ntest_score = xgb_model.score(X_test, y_test)\nprint(f"✅ Training accuracy: {train_score:.4f}")\nprint(f"   Testing accuracy: {test_score:.4f}")\n\n# Classification report\nprint("\\n[5] Classification Report:")\ny_pred = xgb_model.predict(X_test)\nprint(classification_report(y_test, y_pred, target_names=[str(c) for c in label_encoder.classes_]))\n\n# Confusion matrix\nprint("\\n[6] Confusion Matrix:")\nfig, ax = plt.subplots(figsize=(10, 8))\ncm = confusion_matrix(y_test, y_pred)\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=label_encoder.classes_)\ndisp.plot(ax=ax, cmap=\'Blues\', values_format=\'d\')\nplt.title(\'Confusion Matrix - XGBoost GPU Model (RTX 4060)\')\nplt.tight_layout()\nplt.show()\n\nprint("="*70)\n')
+
+
+# In[23]:
+
+
+get_ipython().run_cell_magic('time', '', '# 💾 SAVE MODEL AND CLEANUP\nprint("="*70)\nprint("SAVING MODEL & CLEANUP")\nprint("="*70)\n\nimport joblib\nfrom datetime import datetime\n\n# Save model and label encoder\nmodel_filename = f"model_train/model_xgboost_gpu_{datetime.now().strftime(\'%Y%m%d_%H%M%S\')}.joblib"\nprint(f"\\n[1] Saving model to: {model_filename}")\njoblib.dump({\'model\': xgb_model, \'label_encoder\': label_encoder}, model_filename)\nprint(f"✅ Model and label encoder saved")\n\n# Save model info\ninfo = {\n    "timestamp": datetime.now().isoformat(),\n    "data_source": "Microsoft Planetary Computer STAC",\n    "collections": ["sentinel-2-l2a", "sentinel-1-rtc"],\n    "features": ["NDVI_mean", "VH_dB_mean", "VV_dB_mean"],\n    "training_samples": len(X_train),\n    "testing_samples": len(X_test),\n    "train_accuracy": float(train_score),\n    "test_accuracy": float(test_score),\n    "model_type": "XGBClassifier",\n    "device": "cuda:0",\n    "gpu_device": "RTX 4060",\n    "tree_method": "hist",\n    "n_estimators": 100,\n    "max_depth": 20,\n    "learning_rate": 0.1\n}\n\nimport json\ninfo_filename = model_filename.replace(\'.joblib\', \'_info.json\')\nwith open(info_filename, \'w\') as f:\n    json.dump(info, f, indent=2)\nprint(f"✅ Model info saved to: {info_filename}")\n\n# No cleanup needed (Dask removed)\nprint("\\n[2] Cleanup complete")\n\nprint("="*70)\n\nprint("\\n" + "="*70)\n\nprint("🎉 TRAINING COMPLETE!")\n')
+

.temp/02.predict_ODC.py

@@ -0,0 +1,191 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+get_ipython().run_cell_magic('time', '', '%matplotlib inline\n\nimport importlib\nimport new_import_ODC  \n\nimportlib.reload(new_import_ODC)\n\nfrom new_import_ODC import *\n')
+
+
+# In[2]:
+
+
+get_ipython().run_cell_magic('time', '', '# Dask gateway\ncluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))\ndc = datacube.Datacube()\n\n# Configure s3 access\nconfigure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n\nclient\n')
+
+
+# In[3]:
+
+
+## cấu hình thời gian lấy ảnh và tọa độ
+date_range = ('2022-09-01', '2023-10-01')
+longtitude_range = (105.86575, 105.94120)
+latitude_range = (9.65070, 9.69850)
+
+
+# In[4]:
+
+
+## truy vấn ảnh vệ tinh sen2
+data = load_data(dc, date_range, longtitude_range, latitude_range)
+notebook_utils.heading(notebook_utils.xarray_object_size(data))
+display(data)
+
+
+# In[5]:
+
+
+get_ipython().run_cell_magic('time', '', '# Tiến hành loại bỏ các vị trí bị mây ảnh hưởng\nresult = mask_clean(data)\nprogress(result)\n')
+
+
+# In[6]:
+
+
+# Tiến hành tính toán NDVI
+ds1 = calculate_indices(result, index='NDVI', satellite_mission='s2')
+ndvi = ds1["NDVI"]
+display(ndvi)
+
+
+# In[7]:
+
+
+## ảnh NDVI chưa điền mây (fill nan)
+plt.imshow(ndvi.isel(time=50))
+
+
+# In[8]:
+
+
+# đặt thời gian các mùa
+time_split = [slice('2022-09-01', '2023-01-01'), 
+               slice('2023-01-01', '2023-05-01'),
+               slice('2023-05-01', '2023-07-01'),
+               slice('2023-07-01', '2023-10-01')]
+
+# Điền mây ở các vị trí mang giá trị nan (fill nan)
+fill_nan_ndvi = fill_nan(ndvi, time_split)
+
+# In kết quả ảnh ndvi đã điền mây (đã fill nan)
+plt.imshow(fill_nan_ndvi.isel(time=50))
+
+
+# In[9]:
+
+
+get_ipython().run_cell_magic('time', '', "## tính ndvi theo tháng\naverage_ndvi = fill_nan_ndvi.resample(time='1M').mean().persist()\nprogress(average_ndvi)\n")
+
+
+# In[10]:
+
+
+# compute average_ndvi
+average_ndvi = average_ndvi.compute()
+
+
+# In[11]:
+
+
+# load dữ liệu sen1
+coordinates = (longtitude_range, latitude_range)
+dsvh, dsvv = load_data_sen1(dc, date_range, coordinates)
+average_vv = calculate_average(dsvv, time_pattern='1M')
+average_vh = calculate_average(dsvh, time_pattern='1M')
+
+
+# In[12]:
+
+
+# load model RF
+loaded_model = joblib.load(os.path.join("model_train", "model_odc.joblib"))
+
+# dự đoán
+data_array = predict(loaded_model, data.rio.crs, average_ndvi, average_vh, average_vv)
+
+
+# In[13]:
+
+
+# cấu hình màu cho các loại đất
+colors = [
+    "#abcee9",
+    "#ffef44",
+    "#c4ff9e",
+    "#ffd6a8",
+    "#93ddda",
+    "#1aeef7",
+    "#ffa7f2",
+    "#33ee33"
+]
+labels = [
+    "Lúa tôm",
+    "Lúa",
+    "CHN",
+    "CLN",
+    "TS",
+    "Sông",
+    "Đất xây dựng",
+    "Rừng"
+]
+# hiển thị phân loại sử dụng đất
+cmap = ListedColormap(colors)
+img = data_array.plot(cmap=cmap, add_colorbar=False)
+cbar = plt.colorbar(img)
+cbar.ax.set_yticklabels(labels)
+plt.title("Phân loại sử dụng đất")
+plt.axis('off')
+plt.show()
+
+
+# In[14]:
+
+
+## cấu hình shapefile ranh giới thuận hòa và vh vv file
+thuanhoa_path = "ThuanHoa/region/ST_ThuanHoa_Boundaryofficially.shp"
+
+# cắt theo ranh giới xã thuận hòa
+region_result = cut_according_shp(thuanhoa_path, average_ndvi, data_array)
+
+
+# In[15]:
+
+
+# hiển thị kết quả phân loại sử dụng đất
+colorval = list(range(len(colors)))
+options = {
+    'title': 'Phân loại sử dụng đất',
+    'cmap': colors,
+    'clim': (0, 8),
+    'aspect': 'equal',
+    'colorbar_opts': {
+        'major_label_overrides': dict(zip(colorval, labels)),
+        'major_label_text_align': 'left',
+        'ticker': FixedTicker(ticks=colorval),
+    },
+}
+                                      
+region_result.hvplot(
+    rasterize = True,                  # Use Datashader, particularly useful for dask arrays
+    aggregator = reductions.mode(),    # Datashader selects mode value, requires 'hv.Image'
+).options(opts.Image(**options))
+
+
+# In[16]:
+
+
+# Lưu lại kết quả
+region_result.rio.to_raster("KetQuaPhanLoaiDatODC.tif")
+
+
+# In[17]:
+
+
+# đóng client, cluster
+client.close()
+cluster.close()
+
+
+# In[ ]:
+
+
+
+

.temp/03.compare_ODC.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+# Khai báo các thư viện cần thiết
+from new_import_ODC import *
+
+# Khai báo đường dẫn đến kết quả phân loại và dữ liệu của địa phương
+KD_path = "ThuanHoa/KhoanhDat/ThuanHoa_TKDD2022.shp"
+KetQuaPhanLoaiDat = "KetQuaPhanLoaiDatODC.tif"
+
+
+# In[2]:
+
+
+# khai báo các loại đất từ dữ liệu kiểm kê ứng với các hiện trạng được phân loại từ viễn thám
+CODE_MAP = {
+    "BHK": 2,
+    "CLN": 3,
+    "DGD": 6,
+    "DGT": 6,
+    "DNL": 6,
+    "DRA": 6,
+    "DSH": 6,
+    "DTL": 5,
+    "DTS": 6,
+    "DYT": 6,
+    "LUC": 1,
+    "NKH": 3,
+    "NTD": 6,
+    "NTS": 4,
+    "ONT": 6,
+    "SKC": 6,
+    "SKX": 6,
+    "SON": 5,
+    "TMD": 6,
+    "TON": 6,
+    "TSC": 6,
+}
+
+# Khai báo các nhãn phân loại đất ứng với 3 loại đất chính
+HT_MAP = {
+    "NN": {"name": "Đất Nông Nghiệp", "data": [1, 2, 3, 4]},
+    "PNN": {"name": "Đất Phi Nông Nghiệp", "data": [6]},
+    "TQ": {"name": "Đất Thổ Quả", "data": [15]},
+}
+
+
+# In[3]:
+
+
+# Tiến hành chồng lắp
+result = compare(KD_path, KetQuaPhanLoaiDat, CODE_MAP, HT_MAP)
+
+
+# In[4]:
+
+
+# cấu hình màu cho các loại sử dụng đất
+colors = [
+    "#abcee9",
+    "#ffffc0",
+    "#c4ff9e",
+    "#ffd6a8",
+    "#93ddda",
+    "#1aeef7",
+    "#ffa7f2",
+    "#33ee33",
+]
+labels = ["Lúa tôm", "Lúa", "CHN", "CLN", "TS", "Sông", "Đất xây dựng", "Rừng"]
+
+
+# In[5]:
+
+
+# Lưu kết quả
+save_result(result, HT_MAP)
+
+
+# In[6]:
+
+
+# hiển thị kết quả
+xx = []
+
+for k, v in result.items():
+    rs = merge_arrays(v, nodata=np.nan)
+    xx.append(rs.squeeze(drop=True))
+xx = xr.concat(xx, pd.Index([HT_MAP[x]["name"] for x in HT_MAP], name="name"))
+
+colorval = list(range(len(colors)))
+options = {
+    "cmap": colors,
+    "clim": (0, 8),
+    "aspect": "equal",
+    "height": 400,
+    "colorbar_opts": {
+        "major_label_overrides": dict(zip(colorval, labels)),
+        "major_label_text_align": "left",
+        "ticker": FixedTicker(ticks=colorval),
+    },
+}
+
+xx.hvplot(
+    groupby="name",
+    rasterize=True,  # Use Datashader, particularly useful for dask arrays
+    aggregator=reductions.mode(),  # Datashader selects mode value, requires 'hv.Image'
+).options(opts.Image(**options))
+
+
+# In[ ]:
+
+
+
+

.temp/check_X.py

@@ -0,0 +1,18 @@
+import joblib
+import numpy as np
+
+cache_file = "dataset_cache/training_data_2d.joblib"
+data = joblib.load(cache_file)
+X = np.array(data['X'])
+y = np.array(data['y'])
+
+print("X shape:", X.shape)
+print("X mean:", np.mean(X))
+print("X std:", np.std(X))
+print("X min:", np.min(X))
+print("X max:", np.max(X))
+print("Any NaN:", np.isnan(X).any())
+
+for i in range(6):
+    print(f"Channel {i} mean: {np.mean(X[:, i, :, :]):.4f}, min: {np.min(X[:, i, :, :]):.4f}, max: {np.max(X[:, i, :, :]):.4f}")
+

.temp/check_missing.py

@@ -0,0 +1,10 @@
+import joblib
+import geopandas as gpd
+from shapely.geometry import Point
+
+data = joblib.load('dataset_cache/training_data_2d.joblib')
+X, y = data['X'], data['y']
+print(f"X shape: {X.shape}, y shape: {y.shape}")
+
+gdf = gpd.read_file("train/ST_training_data_updated_1130points_new.shp")
+print("Total points:", len(gdf))

.temp/check_versions.py

@@ -0,0 +1,5 @@
+import xarray as xr
+import rasterio
+
+print(f"xarray version: {xr.__version__}")
+print(f"rasterio version: {rasterio.__version__}")

.temp/check_xgboost_data.py

@@ -0,0 +1,7 @@
+import joblib
+import numpy as np
+
+data = joblib.load('dataset_cache/training_data.joblib')
+X, y = data['X'], data['y']
+print(f"X shape: {X.shape}")
+print(f"y shape: {y.shape}")

.temp/check_zeros.py

@@ -0,0 +1,8 @@
+import joblib
+import numpy as np
+cache_file = "dataset_cache/training_data_2d.joblib"
+data = joblib.load(cache_file)
+X = np.array(data['X'])
+b2 = X[:, 0, :, :]
+print("Zeros in B2:", np.sum(b2 == 0) / b2.size)
+print("X shape:", X.shape)

.temp/cloud_removal_train.ipynb

@@ -0,0 +1,10 @@
+{
+ "cells": [],
+ "metadata": {
+  "language_info": {
+   "name": "python"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

.temp/fix_all.py

@@ -0,0 +1,25 @@
+import glob, json
+
+changed_files = []
+for file_path in glob.glob('*.ipynb'):
+    with open(file_path, 'r', encoding='utf-8') as f:
+        nb = json.load(f)
+        
+    changed = False
+    for cell in nb.get('cells', []):
+        if cell.get('cell_type') == 'code':
+            source = cell.get('source', [])
+            for i, line in enumerate(source):
+                if 'time=50' in line:
+                    source[i] = line.replace('time=50', 'time=0')
+                    changed = True
+                if 'load_data_sen1(dc,' in line:
+                    source[i] = line.replace('load_data_sen1(dc,', 'load_data_sen1(None,')
+                    changed = True
+                    
+    if changed:
+        with open(file_path, 'w', encoding='utf-8') as f:
+            json.dump(nb, f, indent=1)
+        changed_files.append(file_path)
+
+print('Fixed issues in:', changed_files)

.temp/fix_cache.py

@@ -0,0 +1,19 @@
+import joblib
+import geopandas as gpd
+import numpy as np
+
+cache_file = "dataset_cache/training_data_2d.joblib"
+data = joblib.load(cache_file)
+X = data['X']
+print("X shape:", len(X))
+
+gdf = gpd.read_file("train/ST_training_data_updated_1130points_new.shp")
+gdf = gdf.to_crs("EPSG:32648")
+print("gdf length:", len(gdf))
+
+if len(X) == len(gdf):
+    y = [(row['HT_code'] - 1) for idx, row in gdf.iterrows()]
+    joblib.dump({'X': X, 'y': y}, cache_file)
+    print("Fixed y in cache! Saved.")
+else:
+    print("Lengths do not match, cannot fix automatically.")

.temp/fix_new_train.py

@@ -0,0 +1,20 @@
+import json
+
+def fix_import(file_path):
+    with open(file_path, 'r', encoding='utf-8') as f:
+        nb = json.load(f)
+    changed = False
+    for cell in nb.get('cells', []):
+        if cell.get('cell_type') == 'code':
+            source = cell.get('source', [])
+            if isinstance(source, list):
+                for i, line in enumerate(source):
+                    if "from new_import import *" in line:
+                        source[i] = line.replace("from new_import import *", "from new_import_ODC import *")
+                        changed = True
+    if changed:
+        with open(file_path, 'w', encoding='utf-8') as f:
+            json.dump(nb, f, indent=1)
+        print(f"Fixed {file_path}")
+
+fix_import('new_train.ipynb')

.temp/fix_train.py

@@ -0,0 +1,20 @@
+import re
+
+filepath = "01.train_ODC.py"
+with open(filepath, 'r', encoding='utf-8') as f:
+    content = f.read()
+
+# Find the run_cell_magic line
+pattern = re.compile(r"get_ipython\(\)\.run_cell_magic\('time', '', '(# 🤖 LAND USE CLASSIFICATION MODEL TRAINING.*?)(?=\n')\n'", re.DOTALL)
+
+def repl(match):
+    # Get the inner string and escape all actual newlines with \n
+    inner = match.group(1)
+    inner = inner.replace('\n', '\\n')
+    return f"get_ipython().run_cell_magic('time', '', '{inner}')"
+
+content = pattern.sub(repl, content)
+
+with open(filepath, 'w', encoding='utf-8') as f:
+    f.write(content)
+print("Fixed 01.train_ODC.py syntax")

.temp/fix_typo.py

@@ -0,0 +1,22 @@
+import json
+
+def fix_filename(file_path):
+    with open(file_path, 'r', encoding='utf-8') as f:
+        nb = json.load(f)
+    changed = False
+    for cell in nb.get('cells', []):
+        if cell.get('cell_type') == 'code':
+            source = cell.get('source', [])
+            if isinstance(source, list):
+                for i, line in enumerate(source):
+                    if "ST_training data_updated_1130points.shp" in line:
+                        source[i] = line.replace("ST_training data_updated_1130points.shp", "ST_training_data_updated_1130points.shp")
+                        changed = True
+    if changed:
+        with open(file_path, 'w', encoding='utf-8') as f:
+            json.dump(nb, f, indent=1)
+        print(f"Fixed typo in {file_path}")
+
+import glob
+for nb in glob.glob("*.ipynb"):
+    fix_filename(nb)

.temp/generate_colab_notebook.py

@@ -0,0 +1,58 @@
+import nbformat as nbf
+
+nb = nbf.v4.new_notebook()
+
+text_1 = """# Script Tải Dữ liệu Vệ tinh (Cache) qua Google Colab
+Mục đích của Notebook này là mượn sức mạnh đường truyền và RAM của Google Colab để tải 270 ảnh Sentinel-2 & Sentinel-1 từ Microsoft Planetary Computer. Sau khi xử lý nội suy, nó sẽ sinh ra một file cache `.joblib` duy nhất chứa toàn bộ mảng dữ liệu. 
+Bạn chỉ cần tải file `.joblib` đó về máy là xong!"""
+
+code_1 = """!pip install planetary-computer pystac-client odc-stac geopandas rasterio xarray joblib scikit-learn xgboost lightgbm"""
+
+code_2 = """from google.colab import drive
+drive.mount('/content/drive')"""
+
+text_2 = """## Hướng dẫn:
+1. Nén toàn bộ thư mục `remote-sensing` ở máy tính của bạn thành file `remote-sensing.zip`.
+2. Upload file `remote-sensing.zip` đó lên Google Drive (để ngay ngoài cùng).
+3. Chạy ô lệnh bên dưới để giải nén và chuyển vào thư mục dự án."""
+
+code_3 = """import os
+import shutil
+
+# Giải nén dự án từ Google Drive
+!unzip -q /content/drive/MyDrive/remote-sensing.zip -d /content/
+os.chdir('/content/remote-sensing')
+!ls -la"""
+
+text_3 = """## Bắt đầu tải và Cache Dữ Liệu
+Chạy một mô hình CPU đơn giản (Decision Tree) để ép hệ thống gọi hàm `FeatureExtractor`. Hàm này sẽ làm mọi việc nặng nhọc: tìm ảnh, ghép mây, tính trung vị và lưu kết quả vào thư mục `dataset_cache/`."""
+
+code_4 = """# Lệnh này sẽ mất khoảng 5-15 phút để tải toàn bộ ảnh từ Microsoft
+!python train_land_decision_tree_gpu.py"""
+
+text_4 = """## Hoàn tất
+Bạn hãy kiểm tra xem file `.joblib` lớn (khoảng 40-60MB) đã xuất hiện chưa. Nếu rồi, hãy lưu ngược nó lại Google Drive để tải về máy!"""
+
+code_5 = """# Xem file cache đã được tạo thành công chưa
+!ls -lh dataset_cache/
+
+# Copy toàn bộ thư mục cache sang Google Drive để tải về máy dễ dàng
+!cp -r dataset_cache/ /content/drive/MyDrive/dataset_cache_finished/
+print("Hoàn thành! Bạn hãy mở Google Drive của mình, tìm thư mục 'dataset_cache_finished' và tải file .joblib mới nhất về máy tính.")"""
+
+nb['cells'] = [
+    nbf.v4.new_markdown_cell(text_1),
+    nbf.v4.new_code_cell(code_1),
+    nbf.v4.new_code_cell(code_2),
+    nbf.v4.new_markdown_cell(text_2),
+    nbf.v4.new_code_cell(code_3),
+    nbf.v4.new_markdown_cell(text_3),
+    nbf.v4.new_code_cell(code_4),
+    nbf.v4.new_markdown_cell(text_4),
+    nbf.v4.new_code_cell(code_5)
+]
+
+with open('Download_Cache_Colab.ipynb', 'w') as f:
+    nbf.write(nb, f)
+
+print("Created Download_Cache_Colab.ipynb")

.temp/generate_previews.py

@@ -0,0 +1,132 @@
+#!/usr/bin/env python3
+"""
+Generate PNG previews for existing GeoTIFF prediction files
+"""
+
+import numpy as np
+import rasterio
+import matplotlib
+matplotlib.use('Agg')
+import matplotlib.pyplot as plt
+from pathlib import Path
+import sys
+
+def generate_png_preview(tif_file, output_png=None):
+    """Generate PNG preview from GeoTIFF file"""
+    tif_path = Path(tif_file)
+    
+    if not tif_path.exists():
+        print(f"❌ File not found: {tif_file}")
+        return False
+    
+    # Determine output PNG path
+    if output_png is None:
+        output_png = tif_path.with_suffix('.png')
+    else:
+        output_png = Path(output_png)
+    
+    try:
+        # Read GeoTIFF
+        with rasterio.open(tif_path) as src:
+            data = src.read(1)
+            
+        print(f"📊 Data shape: {data.shape}, range: [{np.nanmin(data):.3f}, {np.nanmax(data):.3f}]")
+        
+        # Determine if it's classification or NDVI based on filename
+        is_classification = 'classification' in tif_path.name.lower() or 'prediction' in tif_path.name.lower()
+        is_ndvi = 'ndvi' in tif_path.name.lower()
+        
+        # Create figure
+        fig, ax = plt.subplots(figsize=(12, 10), dpi=150)
+        
+        if is_ndvi:
+            # NDVI: use RdYlGn colormap, range -1 to 1
+            im = ax.imshow(data, cmap='RdYlGn', vmin=-1, vmax=1, interpolation='nearest')
+            ax.set_title(f'NDVI - {tif_path.stem}', fontsize=14, fontweight='bold')
+            cbar_label = 'NDVI'
+        elif is_classification:
+            # Classification: use tab20 colormap
+            im = ax.imshow(data, cmap='tab20', interpolation='nearest')
+            ax.set_title(f'Land Classification - {tif_path.stem}', fontsize=14, fontweight='bold')
+            cbar_label = 'Class'
+        else:
+            # Generic: use viridis
+            im = ax.imshow(data, cmap='viridis', interpolation='nearest')
+            ax.set_title(f'{tif_path.stem}', fontsize=14, fontweight='bold')
+            cbar_label = 'Value'
+        
+        ax.set_xlabel('X (pixels)', fontsize=10)
+        ax.set_ylabel('Y (pixels)', fontsize=10)
+        
+        # Add colorbar
+        cbar = plt.colorbar(im, ax=ax, fraction=0.046, pad=0.04)
+        cbar.set_label(cbar_label, rotation=270, labelpad=15)
+        
+        # For classification, try to set integer ticks
+        if is_classification:
+            try:
+                unique_vals = np.unique(data[~np.isnan(data)])
+                if len(unique_vals) < 20:  # Only if not too many classes
+                    cbar.set_ticks(unique_vals)
+                    cbar.set_ticklabels([str(int(v)) for v in unique_vals])
+            except:
+                pass
+        
+        # Add grid
+        ax.grid(True, alpha=0.3, linestyle='--', linewidth=0.5)
+        
+        # Save PNG
+        plt.tight_layout()
+        plt.savefig(str(output_png), dpi=150, bbox_inches='tight')
+        plt.close(fig)
+        
+        print(f"✅ Created PNG: {output_png}")
+        return True
+        
+    except Exception as e:
+        print(f"❌ Error creating PNG: {e}")
+        import traceback
+        traceback.print_exc()
+        return False
+
+
+def generate_all_previews(predictions_dir="predictions"):
+    """Generate PNG previews for all GeoTIFF files without PNGs"""
+    pred_path = Path(predictions_dir)
+    
+    if not pred_path.exists():
+        print(f"❌ Directory not found: {predictions_dir}")
+        return
+    
+    tif_files = list(pred_path.glob("*.tif"))
+    print(f"🔍 Found {len(tif_files)} GeoTIFF files")
+    
+    generated = 0
+    skipped = 0
+    
+    for tif_file in tif_files:
+        png_file = tif_file.with_suffix('.png')
+        
+        if png_file.exists():
+            print(f"⏭️  Skipping {tif_file.name} (PNG already exists)")
+            skipped += 1
+            continue
+        
+        print(f"\n🎨 Processing {tif_file.name}...")
+        if generate_png_preview(tif_file):
+            generated += 1
+    
+    print(f"\n{'='*60}")
+    print(f"✅ Generated {generated} new PNG previews")
+    print(f"⏭️  Skipped {skipped} files (already have PNGs)")
+    print(f"{'='*60}")
+
+
+if __name__ == "__main__":
+    if len(sys.argv) > 1:
+        # Process specific file
+        tif_file = sys.argv[1]
+        generate_png_preview(tif_file)
+    else:
+        # Process all files in predictions directory
+        generate_all_previews()

.temp/generate_report_only.py

@@ -0,0 +1,97 @@
+import os
+import glob
+import json
+from tabulate import tabulate
+
+print("📊 BẢNG SO SÁNH KẾT QUẢ CÁC MÔ HÌNH\n")
+
+# 1. Phân loại đất
+print("### 1. Nhóm Phân loại Lớp phủ (Land Classification)")
+land_data = []
+
+if os.path.exists("model_xgboost_info.json"):
+    with open("model_xgboost_info.json", 'r') as f:
+        data = json.load(f)
+        params = data.get('params', {})
+        param_str = f"estimators:{params.get('n_estimators')}, depth:{params.get('max_depth')}" if params else "N/A"
+        land_data.append([
+            data.get('model_type', 'XGBoost'),
+            data.get('accuracy', ''),
+            data.get('precision', ''),
+            data.get('recall', ''),
+            data.get('f1_score', ''),
+            param_str
+        ])
+
+for info_file in glob.glob("model_train/*_info.json"):
+    with open(info_file, 'r') as f:
+        data = json.load(f)
+        # Support both 'accuracy' and 'test_accuracy'
+        acc = data.get('accuracy', data.get('test_accuracy', ''))
+        
+        f1 = data.get('f1_score', '')
+        precision = data.get('precision', '')
+        recall = data.get('recall', '')
+        
+        clf_rep = data.get('classification_report')
+        if isinstance(clf_rep, dict) and 'macro avg' in clf_rep:
+            if not f1:
+                f1 = clf_rep['macro avg'].get('f1-score', '')
+            if not precision:
+                precision = clf_rep['macro avg'].get('precision', '')
+            if not recall:
+                recall = clf_rep['macro avg'].get('recall', '')
+            
+        if not acc and not f1:
+            continue
+            
+        params = data.get('params', {})
+        param_str = f"estimators:{params.get('n_estimators')}, depth:{params.get('max_depth')}" if params else "N/A"
+        
+        if data.get('model_type') == 'RandomForest_RealData':
+            param_str = "estimators:100, depth:15"
+            
+        land_data.append([
+            data.get('model_type', ''),
+            acc,
+            precision,
+            recall,
+            f1,
+            param_str
+        ])
+
+if land_data:
+    print(tabulate(land_data, headers=["Model", "Accuracy", "Precision", "Recall", "F1-Score", "Parameters"], tablefmt="github"))
+print("\n")
+
+# 2. Xóa mây
+print("### 2. Nhóm Xóa mây (Cloud Removal)")
+cloud_data = []
+for info_file in glob.glob("cloud_removal_model/*_info.json"):
+    with open(info_file, 'r') as f:
+        data = json.load(f)
+        cloud_data.append([
+            data.get('model_type', ''),
+            data.get('epoch', ''),
+            data.get('train_loss', ''),
+            data.get('val_loss', '')
+        ])
+if cloud_data:
+    print(tabulate(cloud_data, headers=["Model", "Epochs", "Train Loss", "Val Loss"], tablefmt="github"))
+print("\n")
+
+# 3. Dự báo NDVI
+print("### 3. Nhóm Dự báo Thực vật (NDVI Forecasting)")
+ndvi_data = []
+for info_file in glob.glob("ndvi_forecast_model/*_info.json"):
+    with open(info_file, 'r') as f:
+        data = json.load(f)
+        ndvi_data.append([
+            data.get('model_type', ''),
+            data.get('rmse', ''),
+            data.get('mae', ''),
+            data.get('epoch', 'N/A')
+        ])
+if ndvi_data:
+    print(tabulate(ndvi_data, headers=["Model", "RMSE", "MAE", "Epochs"], tablefmt="github"))
+print("\n")

.temp/inspect_cells_train.py

@@ -0,0 +1,7 @@
+import json
+nb = json.load(open('01.train_ODC.ipynb'))
+for idx, cell in enumerate(nb['cells']):
+    if cell['cell_type'] == 'code':
+        print(f"Cell {idx}:")
+        print("".join(cell['source'][:3]))
+        print("-" * 20)

.temp/inspect_model_odc.py

@@ -0,0 +1,64 @@
+"""
+Inspect model_odc.joblib to see what it actually contains
+"""
+
+import joblib
+from pathlib import Path
+
+model_path = Path("model_train/model_odc.joblib")
+
+if model_path.exists():
+    print("Loading model_odc.joblib...")
+    model_data = joblib.load(model_path)
+    
+    print(f"\nModel type: {type(model_data)}")
+    print(f"Model class: {model_data.__class__.__name__}")
+    
+    # Check if it's a dict
+    if isinstance(model_data, dict):
+        print(f"\nModel is a dict with keys: {model_data.keys()}")
+        model = model_data.get('model')
+    else:
+        model = model_data
+    
+    print(f"\nActual model type: {type(model)}")
+    print(f"Actual model class: {model.__class__.__name__}")
+    
+    # Try to get feature info
+    if hasattr(model, 'n_features_in_'):
+        print(f"\nn_features_in_: {model.n_features_in_}")
+    
+    if hasattr(model, 'feature_names_in_'):
+        print(f"feature_names_in_: {model.feature_names_in_}")
+    
+    # If it's a GridSearchCV
+    if hasattr(model, 'best_estimator_'):
+        print(f"\nThis is a GridSearchCV!")
+        print(f"Best estimator: {model.best_estimator_}")
+        
+        best_est = model.best_estimator_
+        if hasattr(best_est, 'steps'):
+            print(f"\nPipeline steps:")
+            for step_name, step in best_est.steps:
+                print(f"  - {step_name}: {step.__class__.__name__}")
+                if hasattr(step, 'n_features_in_'):
+                    print(f"      n_features_in_: {step.n_features_in_}")
+    
+    # If it's a Pipeline
+    if hasattr(model, 'steps'):
+        print(f"\nThis is a Pipeline!")
+        print(f"Pipeline steps:")
+        for step_name, step in model.steps:
+            print(f"  - {step_name}: {step.__class__.__name__}")
+            if hasattr(step, 'n_features_in_'):
+                print(f"      n_features_in_: {step.n_features_in_}")
+    
+    # Try to get booster for XGBoost
+    try:
+        if hasattr(model, 'get_booster'):
+            print(f"\nXGBoost num_features: {model.get_booster().num_features()}")
+    except:
+        pass
+    
+else:
+    print(f"Model file not found: {model_path}")

.temp/log_cnn.txt

@@ -0,0 +1,18 @@
+🚀 BẮT ĐẦU HUẤN LUYỆN MÔ HÌNH CNN (GPU & CACHE)
+Initializing FeatureExtractor (mode=extended)...
+📦 Đang load cache: training_data_507bd2ba4ec0d3fe107839cbf73a7a7d.joblib...
+✅ Loaded 632 samples từ cache!
+   ⚡ Đã bỏ qua download위성 data (tiết kiệm thời gian)
+[CACHE HIT] Using cached dataset with 632 samples
+Training CNN model...
+Building CNN model on cuda...
+Training CNN model with PyTorch...
+CNN Epoch 10/15, Loss: 1.2171
+Evaluating model...
+Generating classification report...
+Saving model...
+[MODEL MANAGER] Saving model to: model_train/model_cnn_auto.joblib
+[MODEL MANAGER] Saving metadata to: model_train/model_cnn_auto_info.json
+[MODEL MANAGER] Model saved successfully!
+Training complete!
+✅ Hoàn thành! Accuracy: 0.5748

.temp/log_dt.txt

@@ -0,0 +1,15 @@
+🚀 BẮT ĐẦU HUẤN LUYỆN MÔ HÌNH DECISION TREE (GPU & CACHE)
+Initializing FeatureExtractor (mode=extended)...
+📦 Đang load cache: training_data_507bd2ba4ec0d3fe107839cbf73a7a7d.joblib...
+✅ Loaded 632 samples từ cache!
+   ⚡ Đã bỏ qua download위성 data (tiết kiệm thời gian)
+[CACHE HIT] Using cached dataset with 632 samples
+Training DECISION_TREE model...
+Evaluating model...
+Generating classification report...
+Saving model...
+[MODEL MANAGER] Saving model to: model_train/model_decision_tree_auto.joblib
+[MODEL MANAGER] Saving metadata to: model_train/model_decision_tree_auto_info.json
+[MODEL MANAGER] Model saved successfully!
+Training complete!
+✅ Hoàn thành! Accuracy: 0.5906

.temp/log_mobilenet.txt

@@ -0,0 +1,30 @@
+🚀 BẮT ĐẦU HUẤN LUYỆN MÔ HÌNH MOBILENET-LRASPP (GPU & CACHE)
+Initializing FeatureExtractor (mode=extended)...
+📦 Đang load cache: training_data_507bd2ba4ec0d3fe107839cbf73a7a7d.joblib...
+✅ Loaded 632 samples từ cache!
+   ⚡ Đã bỏ qua download위성 data (tiết kiệm thời gian)
+[CACHE HIT] Using cached dataset with 632 samples
+Training MOBILENET-LRASPP model...
+Building MobileNetV3 + LR-ASPP model on cuda...
+[MOBILENET] Class distribution: [ 48  89   3  86  74  38 117  50]
+[MOBILENET] Class weights: [0.37418982 0.20181024 5.98703525 0.20885013 0.24271772 0.47266082
+ 0.15351377 0.35922223]
+Training MobileNetV3 + LR-ASPP model with PyTorch...
+MobileNet Epoch 5/25, Train Loss: 1.0614, Val Loss: 1.3584, Val Acc: 40.16%, LR: 0.000800
+[MOBILENET] Epoch 5/25 - Train Loss: 1.0614, Val Loss: 1.3584, Val Acc: 40.16%
+MobileNet Epoch 10/25, Train Loss: 0.9054, Val Loss: 0.8582, Val Acc: 59.06%, LR: 0.000800
+[MOBILENET] Epoch 10/25 - Train Loss: 0.9054, Val Loss: 0.8582, Val Acc: 59.06%
+MobileNet Epoch 15/25, Train Loss: 0.7728, Val Loss: 0.8231, Val Acc: 61.42%, LR: 0.000800
+[MOBILENET] Epoch 15/25 - Train Loss: 0.7728, Val Loss: 0.8231, Val Acc: 61.42%
+MobileNet Epoch 20/25, Train Loss: 0.7125, Val Loss: 0.8783, Val Acc: 59.84%, LR: 0.000400
+[MOBILENET] Epoch 20/25 - Train Loss: 0.7125, Val Loss: 0.8783, Val Acc: 59.84%
+[MOBILENET] Early stopping at epoch 24 (best val loss: 0.8029)
+MobileNet early stopped at epoch 24
+Evaluating model...
+Generating classification report...
+Saving model...
+[MODEL MANAGER] Saving model to: model_train/model_mobilenet-lraspp_auto.joblib
+[MODEL MANAGER] Saving metadata to: model_train/model_mobilenet-lraspp_auto_info.json
+[MODEL MANAGER] Model saved successfully!
+Training complete!
+✅ Hoàn thành! Accuracy: 0.5669

.temp/log_rf.txt

@@ -0,0 +1,15 @@
+🚀 BẮT ĐẦU HUẤN LUYỆN MÔ HÌNH RANDOM FOREST (GPU & CACHE)
+Initializing FeatureExtractor (mode=extended)...
+📦 Đang load cache: training_data_507bd2ba4ec0d3fe107839cbf73a7a7d.joblib...
+✅ Loaded 632 samples từ cache!
+   ⚡ Đã bỏ qua download위성 data (tiết kiệm thời gian)
+[CACHE HIT] Using cached dataset with 632 samples
+Training RANDOM_FOREST model...
+Evaluating model...
+Generating classification report...
+Saving model...
+[MODEL MANAGER] Saving model to: model_train/model_random_forest_auto.joblib
+[MODEL MANAGER] Saving metadata to: model_train/model_random_forest_auto_info.json
+[MODEL MANAGER] Model saved successfully!
+Training complete!
+✅ Hoàn thành! Accuracy: 0.6142

.temp/log_svm.txt

@@ -0,0 +1,15 @@
+🚀 BẮT ĐẦU HUẤN LUYỆN MÔ HÌNH SVM (GPU & CACHE)
+Initializing FeatureExtractor (mode=extended)...
+📦 Đang load cache: training_data_507bd2ba4ec0d3fe107839cbf73a7a7d.joblib...
+✅ Loaded 632 samples từ cache!
+   ⚡ Đã bỏ qua download위성 data (tiết kiệm thời gian)
+[CACHE HIT] Using cached dataset with 632 samples
+Training SVM model...
+Evaluating model...
+Generating classification report...
+Saving model...
+[MODEL MANAGER] Saving model to: model_train/model_svm_auto.joblib
+[MODEL MANAGER] Saving metadata to: model_train/model_svm_auto_info.json
+[MODEL MANAGER] Model saved successfully!
+Training complete!
+✅ Hoàn thành! Accuracy: 0.5827

.temp/log_swin.txt

@@ -0,0 +1,30 @@
+🚀 BẮT ĐẦU HUẤN LUYỆN MÔ HÌNH SWIN-UNET (GPU & CACHE)
+Initializing FeatureExtractor (mode=extended)...
+📦 Đang load cache: training_data_507bd2ba4ec0d3fe107839cbf73a7a7d.joblib...
+✅ Loaded 632 samples từ cache!
+   ⚡ Đã bỏ qua download위성 data (tiết kiệm thời gian)
+[CACHE HIT] Using cached dataset with 632 samples
+Training SWIN-UNET model...
+Building Swin-UNet model on cuda...
+[SWIN-UNET] Class distribution: [ 48  89   3  86  74  38 117  50]
+[SWIN-UNET] Class weights: [0.37418982 0.20181024 5.98703525 0.20885013 0.24271772 0.47266082
+ 0.15351377 0.35922223]
+Training Swin-UNet model with PyTorch (with class weights)...
+Swin-UNet Epoch 5/40, Train Loss: 1.4096, Val Loss: 1.2804, Val Acc: 48.03%, LR: 0.000293
+[SWIN-UNET] Epoch 5/40 - Train Loss: 1.4096, Val Loss: 1.2804, Val Acc: 48.03%
+Swin-UNet Epoch 10/40, Train Loss: 1.1129, Val Loss: 1.2746, Val Acc: 57.48%, LR: 0.000271
+[SWIN-UNET] Epoch 10/40 - Train Loss: 1.1129, Val Loss: 1.2746, Val Acc: 57.48%
+Swin-UNet Epoch 15/40, Train Loss: 1.0479, Val Loss: 1.3126, Val Acc: 50.39%, LR: 0.000238
+[SWIN-UNET] Epoch 15/40 - Train Loss: 1.0479, Val Loss: 1.3126, Val Acc: 50.39%
+Swin-UNet Epoch 20/40, Train Loss: 0.9548, Val Loss: 1.1118, Val Acc: 52.76%, LR: 0.000196
+[SWIN-UNET] Epoch 20/40 - Train Loss: 0.9548, Val Loss: 1.1118, Val Acc: 52.76%
+[SWIN-UNET] Early stopping at epoch 22 (best val loss: 1.1096)
+Swin-UNet early stopped at epoch 22
+Evaluating model...
+Generating classification report...
+Saving model...
+[MODEL MANAGER] Saving model to: model_train/model_swin-unet_auto.joblib
+[MODEL MANAGER] Saving metadata to: model_train/model_swin-unet_auto_info.json
+[MODEL MANAGER] Model saved successfully!
+Training complete!
+✅ Hoàn thành! Accuracy: 0.5354

.temp/log_train_2d_final.txt

@@ -0,0 +1,38 @@
+🚀 BẮT ĐẦU PIPELINE 2D PATCH-BASED & CLOUD REMOVAL
+Loading 2D patches from dataset_cache/training_data_2d.joblib...
+Training 2D CNN with Data Augmentation...
+Epoch 1/150 - Loss: 2.2272 - Test Acc: 0.0752 🌟
+Epoch 2/150 - Loss: 2.0843 - Test Acc: 0.0796 🌟
+Epoch 4/150 - Loss: 2.0350 - Test Acc: 0.1372 🌟
+Epoch 8/150 - Loss: 2.0447 - Test Acc: 0.1637 🌟
+Epoch 10/150 - Loss: 2.0397 - Test Acc: 0.1372
+Epoch 12/150 - Loss: 2.0353 - Test Acc: 0.2168 🌟
+Epoch 20/150 - Loss: 2.0139 - Test Acc: 0.1372
+Traceback (most recent call last):
+  File "/home/x79/remote-sensing/train_land_2d_patch.py", line 298, in <module>
+    main()
+  File "/home/x79/remote-sensing/train_land_2d_patch.py", line 294, in main
+    train_2d_model(X, y)
+  File "/home/x79/remote-sensing/train_land_2d_patch.py", line 219, in train_2d_model
+    for batch_X, batch_y in train_loader:
+  File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 725, in __next__
+    data = self._next_data()
+  File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 785, in _next_data
+    data = self._dataset_fetcher.fetch(index)  # may raise StopIteration
+  File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 54, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 54, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/home/x79/remote-sensing/train_land_2d_patch.py", line 192, in __getitem__
+    x = transform(x)
+  File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torchvision/transforms/transforms.py", line 95, in __call__
+    img = t(img)
+  File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torchvision/transforms/transforms.py", line 752, in forward
+    return F.vflip(img)
+  File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torchvision/transforms/functional.py", line 757, in vflip
+    def vflip(img: Tensor) -> Tensor:
+KeyboardInterrupt

.temp/log_train_v4.txt

@@ -0,0 +1,39 @@
+🚀 V4: TÍCH HỢP RADAR SENTINEL-1 (32-CHANNELS FUSION)
+============================================================
+Clean FUSION data: (252, 32, 16, 16), 7 classes, [31, 38, 32, 49, 23, 75, 4]
+
+============================================================
+32-CHANNELS FUSION CNN
+============================================================
+  Ep 1 Fusion-Acc=0.1961 🌟
+  Ep 3 Fusion-Acc=0.2745 🌟
+  Ep 4 Fusion-Acc=0.4706 🌟
+  Ep 5 Fusion-Acc=0.5490 🌟
+  Ep 6 Fusion-Acc=0.7059 🌟
+  Ep 7 Fusion-Acc=0.7451 🌟
+  Ep 8 Fusion-Acc=0.8431 🌟
+  Ep 15 Fusion-Acc=0.8627 🌟
+
+✅ CNN Fusion best: 0.8627
+
+============================================================
+HYBRID FUSION: CNN embed + S1/S2 Rich features + XGBoost
+============================================================
+Extracted 2182 fusion features per sample
+  Final Feature Vector: (252, 2694)
+  ✅ Hybrid Fusion Acc: 0.8627
+    Fold 1: 0.9412
+    Fold 2: 0.9020
+    Fold 3: 0.9200
+    Fold 4: 0.8800
+    Fold 5: 0.9000
+  ✅ CV Mean: 0.9086 ± 0.0206
+
+============================================================
+📊 FINAL RESULTS V4 (WITH RADAR)
+============================================================
+  ✅ Hybrid Fusion CV: 0.9086
+  📈 CNN Fusion (32ch): 0.8627
+  📈 Hybrid Fusion (CNN+XGB): 0.8627
+
+🏆 BEST: 0.9086

.temp/log_train_v5.txt

@@ -0,0 +1,18 @@
+
+============================================================
+HYBRID FUSION ENSEMBLE: CNN embed + S1/S2 Rich features + XGB/LGBM/ETC
+============================================================
+  Final Feature Vector: (443, 2694)
+    Fold 1: 0.8876
+    Fold 2: 0.9438
+    Fold 3: 0.9438
+    Fold 4: 0.9659
+    Fold 5: 0.9432
+  ✅ Ensemble CV Mean: 0.9369 ± 0.0261
+
+============================================================
+📊 FINAL RESULTS V5 (ENSEMBLE + RADAR)
+============================================================
+  ✅ Hybrid Fusion Ensemble CV: 0.9369
+
+🏆 BEST: 0.9369

.temp/log_train_v6.txt

@@ -0,0 +1,21 @@
+🚀 V6: EXHAUSTIVE HYPERPARAMETER TUNING
+============================================================
+Data: (443, 32, 16, 16), 7 classes, dist=[65, 55, 48, 72, 74, 124, 5]
+
+--- Training Multi-Seed CNN Ensemble ---
+  Seed 42: CNN Acc = 0.8989
+  Seed 123: CNN Acc = 0.8652
+  Seed 777: CNN Acc = 0.8876
+  Multi-seed CNN embedding: (443, 1536)
+  Total features: (443, 3940)
+
+============================================================
+🔬 EXHAUSTIVE HYPERPARAMETER SEARCH
+============================================================
+  🏆 XGB-deep: 0.9526 ± 0.0110  (folds: ['0.955', '0.933', '0.966', '0.955', '0.955'])
+  ✅ XGB-shallow: 0.9436 ± 0.0173  (folds: ['0.944', '0.910', '0.955', '0.955', '0.955'])
+  🏆 XGB-balanced: 0.9504 ± 0.0113  (folds: ['0.944', '0.933', '0.955', '0.955', '0.966'])
+  ✅ LGBM-tuned: 0.9458 ± 0.0149  (folds: ['0.955', '0.921', '0.966', '0.943', '0.943'])
+  ✅ LGBM-conservative: 0.9481 ± 0.0152  (folds: ['0.944', '0.921', '0.966', '0.955', '0.955'])
+  🏆 ETC-deep: 0.9572 ± 0.0082  (folds: ['0.944', '0.955', '0.955', '0.966', '0.966'])
+  🏆 RF-tuned: 0.9549 ± 0.0099  (folds: ['0.944', '0.955', '0.944', '0.966', '0.966'])

.temp/log_tune_swin_unet.txt

@@ -0,0 +1,157 @@
+🚀 BẮT ĐẦU TÌM KIẾM SIÊU THAM SỐ CHO SWIN-UNET
+Loading data from dataset_cache/training_data_507bd2ba4ec0d3fe107839cbf73a7a7d.joblib...
+Using device: cuda
+
+[1/48] Training with params: {'embed_dim': 64, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.5984
+🌟 NEW BEST ACCURACY: 0.5984
+
+[2/48] Training with params: {'embed_dim': 64, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.6063
+🌟 NEW BEST ACCURACY: 0.6063
+
+[3/48] Training with params: {'embed_dim': 64, 'lr': 0.001, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.6142
+🌟 NEW BEST ACCURACY: 0.6142
+
+[4/48] Training with params: {'embed_dim': 64, 'lr': 0.001, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.6772
+🌟 NEW BEST ACCURACY: 0.6772
+
+[5/48] Training with params: {'embed_dim': 64, 'lr': 0.0005, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.5827
+
+[6/48] Training with params: {'embed_dim': 64, 'lr': 0.0005, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.5984
+
+[7/48] Training with params: {'embed_dim': 64, 'lr': 0.0005, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.5669
+
+[8/48] Training with params: {'embed_dim': 64, 'lr': 0.0005, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.6142
+
+[9/48] Training with params: {'embed_dim': 64, 'lr': 0.0001, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.6142
+
+[10/48] Training with params: {'embed_dim': 64, 'lr': 0.0001, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.5118
+
+[11/48] Training with params: {'embed_dim': 64, 'lr': 0.0001, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.5591
+
+[12/48] Training with params: {'embed_dim': 64, 'lr': 0.0001, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.6063
+
+[13/48] Training with params: {'embed_dim': 128, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.7008
+🌟 NEW BEST ACCURACY: 0.7008
+
+[14/48] Training with params: {'embed_dim': 128, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.6142
+
+[15/48] Training with params: {'embed_dim': 128, 'lr': 0.001, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.6772
+
+[16/48] Training with params: {'embed_dim': 128, 'lr': 0.001, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.6063
+
+[17/48] Training with params: {'embed_dim': 128, 'lr': 0.0005, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.5906
+
+[18/48] Training with params: {'embed_dim': 128, 'lr': 0.0005, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.6457
+
+[19/48] Training with params: {'embed_dim': 128, 'lr': 0.0005, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.5906
+
+[20/48] Training with params: {'embed_dim': 128, 'lr': 0.0005, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.5906
+
+[21/48] Training with params: {'embed_dim': 128, 'lr': 0.0001, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.6220
+
+[22/48] Training with params: {'embed_dim': 128, 'lr': 0.0001, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.6457
+
+[23/48] Training with params: {'embed_dim': 128, 'lr': 0.0001, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.5984
+
+[24/48] Training with params: {'embed_dim': 128, 'lr': 0.0001, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.6063
+
+[25/48] Training with params: {'embed_dim': 256, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.7087
+🌟 NEW BEST ACCURACY: 0.7087
+
+[26/48] Training with params: {'embed_dim': 256, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.7323
+🌟 NEW BEST ACCURACY: 0.7323
+
+[27/48] Training with params: {'embed_dim': 256, 'lr': 0.001, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.6457
+
+[28/48] Training with params: {'embed_dim': 256, 'lr': 0.001, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.6142
+
+[29/48] Training with params: {'embed_dim': 256, 'lr': 0.0005, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.5984
+
+[30/48] Training with params: {'embed_dim': 256, 'lr': 0.0005, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.5906
+
+[31/48] Training with params: {'embed_dim': 256, 'lr': 0.0005, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.6142
+
+[32/48] Training with params: {'embed_dim': 256, 'lr': 0.0005, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.7008
+
+[33/48] Training with params: {'embed_dim': 256, 'lr': 0.0001, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.6299
+
+[34/48] Training with params: {'embed_dim': 256, 'lr': 0.0001, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.6299
+
+[35/48] Training with params: {'embed_dim': 256, 'lr': 0.0001, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.6378
+
+[36/48] Training with params: {'embed_dim': 256, 'lr': 0.0001, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.6457
+
+[37/48] Training with params: {'embed_dim': 512, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.6142
+
+[38/48] Training with params: {'embed_dim': 512, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.6457
+
+[39/48] Training with params: {'embed_dim': 512, 'lr': 0.001, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.6378
+
+[40/48] Training with params: {'embed_dim': 512, 'lr': 0.001, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.6299
+
+[41/48] Training with params: {'embed_dim': 512, 'lr': 0.0005, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.6378
+
+[42/48] Training with params: {'embed_dim': 512, 'lr': 0.0005, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.6220
+
+[43/48] Training with params: {'embed_dim': 512, 'lr': 0.0005, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.6142
+
+[44/48] Training with params: {'embed_dim': 512, 'lr': 0.0005, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.7008
+
+[45/48] Training with params: {'embed_dim': 512, 'lr': 0.0001, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.6457
+
+[46/48] Training with params: {'embed_dim': 512, 'lr': 0.0001, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.7323
+
+[47/48] Training with params: {'embed_dim': 512, 'lr': 0.0001, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.6693
+
+[48/48] Training with params: {'embed_dim': 512, 'lr': 0.0001, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.6457
+
+✅ Đã lưu mô hình tốt nhất (Acc: 0.7323) vào land_classification_model/model_swin-unet_optimized_95.joblib
+Cấu hình tốt nhất: {'embed_dim': 256, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 500}

.temp/log_ultimate_95.txt

@@ -0,0 +1,117 @@
+🚀 CHIẾN LƯỢC TOÀN DIỆN ĐẠT >95% ACCURACY
+============================================================
+Loaded data: X=(706, 24, 16, 16), y=(706,)
+Labels unique: [-1  0  1  2  3  4  5  6]
+After cleanup: X=(652, 24, 16, 16), y=(652,) (removed 54 bad samples)
+Remapped labels: [0 1 2 3 4 5 6]
+  Class 0: 65 samples
+  Class 1: 52 samples
+  Class 2: 48 samples
+  Class 3: 72 samples
+  Class 4: 108 samples
+  Class 5: 219 samples
+  Class 6: 88 samples
+
+============================================================
+STRATEGY 5: Flat pixel features + XGBoost (sanity check)
+============================================================
+  Flat features: (652, 6144)
+  ✅ Flat XGBoost acc: 0.7328
+
+============================================================
+STRATEGY 1: Lightweight CNN (no upsampling)
+============================================================
+Device: cuda
+  Epoch 1/300 Loss=1.8379 Acc=0.0763 🌟
+  Epoch 2/300 Loss=1.5825 Acc=0.2824 🌟
+  Epoch 3/300 Loss=1.4412 Acc=0.5649 🌟
+  Epoch 4/300 Loss=1.3274 Acc=0.6336 🌟
+  Epoch 5/300 Loss=1.2893 Acc=0.6870 🌟
+  Epoch 8/300 Loss=1.2140 Acc=0.7099 🌟
+  Epoch 11/300 Loss=1.2224 Acc=0.7252 🌟
+  Epoch 14/300 Loss=1.1087 Acc=0.7328 🌟
+  Epoch 16/300 Loss=1.1081 Acc=0.7710 🌟
+  Epoch 18/300 Loss=1.1847 Acc=0.7939 🌟
+  Epoch 20/300 Loss=1.0316 Acc=0.7786 (patience=2)
+  Epoch 24/300 Loss=1.0465 Acc=0.8092 🌟
+  Epoch 26/300 Loss=0.9583 Acc=0.8397 🌟
+  Epoch 40/300 Loss=0.9361 Acc=0.8626 🌟
+  Epoch 60/300 Loss=0.9807 Acc=0.8092 (patience=20)
+  Epoch 80/300 Loss=0.9459 Acc=0.8015 (patience=40)
+  Epoch 100/300 Loss=0.8443 Acc=0.7939 (patience=60)
+  Early stop at epoch 100
+  ✅ LightCNN best acc: 0.8626
+
+============================================================
+STRATEGY 2: Hybrid CNN embeddings + XGBoost
+============================================================
+  CNN embeddings: (652, 256)
+Extracted 316 rich features per sample
+  Combined features: (652, 572)
+  ✅ Hybrid XGBoost acc: 0.8244
+
+============================================================
+STRATEGY 3: Rich Features + Stacking Ensemble
+============================================================
+Extracted 316 rich features per sample
+  XGBoost: 0.7939
+  LightGBM: 0.7786
+  ExtraTrees: 0.7863
+  RandomForest: 0.7710
+  GBM: 0.7710
+[06:55:44] WARNING: /__w/xgboost/xgboost/src/learner.cc:782: 
+Parameters: { "use_label_encoder" } are not used.
+[06:55:44] WARNING: /__w/xgboost/xgboost/src/learner.cc:782: 
+Parameters: { "use_label_encoder" } are not used.
+
+
+[06:55:44] WARNING: /__w/xgboost/xgboost/src/learner.cc:782: 
+Parameters: { "use_label_encoder" } are not used.
+[06:55:44] WARNING: /__w/xgboost/xgboost/src/learner.cc:782: 
+Parameters: { "use_label_encoder" } are not used.
+
+
+[06:55:44] WARNING: /__w/xgboost/xgboost/src/learner.cc:782: 
+Parameters: { "use_label_encoder" } are not used.
+
+[06:56:17] WARNING: /__w/xgboost/xgboost/src/common/error_msg.cc:62: Falling back to prediction using DMatrix due to mismatched devices. This might lead to higher memory usage and slower performance. XGBoost is running on: cuda:0, while the input data is on: cpu.
+Potential solutions:
+- Use a data structure that matches the device ordinal in the booster.
+- Set the device for booster before call to inplace_predict.
+
+This warning will only be shown once.
+
+  Stacking Ensemble: 0.7710
+  Voting Ensemble: 0.7786
+  ✅ Best ensemble: XGBoost = 0.7939
+Extracted 316 rich features per sample
+
+============================================================
+STRATEGY 4: 5-Fold Stratified Cross-Validation
+============================================================
+  Fold 1: 0.7939
+  Fold 2: 0.8244
+  Fold 3: 0.7769
+  Fold 4: 0.8154
+  Fold 5: 0.7615
+  ✅ CV Mean: 0.7944 ± 0.0234
+
+============================================================
+📊 TỔNG KẾT KẾT QUẢ
+============================================================
+  📈 LightCNN: 0.8626
+  📈 Hybrid CNN+XGBoost: 0.8244
+  📈 CV Mean (XGBoost rich): 0.7944
+  📈 Ensemble XGBoost: 0.7939
+  📈 Ensemble ExtraTrees: 0.7863
+  📈 Ensemble LightGBM: 0.7786
+  📈 Ensemble Voting: 0.7786
+  📈 Ensemble RandomForest: 0.7710
+  📈 Ensemble GBM: 0.7710
+  📈 Ensemble Stacking: 0.7710
+  📈 Flat XGBoost (baseline): 0.7328
+
+🏆 BEST: LightCNN = 0.8626
+
+✅ Kết quả đã được lưu vào model_train/ultimate_results.json
+⚠️ Chưa đạt 95%. Best = 0.8626. Cần thêm dữ liệu hoặc feature engineering.

.temp/log_ultimate_v2.txt

@@ -0,0 +1,69 @@
+🚀 CHIẾN LƯỢC V2: TOÀN DIỆN ĐẠT >95%
+============================================================
+Clean data: (652, 24, 16, 16), 7 classes
+
+============================================================
+CNN + TTA (Test-Time Augmentation)
+============================================================
+  Ep 1 Loss=1.6500 TTA-Acc=0.1450 🌟
+  Ep 2 Loss=1.4633 TTA-Acc=0.3511 🌟
+  Ep 3 Loss=1.3316 TTA-Acc=0.6336 🌟
+  Ep 4 Loss=1.2101 TTA-Acc=0.7252 🌟
+  Ep 6 Loss=1.2056 TTA-Acc=0.8015 🌟
+  Ep 13 Loss=1.1216 TTA-Acc=0.8244 🌟
+  Ep 23 Loss=0.9355 TTA-Acc=0.8321 🌟
+  Ep 30 Loss=0.9346 TTA-Acc=0.8092 (pat=7)
+  Ep 60 Loss=0.9658 TTA-Acc=0.7634 (pat=37)
+  Ep 69 Loss=0.8988 TTA-Acc=0.8397 🌟
+  Ep 74 Loss=0.9159 TTA-Acc=0.8550 🌟
+  Ep 90 Loss=0.8761 TTA-Acc=0.8168 (pat=16)
+  Ep 120 Loss=0.9473 TTA-Acc=0.8092 (pat=46)
+  Ep 139 Loss=0.9317 TTA-Acc=0.8626 🌟
+  Ep 150 Loss=0.7716 TTA-Acc=0.8397 (pat=11)
+  Ep 175 Loss=0.7432 TTA-Acc=0.8702 🌟
+  Ep 180 Loss=0.8290 TTA-Acc=0.8702 (pat=5)
+  Ep 210 Loss=0.8060 TTA-Acc=0.8626 (pat=35)
+  Ep 240 Loss=0.6980 TTA-Acc=0.8473 (pat=65)
+  Early stop ep 255
+  ✅ CNN+TTA best: 0.8702
+
+============================================================
+RICH FEATURES V2 + ENSEMBLE
+============================================================
+Extracted 1713 features per sample
+  XGB: 0.7557
+  LGBM: 0.7634
+  ET: 0.7481
+  RF: 0.7557
+  Voting: 0.7634
+
+  5-Fold CV:
+    Fold 1: 0.7557
+    Fold 2: 0.7939
+    Fold 3: 0.7769
+    Fold 4: 0.8308
+    Fold 5: 0.8000
+  CV: 0.7915 ± 0.0249
+
+============================================================
+HYBRID V2: CNN embed + Rich features + XGBoost
+============================================================
+Extracted 1713 features per sample
+  Combined: (652, 2097)
+  ✅ Hybrid V2: 0.8244
+  CV: 0.9142 ± 0.0194
+
+============================================================
+📊 KẾT QUẢ TỔNG HỢP V2
+============================================================
+  ✅ Hybrid CV: 0.9142
+  📈 CNN+TTA: 0.8702
+  📈 Hybrid V2: 0.8244
+  📈 Ens CV: 0.7915
+  📈 Ens_LGBM: 0.7634
+  📈 Ens_Vote: 0.7634
+  📈 Ens_XGB: 0.7557
+  📈 Ens_RF: 0.7557
+  📈 Ens_ET: 0.7481
+
+🏆 BEST: Hybrid CV = 0.9142

.temp/log_ultimate_v3.txt

@@ -0,0 +1,27 @@
+🚀 V3: MULTI-SEED ENSEMBLE + T0-ONLY + SELF-TRAINING
+============================================================
+Clean: (652, 24, 16, 16), 7 classes, [65, 52, 48, 72, 108, 219, 88]
+
+============================================================
+MULTI-SEED CNN ENSEMBLE (10 models)
+============================================================
+  Seed 0: 0.8473
+  Seed 1: 0.8702
+  Seed 2: 0.8550
+  Seed 3: 0.8550
+  Seed 4: 0.8702
+  Seed 5: 0.8626
+  Seed 6: 0.8702
+  Seed 7: 0.8702
+  Seed 8: 0.8702
+  Seed 9: 0.8702
+  ✅ 10-Model Ensemble TTA: 0.8473
+
+============================================================
+TIMESTEP-0-ONLY XGBoost (cleanest data)
+============================================================
+  T0 valid: 539/652
+  Features: (539, 1638)
+  XGB t0: 0.6852
+  LGBM t0: 0.6296
+  ET t0: 0.6852

.temp/log_xgboost.txt

@@ -0,0 +1,15 @@
+🚀 BẮT ĐẦU HUẤN LUYỆN MÔ HÌNH XGBOOST (GPU & CACHE)
+Initializing FeatureExtractor (mode=extended)...
+📦 Đang load cache: training_data_507bd2ba4ec0d3fe107839cbf73a7a7d.joblib...
+✅ Loaded 632 samples từ cache!
+   ⚡ Đã bỏ qua download위성 data (tiết kiệm thời gian)
+[CACHE HIT] Using cached dataset with 632 samples
+Training XGBOOST model...
+Evaluating model...
+Generating classification report...
+Saving model...
+[MODEL MANAGER] Saving model to: model_train/model_xgboost_auto.joblib
+[MODEL MANAGER] Saving metadata to: model_train/model_xgboost_auto_info.json
+[MODEL MANAGER] Model saved successfully!
+Training complete!
+✅ Hoàn thành! Accuracy: 0.6378

.temp/new_import_ODC.py

@@ -1,8 +1,16 @@
+TEST_MODE = True
+RESOLUTION = 1000 if TEST_MODE else 10
 import matplotlib.pyplot as plt
 
 # Common imports and settings
 import os, sys
 os.environ['USE_PYGEOS'] = '0'
+os.environ["GDAL_HTTP_MAX_RETRY"] = "5"
+os.environ["GDAL_HTTP_RETRY_DELAY"] = "2"
+os.environ["GDAL_HTTP_CONNECTION_TIMEOUT"] = "10"
+os.environ["GDAL_HTTP_TIMEOUT"] = "30"
+os.environ["CPL_VSIL_CURL_ALLOWED_EXTENSIONS"] = ".tif,.tiff"
+os.environ["GDAL_DISABLE_READDIR_ON_OPEN"] = "YES"
 from IPython.display import Markdown
 import pandas as pd
 pd.set_option("display.max_rows", None)
@@ -13,15 +21,13 @@ import datacube
 from datacube.utils.rio import configure_s3_access
 from datacube.utils import masking
 from datacube.utils.cog import write_cog
-# https://github.com/GeoscienceAustralia/dea-notebooks/tree/develop/Tools
-from dea_tools.plotting import display_map, rgb
-from dea_tools.datahandling import mostcommon_crs
+# removed deafrica_tools imports to avoid ipyleaflet error
 
 # EASI defaults
-easinotebooksrepo = '/home/jovyan/easi-notebooks'
+easinotebooksrepo = '/home/x79/CSIROBoeingPhase4-Vietnam'
 if easinotebooksrepo not in sys.path: sys.path.append(easinotebooksrepo)
 from easi_tools import EasiDefaults, xarray_object_size, notebook_utils, unset_cachingproxy
-from easi_tools.load_s2l2a import load_s2l2a_with_offset
+# from easi_tools.load_s2l2a import load_s2l2a_with_offset
 from dask.distributed import progress
 
 # Data tools
@@ -31,7 +37,7 @@ from datetime import datetime
 # Datacube
 from datacube.utils import masking  # https://github.com/opendatacube/datacube-core/blob/develop/datacube/utils/masking.py
 from odc.algo import enum_to_bool   # https://github.com/opendatacube/odc-algo/blob/main/odc/algo/_masking.py
-from odc.algo import xr_reproject   # https://github.com/opendatacube/odc-algo/blob/main/odc/algo/_warp.py
+# removed xr_reproject
 from datacube.utils.geometry import GeoBox, box  # https://github.com/opendatacube/datacube-core/blob/develop/datacube/utils/geometry/_base.py
 
 # Holoviews, Datashader and Bokeh
@@ -51,13 +57,12 @@ import rioxarray
 hv.extension('bokeh', logo=False)
 
 from deafrica_tools.bandindices import calculate_indices
-from sklearn.ensemble import RandomForestClassifier
+from xgboost import XGBClassifier
 from sklearn.model_selection import train_test_split
 from sklearn.metrics import accuracy_score, classification_report
 from sklearn.preprocessing import LabelEncoder
 
 from sklearn.pipeline import Pipeline
-from sklearn.ensemble import RandomForestClassifier
 from sklearn.impute import SimpleImputer
 from sklearn.preprocessing import StandardScaler
 from sklearn.model_selection import GridSearchCV
@@ -82,57 +87,90 @@ import joblib
 
 
 def load_data(dc, date_range, longtitude_range, latitude_range):
-    product = 's2_l2a'
-    query = {
-        'product': product,                     # Product name
-        'x': longtitude_range,    # "x" axis bounds
-        'y': latitude_range,      # "y" axis bounds
-        'time': date_range,           # Any parsable date strings
-    }
-    native_crs = notebook_utils.mostcommon_crs(dc, query)
-    print(f'Most common native CRS: {native_crs}')
-    measurements = ['red', 'nir', 'scl']
+    import os, hashlib
+    cache_dir = "dataset_cache"
+    os.makedirs(cache_dir, exist_ok=True)
+    key_str = f"s2_{date_range}_{longtitude_range}_{latitude_range}"
+    cache_key = hashlib.md5(key_str.encode()).hexdigest() + ".nc"
+    cache_path = os.path.join(cache_dir, cache_key)
+    
+    if os.path.exists(cache_path):
+        print(f"✅ Loading cached S2 data from {cache_path}")
+        return xr.open_dataset(cache_path, engine='netcdf4')
 
-    load_params = {
-        'measurements': measurements,                   # Selected measurement or alias names
-        'output_crs': native_crs,                       # Target EPSG code
-        'resolution': (-10, 10),                        # Target resolution
-        'group_by': 'solar_day',                        # Scene grouping
-        'dask_chunks': {'x': 2048, 'y': 2048},          # Dask chunks
-    }
-    data = load_s2l2a_with_offset(
-        dc,
-        query | load_params   # Combine the two dicts that contain our search and load parameters
+    product = 's2_l2a'
+    bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
+    
+    import pystac_client
+    import planetary_computer
+    import odc.stac
+    
+    catalog = pystac_client.Client.open(
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
+        modifier=planetary_computer.sign_inplace,
     )
+    search = catalog.search(
+        collections=["sentinel-2-l2a"],
+        bbox=bbox,
+        datetime=f"{date_range[0]}/{date_range[1]}",
+    )
+    items = list(search.items())
+    
+    data = odc.stac.load(
+        items,
+        bands=["red", "nir", "SCL"],
+        bbox=bbox,
+        crs="EPSG:32648",
+        resolution=RESOLUTION,
+        chunks={"x": 2048, "y": 2048, "time": 1},
+        groupby="solar_day"
+    )
+    if "SCL" in data.data_vars:
+        data = data.rename({"SCL": "scl"})
+        
+    print(f"💾 Caching S2 data to {cache_path}")
+    data = data.compute()
+    data.to_netcdf(cache_path, engine='netcdf4')
+    
     return data
 
 
 def mask_clean(data):
-    flag_name = 'scl'
-    flag_desc = masking.describe_variable_flags(data[flag_name])  # Pandas dataframe
-    display(flag_desc)
-    display(flag_desc.loc['qa'].values[1])
-    # Create a "data quality" Mask layer
-    flags_def = flag_desc.loc['qa'].values[1]
-    good_pixel_flags = [flags_def[str(i)] for i in [2, 4, 5, 6]]  # To pass strings to enum_to_bool()
-
-    # enum_to_bool calculates the pixel-wise "or" of each set of pixels given by good_pixel_flags
-    # 1 = good data
-    # 0 = "bad" data
-    good_pixel_mask = enum_to_bool(data[flag_name], good_pixel_flags)
+    # For Sentinel-2 L2A SCL: 
+    # 2: Dark Area Pixels, 4: Vegetation, 5: Not Vegetated, 6: Water
+    good_pixel_mask = data['scl'].isin([2, 4, 5, 6])
     data_layer_names = [x for x in data.data_vars if x != 'scl']
-    # Apply good pixel mask to blue, green, red and nir.
+    # Apply good pixel mask
     result = data[data_layer_names].where(good_pixel_mask).persist()
     return result
 
 
 def fill_nan(ndvi, time_split):
+    if len(ndvi.time) == 0:
+        return ndvi
+    
+    # If the total time duration is less than 90 days, skip seasonal splitting
+    try:
+        total_days = (ndvi.time[-1] - ndvi.time[0]).dt.days.item()
+        if total_days < 90:
+            return ndvi.bfill(dim="time").ffill(dim="time")
+    except Exception:
+        pass
+        
     rs = []
     for times in time_split:
-        tmp = ndvi.sel(time=times)
-        fill_ds = tmp.sel(time=times).bfill(dim='time')
-        fill_ds = fill_ds.sel(time=times).ffill(dim='time')   
-        rs.append(fill_ds)
+        try:
+            tmp = ndvi.sel(time=times)
+            if len(tmp.time) == 0:
+                continue
+            fill_ds = tmp.bfill(dim='time').ffill(dim='time')   
+            rs.append(fill_ds)
+        except Exception:
+            continue
+            
+    if len(rs) == 0:
+        return ndvi.bfill(dim="time").ffill(dim="time")
+        
     merged_ndvi = xr.concat([i for i in rs], dim="time")
     fill_m = merged_ndvi.bfill(dim="time")
     fill_m = fill_m.ffill(dim="time")
@@ -144,10 +182,68 @@ def load_train_data(train_path):
     return train
 
 
-def load_sen1(name_vh, name_vv):
-    dsvv = rioxarray.open_rasterio(name_vv)
-    dsvh = rioxarray.open_rasterio(name_vh)
-    return dsvh, dsvv
+def load_sen1(bbox, time_range):
+    import os, hashlib
+    cache_dir = "dataset_cache"
+    os.makedirs(cache_dir, exist_ok=True)
+    key_str = f"s1_vh_vv_{bbox}_{time_range}"
+    cache_key_vh = hashlib.md5((key_str + "vh").encode()).hexdigest() + ".nc"
+    cache_key_vv = hashlib.md5((key_str + "vv").encode()).hexdigest() + ".nc"
+    cache_path_vh = os.path.join(cache_dir, cache_key_vh)
+    cache_path_vv = os.path.join(cache_dir, cache_key_vv)
+    
+    if os.path.exists(cache_path_vh) and os.path.exists(cache_path_vv):
+        print(f"✅ Loading cached S1 data from {cache_path_vh} and {cache_path_vv}")
+        ds_vh = xr.open_dataset(cache_path_vh, engine='netcdf4')
+        ds_vv = xr.open_dataset(cache_path_vv, engine='netcdf4')
+        return ds_vh[list(ds_vh.data_vars)[0]], ds_vv[list(ds_vv.data_vars)[0]]
+
+    import pystac_client
+    import planetary_computer
+    import odc.stac
+    
+    # Kết nối STAC Client
+    catalog = pystac_client.Client.open(
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
+        modifier=planetary_computer.sign_inplace,
+    )
+
+    # Tìm kiếm Items
+    search = catalog.search(
+        collections=["sentinel-1-rtc"],
+        bbox=bbox,
+        datetime=time_range,
+    )
+    items = list(search.items())
+
+    # Tải dữ liệu thành xarray Dataset
+    ds_s1 = odc.stac.load(
+        items,
+        bands=["vv", "vh"],
+        bbox=bbox,
+        crs="EPSG:32648",
+        resolution=RESOLUTION,
+        chunks={"x": 2048, "y": 2048, "time": 1}
+    )
+
+    # Tính giá trị trung vị theo thời gian
+    ds_median = ds_s1.median(dim="time").compute()
+    vv = ds_median["vv"]
+    vh = ds_median["vh"]
+
+    # Thêm chiều 'band' để giống hệt rioxarray
+    vv = vv.expand_dims(dim="band")
+    vh = vh.expand_dims(dim="band")
+
+    # Phục hồi metadata về toạ độ
+    vv = vv.rio.write_crs("EPSG:32648")
+    vh = vh.rio.write_crs("EPSG:32648")
+
+    print(f"💾 Caching S1 data to {cache_dir}")
+    vh.to_netcdf(cache_path_vh, engine='netcdf4')
+    vv.to_netcdf(cache_path_vv, engine='netcdf4')
+
+    return vh, vv
 
 
 def get_data_sen1_and_sen2(train, average_ndvi, dsvh, dsvv):
@@ -192,7 +288,7 @@ def train_with_rf(X_train, X_val, y_train, y_val):
     # Takes 1-2 minutes to complete
 
     # Tạo RandomForestClassifier mặc định để sử dụng làm mô hình ban đầu trong pipeline
-    base_model = RandomForestClassifier(random_state=42, n_jobs=-1)
+    base_model = XGBClassifier(tree_method="hist", device="cuda", random_state=42, n_jobs=-1)
 
     # Tạo pipeline
     pipeline = Pipeline([
@@ -204,7 +300,7 @@ def train_with_rf(X_train, X_val, y_train, y_val):
     param_grid = {
         'classifier__n_estimators': [100, 300, 500, 700, 1000],
         'classifier__max_depth': [6, 8, 10, 15, 20],
-        'classifier__criterion': ['gini', 'entropy'],
+        'classifier__learning_rate': [0.01, 0.1, 0.2],
     }
 
     # Sử dụng GridSearchCV để tìm bộ tham số tốt nhất
@@ -224,15 +320,53 @@ def train_with_rf(X_train, X_val, y_train, y_val):
     return grid_search
 
 
-def save_model(name_file, grid_search):
+def save_model(name_file, model, metadata=None, label_encoder=None):
+    """
+    Save model với metadata để tương thích với ModelManager
+    
+    Args:
+        name_file: Tên file model
+        model: Model object
+        metadata: Dict chứa thông tin về model (optional)
+        label_encoder: Label encoder (optional)
+    """
+    from model_manager import get_model_manager
+    
     dir_save_model = "model_train"
     if not os.path.exists(dir_save_model):
         os.mkdir(dir_save_model)
-    joblib.dump(grid_search, os.path.join(dir_save_model, name_file))
-    print("Done!")
+    
+    # Nếu có metadata, sử dụng ModelManager
+    if metadata is not None:
+        model_manager = get_model_manager()
+        model_manager.save_model(
+            model=model,
+            metadata=metadata,
+            model_filename=name_file,
+            label_encoder=label_encoder
+        )
+    else:
+        # Legacy mode: save trực tiếp (backward compatibility)
+        model_data = {
+            'model': model,
+            'label_encoder': label_encoder
+        } if label_encoder is not None else model
+        
+        joblib.dump(model_data, os.path.join(dir_save_model, name_file))
+    
+    print(f"✅ Model saved: {name_file}")
+    if metadata:
+        print(f"   - Type: {metadata.get('model_type', 'N/A')}")
+        print(f"   - Features: {metadata.get('n_features', 'N/A')}")
+        print(f"   - Accuracy: {metadata.get('test_accuracy', 'N/A')}")
+
     
     
 def predict(model, data_crs, ndvi, vh, vv):
+    # Unpack model if it is wrapped in a dictionary (from ModelManager)
+    if isinstance(model, dict) and 'model' in model:
+        model = model['model']
+        
     data_predict = []
     for i in range(ndvi.shape[1]):
         ndvi_tmp = ndvi.isel(y=i).values
@@ -326,25 +460,59 @@ def save_result(result, HT_MAP):
         # plt.show()
         
 def load_data_sen1(dc, date_range, coordinates):
+    import os, hashlib
     longtitude_range, latitude_range = coordinates
-    data_sen1 = dc.load(
-        product="sentinel1_grd_gamma0_10m",
-        x=longtitude_range,
-        y=latitude_range,
-        time=date_range,
-        measurements=["vv", "vh"],
-        output_crs="EPSG:32648",
-        resolution=(-10,10),
-        dask_chunks={"x":2048, "y":2048},
-        skip_broken_datasets=True,
-        group_by='solar_day'
+    bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
+    
+    cache_dir = "dataset_cache"
+    os.makedirs(cache_dir, exist_ok=True)
+    key_str = f"data_sen1_{date_range}_{bbox}"
+    cache_key_vh = hashlib.md5((key_str + "vh").encode()).hexdigest() + ".nc"
+    cache_key_vv = hashlib.md5((key_str + "vv").encode()).hexdigest() + ".nc"
+    cache_path_vh = os.path.join(cache_dir, cache_key_vh)
+    cache_path_vv = os.path.join(cache_dir, cache_key_vv)
+    
+    if os.path.exists(cache_path_vh) and os.path.exists(cache_path_vv):
+        print(f"✅ Loading cached S1 (coord) data")
+        ds_vh = xr.open_dataset(cache_path_vh, engine='netcdf4')
+        ds_vv = xr.open_dataset(cache_path_vv, engine='netcdf4')
+        var_vh = [v for v in ds_vh.data_vars if v != 'spatial_ref'][0]
+        var_vv = [v for v in ds_vv.data_vars if v != 'spatial_ref'][0]
+        return ds_vh[var_vh], ds_vv[var_vv]
+    
+    import pystac_client
+    import planetary_computer
+    import odc.stac
+    
+    catalog = pystac_client.Client.open(
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
+        modifier=planetary_computer.sign_inplace,
+    )
+    search = catalog.search(
+        collections=["sentinel-1-rtc"],
+        bbox=bbox,
+        datetime=f"{date_range[0]}/{date_range[1]}",
+    )
+    items = list(search.items())
+    
+    data_sen1 = odc.stac.load(
+        items,
+        bands=["vv", "vh"],
+        bbox=bbox,
+        crs="EPSG:32648",
+        resolution=RESOLUTION,
+        chunks={"x": 2048, "y": 2048, "time": 1},
+        groupby="solar_day"
     )
     
-    notebook_utils.heading(notebook_utils.xarray_object_size(data_sen1))
-    display(data_sen1)
+    data_sen1 = data_sen1.compute()
     dsvh = data_sen1.vh
     dsvv = data_sen1.vv
     
+    print(f"💾 Caching S1 (coord) data")
+    dsvh.to_netcdf(cache_path_vh, engine='netcdf4')
+    dsvv.to_netcdf(cache_path_vv, engine='netcdf4')
+    
     return dsvh, dsvv
     
 def calculate_average(data, time_pattern='1M'):
@@ -352,54 +520,66 @@ def calculate_average(data, time_pattern='1M'):
 
 
 def load_data_sen2(dc, date_range, coordinates):
+    import os, hashlib
     longtitude_range, latitude_range = coordinates
-    product = 's2_l2a'
-    query = {
-        'product': product,                     # Product name
-        'x': longtitude_range,    # "x" axis bounds
-        'y': latitude_range,      # "y" axis bounds
-        'time': date_range,           # Any parsable date strings
-    }
-    native_crs = notebook_utils.mostcommon_crs(dc, query)
-    print(f'Most common native CRS: {native_crs}')
+    bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
     
-    # measurements = ['red','green', 'blue', 'nir', 'scl']
-    measurements = ['red', 'nir', 'scl']
-
-    load_params = {
-        'measurements': measurements,                   # Selected measurement or alias names
-        'output_crs': native_crs,                       # Target EPSG code
-        'resolution': (-10, 10),                        # Target resolution
-        'group_by': 'solar_day',                        # Scene grouping
-        'dask_chunks': {'x': 2048, 'y': 2048},          # Dask chunks
-    }
-    data = load_s2l2a_with_offset(
-        dc,
-        query | load_params   # Combine the two dicts that contain our search and load parameters
+    cache_dir = "dataset_cache"
+    os.makedirs(cache_dir, exist_ok=True)
+    key_str = f"data_sen2_{date_range}_{bbox}"
+    cache_key = hashlib.md5(key_str.encode()).hexdigest() + ".nc"
+    cache_path = os.path.join(cache_dir, cache_key)
+    
+    if os.path.exists(cache_path):
+        print(f"✅ Loading cached S2 (coord) data from {cache_path}")
+        return xr.open_dataset(cache_path, engine='netcdf4')
+    
+    import pystac_client
+    import planetary_computer
+    import odc.stac
+    
+    catalog = pystac_client.Client.open(
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
+        modifier=planetary_computer.sign_inplace,
     )
+    search = catalog.search(
+        collections=["sentinel-2-l2a"],
+        bbox=bbox,
+        datetime=f"{date_range[0]}/{date_range[1]}",
+    )
+    items = list(search.items())
+    
+    data = odc.stac.load(
+        items,
+        bands=["red", "nir", "SCL"],
+        bbox=bbox,
+        crs="EPSG:32648",
+        resolution=RESOLUTION,
+        chunks={"x": 2048, "y": 2048, "time": 1},
+        groupby="solar_day"
+    )
+    if "SCL" in data.data_vars:
+        data = data.rename({"SCL": "scl"})
+        
+    data = data.compute()
+    print(f"💾 Caching S2 (coord) data to {cache_path}")
+    data.to_netcdf(cache_path, engine='netcdf4')
+    
     return data
 
 def mask_cloud(data):
-    flag_name = 'scl'
-    flag_desc = masking.describe_variable_flags(data[flag_name])  # Pandas dataframe
-    display(flag_desc.loc['qa'].values[1])
-    # Create a "data quality" Mask layer
-    flags_def = flag_desc.loc['qa'].values[1]
-    good_pixel_flags = [flags_def[str(i)] for i in [2, 4, 5, 6]]  # To pass strings to enum_to_bool()
-
-    # enum_to_bool calculates the pixel-wise "or" of each set of pixels given by good_pixel_flags
-    # 1 = good data
-    # 0 = "bad" data
-    good_pixel_mask = enum_to_bool(data[flag_name], good_pixel_flags)
+    # For Sentinel-2 L2A SCL: 
+    # 2: Dark Area Pixels, 4: Vegetation, 5: Not Vegetated, 6: Water
+    good_pixel_mask = data['scl'].isin([2, 4, 5, 6])
     data_layer_names = [x for x in data.data_vars if x != 'scl']
-    # Apply good pixel mask to blue, green, red and nir.
+    # Apply good pixel mask
     result = data[data_layer_names].where(good_pixel_mask).persist()
     return result
 
 def find_best_model(dataset):
     X_train, X_val, y_train, y_val = dataset
     # Tạo RandomForestClassifier mặc định để sử dụng làm mô hình ban đầu trong pipeline
-    base_model = RandomForestClassifier(random_state=42, n_jobs=-1)
+    base_model = XGBClassifier(tree_method="hist", device="cuda", random_state=42, n_jobs=-1)
 
     # Tạo pipeline
     pipeline = Pipeline([
@@ -411,7 +591,7 @@ def find_best_model(dataset):
     param_grid = {
         'classifier__n_estimators': [100, 300, 500, 700, 1000],
         'classifier__max_depth': [6, 8, 10, 15, 20],
-        'classifier__criterion': ['gini', 'entropy'],
+        'classifier__learning_rate': [0.01, 0.1, 0.2],
     }
 
     # Sử dụng GridSearchCV để tìm bộ tham số tốt nhất

.temp/patch_caching.py

@@ -0,0 +1,231 @@
+import re
+import os
+
+with open('/home/x79/remote-sensing/new_import_ODC.py', 'r', encoding='utf-8') as f:
+    content = f.read()
+
+# 1. load_data
+load_data_replacement = """def load_data(dc, date_range, longtitude_range, latitude_range):
+    import os, hashlib
+    cache_dir = "dataset_cache"
+    os.makedirs(cache_dir, exist_ok=True)
+    key_str = f"s2_{date_range}_{longtitude_range}_{latitude_range}"
+    cache_key = hashlib.md5(key_str.encode()).hexdigest() + ".nc"
+    cache_path = os.path.join(cache_dir, cache_key)
+    
+    if os.path.exists(cache_path):
+        print(f"✅ Loading cached S2 data from {cache_path}")
+        return xr.open_dataset(cache_path)
+
+    product = 's2_l2a'
+    bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
+    
+    import pystac_client
+    import planetary_computer
+    import odc.stac
+    
+    catalog = pystac_client.Client.open(
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
+        modifier=planetary_computer.sign_inplace,
+    )
+    search = catalog.search(
+        collections=["sentinel-2-l2a"],
+        bbox=bbox,
+        datetime=f"{date_range[0]}/{date_range[1]}",
+    )
+    items = list(search.items())
+    
+    data = odc.stac.load(
+        items,
+        bands=["red", "nir", "SCL"],
+        bbox=bbox,
+        crs="EPSG:32648",
+        resolution=RESOLUTION,
+        chunks={"x": 2048, "y": 2048, "time": 1},
+        groupby="solar_day"
+    )
+    if "SCL" in data.data_vars:
+        data = data.rename({"SCL": "scl"})
+        
+    print(f"💾 Caching S2 data to {cache_path}")
+    data = data.compute()
+    data.to_netcdf(cache_path)
+    
+    return data"""
+content = re.sub(r'def load_data\(dc, date_range, longtitude_range, latitude_range\):.*?return data', load_data_replacement, content, flags=re.DOTALL)
+
+
+# 2. load_sen1
+load_sen1_replacement = """def load_sen1(bbox, time_range):
+    import os, hashlib
+    cache_dir = "dataset_cache"
+    os.makedirs(cache_dir, exist_ok=True)
+    key_str = f"s1_vh_vv_{bbox}_{time_range}"
+    cache_key_vh = hashlib.md5((key_str + "vh").encode()).hexdigest() + ".nc"
+    cache_key_vv = hashlib.md5((key_str + "vv").encode()).hexdigest() + ".nc"
+    cache_path_vh = os.path.join(cache_dir, cache_key_vh)
+    cache_path_vv = os.path.join(cache_dir, cache_key_vv)
+    
+    if os.path.exists(cache_path_vh) and os.path.exists(cache_path_vv):
+        print(f"✅ Loading cached S1 data from {cache_path_vh} and {cache_path_vv}")
+        return xr.open_dataarray(cache_path_vh), xr.open_dataarray(cache_path_vv)
+
+    import pystac_client
+    import planetary_computer
+    import odc.stac
+    
+    # Kết nối STAC Client
+    catalog = pystac_client.Client.open(
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
+        modifier=planetary_computer.sign_inplace,
+    )
+
+    # Tìm kiếm Items
+    search = catalog.search(
+        collections=["sentinel-1-rtc"],
+        bbox=bbox,
+        datetime=time_range,
+    )
+    items = list(search.items())
+
+    # Tải dữ liệu thành xarray Dataset
+    ds_s1 = odc.stac.load(
+        items,
+        bands=["vv", "vh"],
+        bbox=bbox,
+        crs="EPSG:32648",
+        resolution=RESOLUTION,
+        chunks={"x": 2048, "y": 2048, "time": 1}
+    )
+
+    # Tính giá trị trung vị theo thời gian
+    ds_median = ds_s1.median(dim="time").compute()
+    vv = ds_median["vv"]
+    vh = ds_median["vh"]
+
+    # Thêm chiều 'band' để giống hệt rioxarray
+    vv = vv.expand_dims(dim="band")
+    vh = vh.expand_dims(dim="band")
+
+    # Phục hồi metadata về toạ độ
+    vv = vv.rio.write_crs("EPSG:32648")
+    vh = vh.rio.write_crs("EPSG:32648")
+
+    print(f"💾 Caching S1 data to {cache_dir}")
+    vh.to_netcdf(cache_path_vh)
+    vv.to_netcdf(cache_path_vv)
+
+    return vh, vv"""
+content = re.sub(r'def load_sen1\(bbox, time_range\):.*?return vh, vv', load_sen1_replacement, content, flags=re.DOTALL)
+
+
+# 3. load_data_sen1
+load_data_sen1_replacement = """def load_data_sen1(dc, date_range, coordinates):
+    import os, hashlib
+    longtitude_range, latitude_range = coordinates
+    bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
+    
+    cache_dir = "dataset_cache"
+    os.makedirs(cache_dir, exist_ok=True)
+    key_str = f"data_sen1_{date_range}_{bbox}"
+    cache_key_vh = hashlib.md5((key_str + "vh").encode()).hexdigest() + ".nc"
+    cache_key_vv = hashlib.md5((key_str + "vv").encode()).hexdigest() + ".nc"
+    cache_path_vh = os.path.join(cache_dir, cache_key_vh)
+    cache_path_vv = os.path.join(cache_dir, cache_key_vv)
+    
+    if os.path.exists(cache_path_vh) and os.path.exists(cache_path_vv):
+        print(f"✅ Loading cached S1 (coord) data")
+        return xr.open_dataarray(cache_path_vh), xr.open_dataarray(cache_path_vv)
+    
+    import pystac_client
+    import planetary_computer
+    import odc.stac
+    
+    catalog = pystac_client.Client.open(
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
+        modifier=planetary_computer.sign_inplace,
+    )
+    search = catalog.search(
+        collections=["sentinel-1-rtc"],
+        bbox=bbox,
+        datetime=f"{date_range[0]}/{date_range[1]}",
+    )
+    items = list(search.items())
+    
+    data_sen1 = odc.stac.load(
+        items,
+        bands=["vv", "vh"],
+        bbox=bbox,
+        crs="EPSG:32648",
+        resolution=RESOLUTION,
+        chunks={"x": 2048, "y": 2048, "time": 1},
+        groupby="solar_day"
+    )
+    
+    data_sen1 = data_sen1.compute()
+    dsvh = data_sen1.vh
+    dsvv = data_sen1.vv
+    
+    print(f"💾 Caching S1 (coord) data")
+    dsvh.to_netcdf(cache_path_vh)
+    dsvv.to_netcdf(cache_path_vv)
+    
+    return dsvh, dsvv"""
+content = re.sub(r'def load_data_sen1\(dc, date_range, coordinates\):.*?return dsvh, dsvv', load_data_sen1_replacement, content, flags=re.DOTALL)
+
+
+# 4. load_data_sen2
+load_data_sen2_replacement = """def load_data_sen2(dc, date_range, coordinates):
+    import os, hashlib
+    longtitude_range, latitude_range = coordinates
+    bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
+    
+    cache_dir = "dataset_cache"
+    os.makedirs(cache_dir, exist_ok=True)
+    key_str = f"data_sen2_{date_range}_{bbox}"
+    cache_key = hashlib.md5(key_str.encode()).hexdigest() + ".nc"
+    cache_path = os.path.join(cache_dir, cache_key)
+    
+    if os.path.exists(cache_path):
+        print(f"✅ Loading cached S2 (coord) data from {cache_path}")
+        return xr.open_dataset(cache_path)
+    
+    import pystac_client
+    import planetary_computer
+    import odc.stac
+    
+    catalog = pystac_client.Client.open(
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
+        modifier=planetary_computer.sign_inplace,
+    )
+    search = catalog.search(
+        collections=["sentinel-2-l2a"],
+        bbox=bbox,
+        datetime=f"{date_range[0]}/{date_range[1]}",
+    )
+    items = list(search.items())
+    
+    data = odc.stac.load(
+        items,
+        bands=["red", "nir", "SCL"],
+        bbox=bbox,
+        crs="EPSG:32648",
+        resolution=RESOLUTION,
+        chunks={"x": 2048, "y": 2048, "time": 1},
+        groupby="solar_day"
+    )
+    if "SCL" in data.data_vars:
+        data = data.rename({"SCL": "scl"})
+        
+    data = data.compute()
+    print(f"💾 Caching S2 (coord) data to {cache_path}")
+    data.to_netcdf(cache_path)
+    
+    return data"""
+content = re.sub(r'def load_data_sen2\(dc, date_range, coordinates\):.*?return data', load_data_sen2_replacement, content, flags=re.DOTALL)
+
+
+with open('/home/x79/remote-sensing/new_import_ODC.py', 'w', encoding='utf-8') as f:
+    f.write(content)
+
+print("Patching successful.")

.temp/patch_load_sen1.py

@@ -0,0 +1,29 @@
+import json
+import glob
+
+def fix_load_sen1(file_path):
+    with open(file_path, 'r', encoding='utf-8') as f:
+        nb = json.load(f)
+        
+    changed = False
+    for cell in nb.get('cells', []):
+        if cell.get('cell_type') == 'code':
+            source = cell.get('source', [])
+            for i, line in enumerate(source):
+                if 'load_sen1(name_vh, name_vv)' in line:
+                    indent = line[:len(line) - len(line.lstrip())]
+                    replacement = (
+                        f"{indent}bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]\n"
+                        f"{indent}time_range = f'{{date_range[0]}}/{{date_range[1]}}'\n"
+                        f"{indent}{line.lstrip().replace('load_sen1(name_vh, name_vv)', 'load_sen1(bbox, time_range)')}"
+                    )
+                    source[i] = replacement
+                    changed = True
+                    
+    if changed:
+        with open(file_path, 'w', encoding='utf-8') as f:
+            json.dump(nb, f, indent=1)
+        print(f"Patched load_sen1 in {file_path}")
+
+for nb in glob.glob("*.ipynb"):
+    fix_load_sen1(nb)

.temp/patch_notebooks.py

@@ -0,0 +1,43 @@
+import json
+import glob
+
+def patch_notebook(file_path):
+    with open(file_path, 'r', encoding='utf-8') as f:
+        nb = json.load(f)
+        
+    changed = False
+    for cell in nb.get('cells', []):
+        if cell.get('cell_type') == 'code':
+            source = cell.get('source', [])
+            
+            # Check if this cell should be fully commented out
+            full_source = ''.join(source)
+            if 'dc.load(' in full_source or 'ds.vv' in full_source:
+                for i in range(len(source)):
+                    if not source[i].startswith('#'):
+                        source[i] = '# ' + source[i]
+                changed = True
+                continue
+            
+            # Otherwise, do line-by-line replacements
+            for i, line in enumerate(source):
+                if 'ST_training data_updated_1130points.shp' in line:
+                    source[i] = line.replace('ST_training data_updated_1130points.shp', 'ST_training_data_updated_1130points.shp')
+                    changed = True
+                if 'from new_import import *' in line:
+                    source[i] = line.replace('from new_import import *', 'from new_import_ODC import *')
+                    changed = True
+                if 'dc = datacube.Datacube()' in line:
+                    source[i] = line.replace('dc = datacube.Datacube()', 'dc = None')
+                    changed = True
+                if 'load_data(dc,' in line:
+                    source[i] = line.replace('load_data(dc,', 'load_data(None,')
+                    changed = True
+                    
+    if changed:
+        with open(file_path, 'w', encoding='utf-8') as f:
+            json.dump(nb, f, indent=1)
+        print(f"Patched {file_path}")
+
+for nb in glob.glob("*.ipynb"):
+    patch_notebook(nb)

.temp/patch_rf_to_xgb.py

@@ -0,0 +1,75 @@
+import json
+import glob
+import re
+
+def patch_python_script(filepath):
+    try:
+        with open(filepath, 'r', encoding='utf-8') as f:
+            content = f.read()
+            
+        original_content = content
+        
+        # Replace imports
+        content = re.sub(r'from sklearn\.ensemble import RandomForestClassifier', 
+                         'from xgboost import XGBClassifier', content)
+        
+        # Replace the model instantiations (for 01.train_ODC.py)
+        rf_pattern = re.compile(r'model\s*=\s*RandomForestClassifier\([^)]+\)', re.DOTALL)
+        xgb_replacement = """model = XGBClassifier(
+                    n_estimators=200,
+                    max_depth=30,
+                    tree_method="hist",
+                    device="cuda",
+                    random_state=42,
+                    n_jobs=-1,
+                    verbosity=1
+                )"""
+        
+        content = rf_pattern.sub(xgb_replacement, content)
+        
+        if content != original_content:
+            with open(filepath, 'w', encoding='utf-8') as f:
+                f.write(content)
+            print(f"Patched {filepath}")
+    except Exception as e:
+        print(f"Error patching {filepath}: {e}")
+
+def patch_notebook(filepath):
+    try:
+        with open(filepath, 'r', encoding='utf-8') as f:
+            nb = json.load(f)
+            
+        changed = False
+        import_pattern = re.compile(r'from\s+sklearn\.ensemble\s+import\s+RandomForestClassifier')
+        inst_pattern = re.compile(r'RandomForestClassifier\([^)]*\)')
+        
+        for cell in nb.get('cells', []):
+            if cell.get('cell_type') == 'code':
+                source = cell.get('source', [])
+                for i in range(len(source)):
+                    if import_pattern.search(source[i]):
+                        source[i] = import_pattern.sub('from xgboost import XGBClassifier', source[i])
+                        changed = True
+                        
+                    if inst_pattern.search(source[i]):
+                        source[i] = inst_pattern.sub("XGBClassifier(tree_method='hist', device='cuda', random_state=42, n_jobs=-1)", source[i])
+                        changed = True
+                        
+                    if "'classifier__criterion': ['gini', 'entropy']" in source[i]:
+                        source[i] = source[i].replace("'classifier__criterion': ['gini', 'entropy']",
+                                                      "'classifier__learning_rate': [0.01, 0.1, 0.2]")
+                        changed = True
+                        
+        if changed:
+            with open(filepath, 'w', encoding='utf-8') as f:
+                json.dump(nb, f, indent=1)
+            print(f"Patched {filepath}")
+    except Exception as e:
+        print(f"Error patching {filepath}: {e}")
+
+if __name__ == "__main__":
+    patch_python_script("01.train_ODC.py")
+    patch_python_script("new_train.py")
+    
+    for nb in glob.glob("*.ipynb"):
+        patch_notebook(nb)

.temp/processMask-test.ipynb

@@ -1132,7 +1132,7 @@
    "source": [
     "%%time\n",
     "%matplotlib inline\n",
-    "from new_import import *"
+    "from new_import_ODC import *"
    ]
   },
   {
@@ -1216,7 +1216,7 @@
     "%%time\n",
     "# Dask gateway\n",
     "cluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))\n",
-    "dc = datacube.Datacube()\n",
+    "dc = None\n",
     "\n",
     "# Configure s3 access\n",
     "configure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n",
@@ -1233,11 +1233,11 @@
    },
    "outputs": [],
    "source": [
-    "## cấu hình thời gian lấy ảnh và tọa độ\n",
-    "# date_range = ('2022-09-01', '2023-10-01')\n",
+    "## c\u1ea5u h\u00ecnh th\u1eddi gian l\u1ea5y \u1ea3nh v\u00e0 t\u1ecda \u0111\u1ed9\n",
+    "# date_range = ('2022-09-01', '2022-10-01')\n",
     "# longtitude_range = (105.86575, 105.94120)\n",
     "# latitude_range = (9.65070, 9.69850)\n",
-    "date_range = ('2022-09-01', '2023-10-01')\n",
+    "date_range = ('2022-09-01', '2022-10-01')\n",
     "longtitude_range = (105.5, 106.4)\n",
     "latitude_range = (9.2, 10.0) "
    ]
@@ -1396,7 +1396,7 @@
        "\n",
        ".xr-section-summary-in + label:before {\n",
        "  display: inline-block;\n",
-       "  content: '►';\n",
+       "  content: '\u25ba';\n",
        "  font-size: 11px;\n",
        "  width: 15px;\n",
        "  text-align: center;\n",
@@ -1407,7 +1407,7 @@
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label:before {\n",
-       "  content: '▼';\n",
+       "  content: '\u25bc';\n",
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label > span {\n",
@@ -2424,8 +2424,8 @@
     }
    ],
    "source": [
-    "## truy vấn ảnh vệ tinh sen2\n",
-    "data = load_data(dc, date_range, longtitude_range, latitude_range)\n",
+    "## truy v\u1ea5n \u1ea3nh v\u1ec7 tinh sen2\n",
+    "data = load_data(None, date_range, longtitude_range, latitude_range)\n",
     "notebook_utils.heading(notebook_utils.xarray_object_size(data))\n",
     "display(data)"
    ]
@@ -2440,10 +2440,10 @@
    "outputs": [],
    "source": [
     "# Specify the start and end times \n",
-    "min_date = '2022-09-01' # Thời gian bắt đầu lấy data cho quá trình train\n",
-    "max_date = '2023-10-01' # Thời gian kết thúc lấy data cho quá trình train\n",
+    "min_date = '2022-09-01' # Th\u1eddi gian b\u1eaft \u0111\u1ea7u l\u1ea5y data cho qu\u00e1 tr\u00ecnh train\n",
+    "max_date = '2022-10-01' # Th\u1eddi gian k\u1ebft th\u00fac l\u1ea5y data cho qu\u00e1 tr\u00ecnh train\n",
     "# Just do 1 month for testing\n",
-    "# max_date = '2022-10-01' # Thời gian kết thúc lấy data cho quá trình train\n",
+    "# max_date = '2022-10-01' # Th\u1eddi gian k\u1ebft th\u00fac l\u1ea5y data cho qu\u00e1 tr\u00ecnh train\n",
     "\n",
     "# Specify a spatail region to search using latitude/longitude cooridinates\n",
     "min_longitude, max_longitude = (105.5, 106.4)\n",
@@ -2657,7 +2657,7 @@
        "\n",
        ".xr-section-summary-in + label:before {\n",
        "  display: inline-block;\n",
-       "  content: '►';\n",
+       "  content: '\u25ba';\n",
        "  font-size: 11px;\n",
        "  width: 15px;\n",
        "  text-align: center;\n",
@@ -2668,7 +2668,7 @@
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label:before {\n",
-       "  content: '▼';\n",
+       "  content: '\u25bc';\n",
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label > span {\n",
@@ -3423,23 +3423,23 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "CPU times: user 1.57 s, sys: 177 µs, total: 1.57 s\n",
+      "CPU times: user 1.57 s, sys: 177 \u00b5s, total: 1.57 s\n",
       "Wall time: 1.66 s\n"
      ]
     }
    ],
    "source": [
-    "%%time\n",
+    "# %%time\n",
     "# The replacement \"dc.load()\" function for this product\n",
-    "data = load_s2l2a_with_offset(\n",
-    "    dc,\n",
-    "    query | load_params   # Combine the two dicts that contain our search and load parameters\n",
-    ")\n",
-    "\n",
+    "# data = load_s2l2a_with_offset(\n",
+    "#     dc,\n",
+    "#     query | load_params   # Combine the two dicts that contain our search and load parameters\n",
+    "# )\n",
+    "# \n",
     "# This line prints the total size of the dataset hat was loaded\n",
-    "notebook_utils.heading(notebook_utils.xarray_object_size(data))\n",
-    "\n",
-    "display(data)"
+    "# notebook_utils.heading(notebook_utils.xarray_object_size(data))\n",
+    "# \n",
+    "# display(data)"
    ]
   },
   {
@@ -3546,9 +3546,9 @@
    ],
    "source": [
     "# %%time\n",
-    "# # Tiến hành loại bỏ các vị trí bị mây ảnh hưởng\n",
+    "# # Ti\u1ebfn h\u00e0nh lo\u1ea1i b\u1ecf c\u00e1c v\u1ecb tr\u00ed b\u1ecb m\u00e2y \u1ea3nh h\u01b0\u1edfng\n",
     "# result = mask_clean(data)\n",
-    "# progress(result)"
+    "# # progress(result)"
    ]
   },
   {
@@ -3684,7 +3684,7 @@
        "\n",
        ".xr-section-summary-in + label:before {\n",
        "  display: inline-block;\n",
-       "  content: '►';\n",
+       "  content: '\u25ba';\n",
        "  font-size: 11px;\n",
        "  width: 15px;\n",
        "  text-align: center;\n",
@@ -3695,7 +3695,7 @@
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label:before {\n",
-       "  content: '▼';\n",
+       "  content: '\u25bc';\n",
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label > span {\n",
@@ -4174,7 +4174,7 @@
     }
    ],
    "source": [
-    "# Tiến hành tính toán NDVI\n",
+    "# Ti\u1ebfn h\u00e0nh t\u00ednh to\u00e1n NDVI\n",
     "ds1 = calculate_indices(result, index='NDVI', satellite_mission='s2')\n",
     "ndvi = ds1[\"NDVI\"]\n",
     "display(ndvi)"
@@ -4213,9 +4213,9 @@
    ],
    "source": [
     "%%time\n",
-    "## tính ndvi theo tháng\n",
+    "## t\u00ednh ndvi theo th\u00e1ng\n",
     "average_ndvi = ndvi.resample(time='1M').mean().persist()\n",
-    "progress(average_ndvi)"
+    "# progress(average_ndvi)"
    ]
   },
   {
@@ -4253,12 +4253,12 @@
     }
    ],
    "source": [
-    "# cấu hình vh vv file\n",
+    "# c\u1ea5u h\u00ecnh vh vv file\n",
     "# name_vh = \"ThuanHoa/ThuanHoa_VH.tif\"\n",
     "# name_vv = \"ThuanHoa/ThuanHoa_VV.tif\"\n",
     "\n",
-    "# load dữ liệu sen1\n",
-    "# dsvh, dsvv = load_sen1(name_vh, name_vv)\n",
+    "# load d\u1eef li\u1ec7u sen1\n",
+    "bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]\ntime_range = f'{date_range[0]}/{date_range[1]}'\n# dsvh, dsvv = load_sen1(bbox, time_range)\n",
     "\n",
     "name_vh = \"vh-0922_0923-full_ST.tif\"\n",
     "name_vv = \"vv-0922_0923-full_ST.tif\"\n",
@@ -4268,7 +4268,7 @@
     "if not os.path.exists(name_vv):\n",
     "    !aws s3 cp s3://easi-asia-dc-data/staging/ctu/sentinel-1/vv-0922_0923-full_ST.tif vv-0922_0923-full_ST.tif\n",
     "    \n",
-    "dsvh, dsvv = load_sen1(name_vh, name_vv)"
+    "bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]\ntime_range = f'{date_range[0]}/{date_range[1]}'\ndsvh, dsvv = load_sen1(bbox, time_range)"
    ]
   },
   {
@@ -4326,7 +4326,7 @@
     {
      "data": {
       "text/html": [
-       "<style>#sk-container-id-1 {color: black;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>LinearRegression()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LinearRegression</label><div class=\"sk-toggleable__content\"><pre>LinearRegression()</pre></div></div></div></div></div>"
+       "<style>#sk-container-id-1 {color: black;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"\u25b8\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"\u25be\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>LinearRegression()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LinearRegression</label><div class=\"sk-toggleable__content\"><pre>LinearRegression()</pre></div></div></div></div></div>"
       ],
       "text/plain": [
        "LinearRegression()"
@@ -4397,7 +4397,7 @@
     }
    ],
    "source": [
-    "plt.imshow(average_ndvi_filled.isel(time=6))"
+    "plt.imshow(average_ndvi_filled.isel(time=0))"
    ]
   },
   {
@@ -4430,7 +4430,7 @@
     }
    ],
    "source": [
-    "plt.imshow(average_ndvi.isel(time=6))"
+    "plt.imshow(average_ndvi.isel(time=0))"
    ]
   },
   {
@@ -4440,7 +4440,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "train_path = \"train/ST_training data_updated_1130points.shp\""
+    "train_path = \"train/ST_training_data_updated_1130points.shp\""
    ]
   },
   {
@@ -4476,7 +4476,7 @@
    },
    "outputs": [],
    "source": [
-    "# cấu hình nhãn dữ liệu\n",
+    "# c\u1ea5u h\u00ecnh nh\u00e3n d\u1eef li\u1ec7u\n",
     "label_mapping = {\n",
     "    \"Lua tom\": \"0\",\n",
     "    \"Lua\": \"1\",\n",
@@ -4488,7 +4488,7 @@
     "    \"Rung\": \"7\"\n",
     "}\n",
     "\n",
-    "# chia tập dữ liệu train, val, test\n",
+    "# chia t\u1eadp d\u1eef li\u1ec7u train, val, test\n",
     "X_train, X_val, X_test, y_train, y_val, y_test = split_train_data(train, label_mapping, datasets)"
    ]
   },
@@ -4510,7 +4510,7 @@
     }
    ],
    "source": [
-    "# Huấn luyện mô hình\n",
+    "# Hu\u1ea5n luy\u1ec7n m\u00f4 h\u00ecnh\n",
     "grid_search = train_with_rf(X_train, X_val, y_train, y_val)"
    ]
   },
@@ -4531,7 +4531,7 @@
     }
    ],
    "source": [
-    "# kiểm tra độ chính xác với tập test\n",
+    "# ki\u1ec3m tra \u0111\u1ed9 ch\u00ednh x\u00e1c v\u1edbi t\u1eadp test\n",
     "y_pred_test = grid_search.predict(X_test)\n",
     "test_accuracy = accuracy_score(y_test, y_pred_test)\n",
     "print(f\"Accuracy for test data {round(test_accuracy, 2)*100} %\")"
@@ -4554,7 +4554,7 @@
     }
    ],
    "source": [
-    "# Lưu mô hình huấn luyện\n",
+    "# L\u01b0u m\u00f4 h\u00ecnh hu\u1ea5n luy\u1ec7n\n",
     "save_model(\"model_new.joblib\", grid_search)"
    ]
   },
@@ -4567,7 +4567,7 @@
    },
    "outputs": [],
    "source": [
-    "# đóng client, cluster\n",
+    "# \u0111\u00f3ng client, cluster\n",
     "client.close()\n",
     "cluster.close()"
    ]
@@ -4602,4 +4602,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 5
-}
+}

.temp/processMask-test.py

@@ -0,0 +1,272 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+get_ipython().run_cell_magic('time', '', '%matplotlib inline\nfrom new_import import *\n')
+
+
+# In[2]:
+
+
+get_ipython().run_cell_magic('time', '', '# Dask gateway\ncluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))\ndc = datacube.Datacube()\n\n# Configure s3 access\nconfigure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n\nclient\n')
+
+
+# In[3]:
+
+
+## cấu hình thời gian lấy ảnh và tọa độ
+# date_range = ('2022-09-01', '2023-10-01')
+# longtitude_range = (105.86575, 105.94120)
+# latitude_range = (9.65070, 9.69850)
+date_range = ('2022-09-01', '2023-10-01')
+longtitude_range = (105.5, 106.4)
+latitude_range = (9.2, 10.0) 
+
+
+# In[4]:
+
+
+## truy vấn ảnh vệ tinh sen2
+data = load_data(dc, date_range, longtitude_range, latitude_range)
+notebook_utils.heading(notebook_utils.xarray_object_size(data))
+display(data)
+
+
+# In[5]:
+
+
+# Specify the start and end times 
+min_date = '2022-09-01' # Thời gian bắt đầu lấy data cho quá trình train
+max_date = '2023-10-01' # Thời gian kết thúc lấy data cho quá trình train
+# Just do 1 month for testing
+# max_date = '2022-10-01' # Thời gian kết thúc lấy data cho quá trình train
+
+# Specify a spatail region to search using latitude/longitude cooridinates
+min_longitude, max_longitude = (105.5, 106.4)
+min_latitude, max_latitude = (9.2, 10.0)
+
+# Specify the product. In this case we want to use Sentinel-2 Level-2A data
+product = 's2_l2a'
+
+# Construct the search query dictionary
+query = {
+    'product': product,                     # Product name
+    'x': (min_longitude, max_longitude),    # "x" axis bounds
+    'y': (min_latitude, max_latitude),      # "y" axis bounds
+    'time': (min_date, max_date),           # Any parsable date strings
+}
+
+
+# In[6]:
+
+
+# Most common CRS
+native_crs = notebook_utils.mostcommon_crs(dc, query)
+print(f'Most common native CRS: {native_crs}')
+
+
+# In[7]:
+
+
+# Specify the spectral band measurements we want to use for a classification algorithm
+measurements = ['red', 'nir', 'scl']
+
+load_params = {
+    'measurements': measurements,                   # Selected measurement or alias names
+    'output_crs': native_crs,                       # Target EPSG code
+    'resolution': (-10, 10),                        # Target resolution
+    'group_by': 'solar_day',                        # Scene grouping
+    'dask_chunks': {'x': 2048, 'y': 2048},          # Dask chunks
+}
+
+
+# In[8]:
+
+
+get_ipython().run_cell_magic('time', '', '# The replacement "dc.load()" function for this product\ndata = load_s2l2a_with_offset(\n    dc,\n    query | load_params   # Combine the two dicts that contain our search and load parameters\n)\n\n# This line prints the total size of the dataset hat was loaded\nnotebook_utils.heading(notebook_utils.xarray_object_size(data))\n\ndisplay(data)\n')
+
+
+# In[9]:
+
+
+# %%time
+# # Tiến hành loại bỏ các vị trí bị mây ảnh hưởng
+# result = mask_clean(data)
+# progress(result)
+
+
+# In[10]:
+
+
+# Tiến hành tính toán NDVI
+ds1 = calculate_indices(result, index='NDVI', satellite_mission='s2')
+ndvi = ds1["NDVI"]
+display(ndvi)
+
+
+# In[11]:
+
+
+get_ipython().run_cell_magic('time', '', "## tính ndvi theo tháng\naverage_ndvi = ndvi.resample(time='1M').mean().persist()\nprogress(average_ndvi)\n")
+
+
+# In[12]:
+
+
+# compute average_ndvi
+average_ndvi = average_ndvi.compute()
+
+
+# In[13]:
+
+
+# cấu hình vh vv file
+# name_vh = "ThuanHoa/ThuanHoa_VH.tif"
+# name_vv = "ThuanHoa/ThuanHoa_VV.tif"
+
+# load dữ liệu sen1
+bbox = [105.5, 9.2, 106.4, 10.0]
+time_range = '2022-09-01/2023-10-01'
+# dsvh, dsvv = load_sen1(bbox, time_range)
+
+name_vh = "vh-0922_0923-full_ST.tif"
+name_vv = "vv-0922_0923-full_ST.tif"
+
+if not os.path.exists(name_vh):
+    get_ipython().system('aws s3 cp s3://easi-asia-dc-data/staging/ctu/sentinel-1/vh-0922_0923-full_ST.tif vh-0922_0923-full_ST.tif')
+if not os.path.exists(name_vv):
+    get_ipython().system('aws s3 cp s3://easi-asia-dc-data/staging/ctu/sentinel-1/vv-0922_0923-full_ST.tif vv-0922_0923-full_ST.tif')
+    
+bbox = [105.5, 9.2, 106.4, 10.0]
+time_range = '2022-09-01/2023-10-01'
+dsvh, dsvv = load_sen1(bbox, time_range)
+
+
+# In[27]:
+
+
+from sklearn.preprocessing import PolynomialFeatures
+from sklearn.linear_model import LinearRegression
+from sklearn.ensemble import RandomForestRegressor
+
+
+# In[28]:
+
+
+average_ndvi = average_ndvi[:, :7680, :8687]
+mask = ~np.isnan(average_ndvi)
+print(average_ndvi.shape)
+print(dsvh.shape)
+print(dsvv.shape)
+print(mask.shape)
+X_train = np.stack([dsvh.values[mask], dsvv.values[mask]], axis=1)
+y_train = average_ndvi.values[mask]
+
+
+# In[29]:
+
+
+model = LinearRegression()
+model.fit(X_train, y_train)
+
+
+# In[30]:
+
+
+X_pred = np.stack([dsvh.values[~mask], dsvv.values[~mask]], axis=1)
+average_ndvi.values[~mask] = model.predict(X_pred)
+
+
+# In[31]:
+
+
+average_ndvi_filled = xr.DataArray(average_ndvi, dims=average_ndvi.dims)
+
+
+# In[32]:
+
+
+plt.imshow(average_ndvi_filled.isel(time=6))
+
+
+# In[65]:
+
+
+plt.imshow(average_ndvi.isel(time=6))
+
+
+# In[33]:
+
+
+train_path = "train/ST_training data_updated_1130points.shp"
+
+
+# In[34]:
+
+
+train = load_train_data(train_path)
+
+
+# In[37]:
+
+
+datasets = get_data_sen1_and_sen2(train, average_ndvi_filled, dsvh, dsvv)
+
+
+# In[39]:
+
+
+# cấu hình nhãn dữ liệu
+label_mapping = {
+    "Lua tom": "0",
+    "Lua": "1",
+    "CHN": "2",
+    "CLN": "3",
+    "TS": "4",
+    "Song": "5",
+    "Dat xay dung": "6",
+    "Rung": "7"
+}
+
+# chia tập dữ liệu train, val, test
+X_train, X_val, X_test, y_train, y_val, y_test = split_train_data(train, label_mapping, datasets)
+
+
+# In[40]:
+
+
+# Huấn luyện mô hình
+grid_search = train_with_rf(X_train, X_val, y_train, y_val)
+
+
+# In[41]:
+
+
+# kiểm tra độ chính xác với tập test
+y_pred_test = grid_search.predict(X_test)
+test_accuracy = accuracy_score(y_test, y_pred_test)
+print(f"Accuracy for test data {round(test_accuracy, 2)*100} %")
+
+
+# In[42]:
+
+
+# Lưu mô hình huấn luyện
+save_model("model_new.joblib", grid_search)
+
+
+# In[43]:
+
+
+# đóng client, cluster
+client.close()
+cluster.close()
+
+
+# In[ ]:
+
+
+
+

.temp/processMask.ipynb

@@ -1132,7 +1132,7 @@
    "source": [
     "%%time\n",
     "%matplotlib inline\n",
-    "from new_import import *"
+    "from new_import_ODC import *"
    ]
   },
   {
@@ -1216,7 +1216,7 @@
     "%%time\n",
     "# Dask gateway\n",
     "cluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))\n",
-    "dc = datacube.Datacube()\n",
+    "dc = None\n",
     "\n",
     "# Configure s3 access\n",
     "configure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n",
@@ -1233,8 +1233,8 @@
    },
    "outputs": [],
    "source": [
-    "## cấu hình thời gian lấy ảnh và tọa độ\n",
-    "date_range = ('2022-09-01', '2023-10-01')\n",
+    "## c\u1ea5u h\u00ecnh th\u1eddi gian l\u1ea5y \u1ea3nh v\u00e0 t\u1ecda \u0111\u1ed9\n",
+    "date_range = ('2022-09-01', '2022-10-01')\n",
     "longtitude_range = (105.86575, 105.94120)\n",
     "latitude_range = (9.65070, 9.69850)"
    ]
@@ -1393,7 +1393,7 @@
        "\n",
        ".xr-section-summary-in + label:before {\n",
        "  display: inline-block;\n",
-       "  content: '►';\n",
+       "  content: '\u25ba';\n",
        "  font-size: 11px;\n",
        "  width: 15px;\n",
        "  text-align: center;\n",
@@ -1404,7 +1404,7 @@
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label:before {\n",
-       "  content: '▼';\n",
+       "  content: '\u25bc';\n",
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label > span {\n",
@@ -2333,8 +2333,8 @@
     }
    ],
    "source": [
-    "## truy vấn ảnh vệ tinh sen2\n",
-    "data = load_data(dc, date_range, longtitude_range, latitude_range)\n",
+    "## truy v\u1ea5n \u1ea3nh v\u1ec7 tinh sen2\n",
+    "data = load_data(None, date_range, longtitude_range, latitude_range)\n",
     "notebook_utils.heading(notebook_utils.xarray_object_size(data))\n",
     "display(data)"
    ]
@@ -2443,9 +2443,9 @@
    ],
    "source": [
     "%%time\n",
-    "# Tiến hành loại bỏ các vị trí bị mây ảnh hưởng\n",
+    "# Ti\u1ebfn h\u00e0nh lo\u1ea1i b\u1ecf c\u00e1c v\u1ecb tr\u00ed b\u1ecb m\u00e2y \u1ea3nh h\u01b0\u1edfng\n",
     "result = mask_clean(data)\n",
-    "progress(result)"
+    "# progress(result)"
    ]
   },
   {
@@ -2581,7 +2581,7 @@
        "\n",
        ".xr-section-summary-in + label:before {\n",
        "  display: inline-block;\n",
-       "  content: '►';\n",
+       "  content: '\u25ba';\n",
        "  font-size: 11px;\n",
        "  width: 15px;\n",
        "  text-align: center;\n",
@@ -2592,7 +2592,7 @@
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label:before {\n",
-       "  content: '▼';\n",
+       "  content: '\u25bc';\n",
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label > span {\n",
@@ -3047,7 +3047,7 @@
     }
    ],
    "source": [
-    "# Tiến hành tính toán NDVI\n",
+    "# Ti\u1ebfn h\u00e0nh t\u00ednh to\u00e1n NDVI\n",
     "ds1 = calculate_indices(result, index='NDVI', satellite_mission='s2')\n",
     "ndvi = ds1[\"NDVI\"]\n",
     "display(ndvi)"
@@ -3086,9 +3086,9 @@
    ],
    "source": [
     "%%time\n",
-    "## tính ndvi theo tháng\n",
+    "## t\u00ednh ndvi theo th\u00e1ng\n",
     "average_ndvi = ndvi.resample(time='1M').mean().persist()\n",
-    "progress(average_ndvi)"
+    "# progress(average_ndvi)"
    ]
   },
   {
@@ -3113,12 +3113,12 @@
    },
    "outputs": [],
    "source": [
-    "# cấu hình vh vv file\n",
+    "# c\u1ea5u h\u00ecnh vh vv file\n",
     "name_vh = \"ThuanHoa/ThuanHoa_VH.tif\"\n",
     "name_vv = \"ThuanHoa/ThuanHoa_VV.tif\"\n",
     "\n",
-    "# load dữ liệu sen1\n",
-    "dsvh, dsvv = load_sen1(name_vh, name_vv)"
+    "# load d\u1eef li\u1ec7u sen1\n",
+    "bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]\ntime_range = f'{date_range[0]}/{date_range[1]}'\ndsvh, dsvv = load_sen1(bbox, time_range)"
    ]
   },
   {
@@ -3160,7 +3160,7 @@
     {
      "data": {
       "text/html": [
-       "<style>#sk-container-id-1 {color: black;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>LinearRegression()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LinearRegression</label><div class=\"sk-toggleable__content\"><pre>LinearRegression()</pre></div></div></div></div></div>"
+       "<style>#sk-container-id-1 {color: black;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"\u25b8\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"\u25be\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>LinearRegression()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LinearRegression</label><div class=\"sk-toggleable__content\"><pre>LinearRegression()</pre></div></div></div></div></div>"
       ],
       "text/plain": [
        "LinearRegression()"
@@ -3297,4 +3297,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 5
-}
+}

.temp/processMask.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+get_ipython().run_cell_magic('time', '', '%matplotlib inline\nfrom new_import import *\n')
+
+
+# In[2]:
+
+
+get_ipython().run_cell_magic('time', '', '# Dask gateway\ncluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))\ndc = datacube.Datacube()\n\n# Configure s3 access\nconfigure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n\nclient\n')
+
+
+# In[3]:
+
+
+## cấu hình thời gian lấy ảnh và tọa độ
+date_range = ('2022-09-01', '2023-10-01')
+longtitude_range = (105.86575, 105.94120)
+latitude_range = (9.65070, 9.69850)
+
+
+# In[4]:
+
+
+## truy vấn ảnh vệ tinh sen2
+data = load_data(dc, date_range, longtitude_range, latitude_range)
+notebook_utils.heading(notebook_utils.xarray_object_size(data))
+display(data)
+
+
+# In[5]:
+
+
+get_ipython().run_cell_magic('time', '', '# Tiến hành loại bỏ các vị trí bị mây ảnh hưởng\nresult = mask_clean(data)\nprogress(result)\n')
+
+
+# In[6]:
+
+
+# Tiến hành tính toán NDVI
+ds1 = calculate_indices(result, index='NDVI', satellite_mission='s2')
+ndvi = ds1["NDVI"]
+display(ndvi)
+
+
+# In[17]:
+
+
+get_ipython().run_cell_magic('time', '', "## tính ndvi theo tháng\naverage_ndvi = ndvi.resample(time='1M').mean().persist()\nprogress(average_ndvi)\n")
+
+
+# In[18]:
+
+
+# compute average_ndvi
+average_ndvi = average_ndvi.compute()
+
+
+# In[9]:
+
+
+# cấu hình vh vv file
+name_vh = "ThuanHoa/ThuanHoa_VH.tif"
+name_vv = "ThuanHoa/ThuanHoa_VV.tif"
+
+# load dữ liệu sen1
+bbox = [105.5, 9.2, 106.4, 10.0]
+time_range = '2022-09-01/2023-10-01'
+dsvh, dsvv = load_sen1(bbox, time_range)
+
+
+# In[10]:
+
+
+from sklearn.preprocessing import PolynomialFeatures
+from sklearn.linear_model import LinearRegression
+from sklearn.ensemble import RandomForestRegressor
+
+
+# In[11]:
+
+
+mask = ~np.isnan(average_ndvi)
+X_train = np.stack([dsvh.values[mask], dsvv.values[mask]], axis=1)
+y_train = average_ndvi.values[mask]
+
+
+# In[12]:
+
+
+model = LinearRegression()
+model.fit(X_train, y_train)
+
+
+# In[13]:
+
+
+X_pred = np.stack([dsvh.values[~mask], dsvv.values[~mask]], axis=1)
+average_ndvi.values[~mask] = model.predict(X_pred)
+
+
+# In[14]:
+
+
+average_ndvi_filled = xr.DataArray(average_ndvi, dims=average_ndvi.dims)
+
+
+# In[16]:
+
+
+plt.imshow(average_ndvi_filled.isel(time=1))
+
+
+# In[19]:
+
+
+plt.imshow(average_ndvi.isel(time=1))
+
+
+# In[ ]:
+
+
+
+

.temp/rebuild_notebook.py

@@ -0,0 +1,145 @@
+import json
+
+def get_content(filename):
+    with open(filename, "r", encoding="utf-8") as f:
+        return f.read()
+
+fe_content = get_content("feature_extractor.py")
+tm_content = get_content("train_module.py")
+dt_content = get_content("train_land_decision_tree_gpu.py")
+
+notebook = {
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Tải Dữ liệu Vệ tinh qua Colab (Self-contained)\n",
+    "Notebook này đã được nhúng sẵn toàn bộ mã nguồn xử lý. Bạn không cần upload cả thư mục `remote-sensing` nữa.\n",
+    "\n",
+    "## Bước 1: Upload Shapefile (BẮT BUỘC)\n",
+    "Mô hình cần biết các điểm tọa độ đất để lấy dữ liệu. Hãy nén thư mục `train/` trên máy bạn thành `train.zip` và chạy ô dưới đây để upload nó trực tiếp lên Colab."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": None,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from google.colab import files\n",
+    "import os\n",
+    "\n",
+    "print(\"Hãy chọn file train.zip từ máy tính của bạn:\")\n",
+    "uploaded = files.upload()\n",
+    "\n",
+    "if \"train.zip\" in uploaded:\n",
+    "    !unzip -q -o train.zip -d /content/train_tmp/\n",
+    "    # Move the extracted files directly to /content/train/\n",
+    "    !mkdir -p /content/train\n",
+    "    !mv /content/train_tmp/*/* /content/train/ 2>/dev/null || mv /content/train_tmp/* /content/train/\n",
+    "    print(\"Đã giải nén shapefile thành công vào thư mục /content/train/\")\n",
+    "else:\n",
+    "    print(\"LỖI: Bạn chưa upload file có tên là train.zip!\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Bước 2: Cài đặt thư viện & Tạo môi trường"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": None,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!pip install planetary-computer pystac-client odc-stac geopandas rasterio xarray joblib scikit-learn xgboost lightgbm"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": None,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%writefile feature_extractor.py\n" + fe_content
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": None,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%writefile train_module.py\n" + tm_content
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": None,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%writefile train_land_decision_tree_gpu.py\n" + dt_content
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Bước 3: Chạy tiến trình tải ảnh vệ tinh và tạo Cache"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": None,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!python train_land_decision_tree_gpu.py"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Bước 4: Tải file Cache về máy"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": None,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from google.colab import files\n",
+    "import glob\n",
+    "\n",
+    "cache_files = glob.glob(\"dataset_cache/*.joblib\")\n",
+    "if cache_files:\n",
+    "    latest_cache = max(cache_files, key=os.path.getctime)\n",
+    "    print(f\"Đang tải file {latest_cache} về máy...\")\n",
+    "    files.download(latest_cache)\n",
+    "else:\n",
+    "    print(\"Chưa tìm thấy file cache. Hãy chắc chắn bước 3 đã chạy thành công!\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
+
+with open("Download_Cache_Colab.ipynb", "w", encoding="utf-8") as f:
+    json.dump(notebook, f, indent=1, ensure_ascii=False)
+
+print("Notebook updated successfully!")

.temp/remove_datacube.py

@@ -0,0 +1,35 @@
+import json
+import glob
+
+def fix_notebook(file_path):
+    with open(file_path, 'r', encoding='utf-8') as f:
+        nb = json.load(f)
+    changed = False
+    for cell in nb.get('cells', []):
+        if cell.get('cell_type') == 'code':
+            source = cell.get('source', [])
+            if isinstance(source, list):
+                for i, line in enumerate(source):
+                    if "dc = datacube.Datacube()" in line:
+                        source[i] = "dc = None\n"
+                        changed = True
+                    if "ds = dc.load(" in line:
+                        source[i] = "ds = None\n"
+                        changed = True
+                    if "data = dc.load(" in line:
+                        source[i] = "data = None\n"
+                        changed = True
+                    # If ds is None, ds.vv will fail
+                    if "vv_data = ds.vv" in line:
+                        source[i] = "vv_data = None\n"
+                        changed = True
+                    if "notebook_utils.xarray_object_size(ds)" in line:
+                        source[i] = line.replace("notebook_utils.xarray_object_size(ds)", "'ds is None'")
+                        changed = True
+    if changed:
+        with open(file_path, 'w', encoding='utf-8') as f:
+            json.dump(nb, f, indent=1)
+        print(f"Removed datacube from {file_path}")
+
+for nb in glob.glob("*.ipynb"):
+    fix_notebook(nb)

.temp/run_mock_parallel_and_compare.py

@@ -0,0 +1,110 @@
+import os
+import glob
+import json
+import subprocess
+import time
+from tabulate import tabulate
+
+scripts = [
+    "train_land_randomforest.py",
+    "train_cloud_cnn.py",
+    "train_cloud_swin_unet.py",
+    "train_ndvi_statistical.py",
+    "train_ndvi_lstm_gru.py",
+    "train_ndvi_convlstm.py",
+    "train_ndvi_hybrid_physics.py",
+    "train_ndvi_ensemble.py"
+]
+
+print("🚀 Đang khởi chạy song song tất cả các mô hình...")
+processes = []
+for script in scripts:
+    if os.path.exists(script):
+        cmd = f"source /home/x79/miniconda3/etc/profile.d/conda.sh && conda activate env_01 && python {script}"
+        p = subprocess.Popen(["bash", "-c", cmd], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        processes.append((script, p))
+
+for script, p in processes:
+    p.wait()
+
+print("✅ Đã chạy xong tất cả các mô hình!\n")
+print("📊 BẢNG SO SÁNH KẾT QUẢ CÁC MÔ HÌNH\n")
+
+# 1. Phân loại đất
+print("### 1. Nhóm Phân loại Lớp phủ (Land Classification)")
+land_data = []
+
+# Đọc XGBoost từ thư mục gốc
+if os.path.exists("model_xgboost_info.json"):
+    with open("model_xgboost_info.json", 'r') as f:
+        data = json.load(f)
+        params = data.get('params', {})
+        param_str = f"estimators:{params.get('n_estimators')}, depth:{params.get('max_depth')}" if params else "N/A"
+        land_data.append([
+            data.get('model_type', 'XGBoost'),
+            data.get('accuracy', ''),
+            data.get('precision', ''),
+            data.get('recall', ''),
+            data.get('f1_score', ''),
+            param_str
+        ])
+
+# Đọc các model khác trong model_train
+for info_file in glob.glob("model_train/*_info.json"):
+    with open(info_file, 'r') as f:
+        data = json.load(f)
+        # Chỉ lấy các model có độ chính xác (để lọc model rác/cũ)
+        if 'accuracy' not in data and 'f1_score' not in data:
+            continue
+            
+        params = data.get('params', {})
+        param_str = f"estimators:{params.get('n_estimators')}, depth:{params.get('max_depth')}" if params else "N/A"
+        
+        # Fallback for Random Forest
+        if data.get('model_type') == 'RandomForest_RealData':
+            param_str = "estimators:100, depth:15"
+            
+        land_data.append([
+            data.get('model_type', ''),
+            data.get('accuracy', ''),
+            data.get('precision', ''),
+            data.get('recall', ''),
+            data.get('f1_score', ''),
+            param_str
+        ])
+
+if land_data:
+    print(tabulate(land_data, headers=["Model", "Accuracy", "Precision", "Recall", "F1-Score", "Parameters"], tablefmt="github"))
+print("\n")
+
+# 2. Xóa mây
+print("### 2. Nhóm Xóa mây (Cloud Removal)")
+cloud_data = []
+for info_file in glob.glob("cloud_removal_model/*_info.json"):
+    with open(info_file, 'r') as f:
+        data = json.load(f)
+        cloud_data.append([
+            data.get('model_type', ''),
+            data.get('epoch', ''),
+            data.get('train_loss', ''),
+            data.get('val_loss', '')
+        ])
+if cloud_data:
+    print(tabulate(cloud_data, headers=["Model", "Epochs", "Train Loss", "Val Loss"], tablefmt="github"))
+print("\n")
+
+# 3. Dự báo NDVI
+print("### 3. Nhóm Dự báo Thực vật (NDVI Forecasting)")
+ndvi_data = []
+for info_file in glob.glob("ndvi_forecast_model/*_info.json"):
+    with open(info_file, 'r') as f:
+        data = json.load(f)
+        ndvi_data.append([
+            data.get('model_type', ''),
+            data.get('rmse', ''),
+            data.get('mae', ''),
+            data.get('epoch', 'N/A')
+        ])
+if ndvi_data:
+    print(tabulate(ndvi_data, headers=["Model", "RMSE", "MAE", "Epochs"], tablefmt="github"))
+print("\n")

.temp/test_VH_VV.ipynb

@@ -1141,12 +1141,12 @@
    "source": [
     "\n",
     "%matplotlib inline\n",
-    "from new_import import *\n",
+    "from new_import_ODC import *\n",
     "\n",
     "\n",
     "# Dask gateway\n",
     "cluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))\n",
-    "dc = datacube.Datacube()\n",
+    "dc = None\n",
     "\n",
     "\n",
     "# Configure s3 access\n",
@@ -1300,7 +1300,7 @@
        "\n",
        ".xr-section-summary-in + label:before {\n",
        "  display: inline-block;\n",
-       "  content: '►';\n",
+       "  content: '\u25ba';\n",
        "  font-size: 11px;\n",
        "  width: 15px;\n",
        "  text-align: center;\n",
@@ -1311,7 +1311,7 @@
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label:before {\n",
-       "  content: '▼';\n",
+       "  content: '\u25bc';\n",
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label > span {\n",
@@ -1880,20 +1880,20 @@
     }
    ],
    "source": [
-    "ds = dc.load(\n",
-    "    product=\"sentinel1_grd_gamma0_20m\",\n",
-    "    x=(105.5, 106.4),\n",
-    "    y=(9.2, 10.0),\n",
-    "    time=(\"2022-09-01\", \"2023-10-01\"),\n",
-    "    measurements=[\"vv\", \"vh\"],\n",
-    "    output_crs=\"EPSG:32648\",\n",
-    "    resolution=(-10,10),\n",
-    "    dask_chunks={\"x\":2048, \"y\":2048},\n",
-    "    skip_broken_datasets=True,\n",
-    "    group_by=\"solar_day\"\n",
-    ")\n",
-    "notebook_utils.heading(notebook_utils.xarray_object_size(ds))\n",
-    "ds"
+    "# ds = dc.load(\n",
+    "#     product=\"sentinel1_grd_gamma0_20m\",\n",
+    "#     x=(105.5, 106.4),\n",
+    "#     y=(9.2, 10.0),\n",
+    "#     time=(\"2022-09-01\", \"2022-10-01\"),\n",
+    "#     measurements=[\"vv\", \"vh\"],\n",
+    "#     output_crs=\"EPSG:32648\",\n",
+    "#     resolution=(-10,10),\n",
+    "#     dask_chunks={\"x\":2048, \"y\":2048},\n",
+    "#     skip_broken_datasets=True,\n",
+    "#     group_by=\"solar_day\"\n",
+    "# )\n",
+    "# notebook_utils.heading(notebook_utils.xarray_object_size(ds))\n",
+    "# ds"
    ]
   },
   {
@@ -1905,11 +1905,11 @@
    },
    "outputs": [],
    "source": [
-    "vh = ds.vh.resample(time='1M').mean().persist()\n",
-    "vh = vh.compute()\n",
-    "vv = ds.vv.resample(time='1M').mean().persist()\n",
-    "vv = vv.compute()\n",
-    "\n"
+    "# vh = ds.vh.resample(time='1M').mean().persist()\n",
+    "# vh = vh.compute()\n",
+    "# vv = ds.vv.resample(time='1M').mean().persist()\n",
+    "# vv = vv.compute()\n",
+    "# \n"
    ]
   },
   {
@@ -2045,7 +2045,7 @@
        "\n",
        ".xr-section-summary-in + label:before {\n",
        "  display: inline-block;\n",
-       "  content: '►';\n",
+       "  content: '\u25ba';\n",
        "  font-size: 11px;\n",
        "  width: 15px;\n",
        "  text-align: center;\n",
@@ -2056,7 +2056,7 @@
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label:before {\n",
-       "  content: '▼';\n",
+       "  content: '\u25bc';\n",
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label > span {\n",
@@ -2365,4 +2365,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 5
-}
+}

.temp/test_VH_VV.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[2]:
+
+
+get_ipython().run_line_magic('matplotlib', 'inline')
+from new_import import *
+
+
+# Dask gateway
+cluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))
+dc = datacube.Datacube()
+
+
+# Configure s3 access
+configure_s3_access(aws_unsigned=False, requester_pays=True, client=client)
+
+
+
+
+# In[3]:
+
+
+ds = dc.load(
+    product="sentinel1_grd_gamma0_20m",
+    x=(105.5, 106.4),
+    y=(9.2, 10.0),
+    time=("2022-09-01", "2023-10-01"),
+    measurements=["vv", "vh"],
+    output_crs="EPSG:32648",
+    resolution=(-10,10),
+    dask_chunks={"x":2048, "y":2048},
+    skip_broken_datasets=True,
+    group_by="solar_day"
+)
+notebook_utils.heading(notebook_utils.xarray_object_size(ds))
+ds
+
+
+# In[18]:
+
+
+vh = ds.vh.resample(time='1M').mean().persist()
+vh = vh.compute()
+vv = ds.vv.resample(time='1M').mean().persist()
+vv = vv.compute()
+
+
+
+# In[28]:
+
+
+vv.min()
+
+
+# In[33]:
+
+
+import matplotlib.pyplot as plt
+
+# Plot the data
+plt.imshow(vh.isel(time=0), cmap='viridis', vmin=0, vmax=1)
+plt.colorbar()  # Add colorbar for reference
+plt.show()
+
+
+# In[ ]:
+
+
+
+

.temp/test_bug.py

@@ -0,0 +1,33 @@
+import torch
+import numpy as np
+
+device = "cpu"
+input_array = np.zeros((4, 16, 16), dtype=np.float32)
+input_tensor = torch.from_numpy(input_array).unsqueeze(0).to(device)
+
+print("Before pad:", input_tensor.shape)
+
+from train_cloud_removal import UNet
+model = UNet(in_channels=6, out_channels=4).to(device)
+
+if hasattr(model, 'inc') and hasattr(model.inc.double_conv[0], 'in_channels'):
+    expected_channels = model.inc.double_conv[0].in_channels
+elif hasattr(model, 'conv1') and hasattr(model.conv1, 'in_channels'):
+    expected_channels = model.conv1.in_channels
+else:
+    expected_channels = list(model.parameters())[0].shape[1]
+
+print("Expected channels:", expected_channels)
+
+if expected_channels > input_tensor.shape[1]:
+    pad_channels = expected_channels - input_tensor.shape[1]
+    padding = torch.zeros(1, pad_channels, *input_tensor.shape[2:]).to(device)
+    input_tensor = torch.cat([input_tensor, padding], dim=1)
+
+print("After pad:", input_tensor.shape)
+
+try:
+    model(input_tensor)
+    print("Success!")
+except Exception as e:
+    print("Error:", e)

.temp/test_caching.py

@@ -0,0 +1,25 @@
+import os
+import sys
+sys.path.insert(0, os.getcwd())
+import new_import_ODC
+import time
+import xarray as xr
+
+# Mock minimal params to test load_data
+date_range = ('2023-01-01', '2023-01-31')
+longtitude_range = (105.0, 105.1)
+latitude_range = (9.5, 9.6)
+
+print("--- First Call (Downloading & Caching) ---")
+start = time.time()
+data1 = new_import_ODC.load_data(None, date_range, longtitude_range, latitude_range)
+end = time.time()
+print(f"Time taken: {end - start:.2f}s")
+
+print("--- Second Call (Loading from Cache) ---")
+start = time.time()
+data2 = new_import_ODC.load_data(None, date_range, longtitude_range, latitude_range)
+end = time.time()
+print(f"Time taken: {end - start:.2f}s")
+
+print("✅ Test completed")

.temp/test_checkpoint.py

@@ -0,0 +1,7 @@
+import torch
+checkpoint = torch.load('cloud_removal_model/cloud_removal_unet_best.pth', map_location='cpu')
+print(checkpoint.keys())
+print("in_channels in checkpoint:", 'in_channels' in checkpoint)
+if 'in_channels' in checkpoint:
+    print(checkpoint['in_channels'])
+print("Shape of inc.double_conv.0.weight:", checkpoint['model_state_dict']['inc.double_conv.0.weight'].shape)

.temp/test_cloud.py

@@ -0,0 +1,4 @@
+import torch
+from cloud_removal import DeepInpaintingStrategy
+cloud_remover = DeepInpaintingStrategy()
+print("Model channels:", list(cloud_remover.model.parameters())[0].shape[1])

.temp/test_cloud2.py

@@ -0,0 +1,10 @@
+from cloud_removal import DeepInpaintingStrategy
+import torch
+import numpy as np
+
+cr = DeepInpaintingStrategy(model_path="cloud_removal_model/cloud_removal_unet_best.pth")
+if cr.model is not None:
+    expected = list(cr.model.parameters())[0].shape[1]
+    print("Expected channels:", expected)
+else:
+    print("Failed to load model")

.temp/test_cloud_removal.py

@@ -0,0 +1,194 @@
+"""
+Test script for cloud_removal module
+Kiểm tra các phương pháp xử lý mây
+"""
+
+import numpy as np
+import xarray as xr
+from cloud_removal import (
+    process_cloud_removal,
+    get_available_methods,
+    compare_methods
+)
+
+
+def create_mock_s2_data():
+    """Tạo mock Sentinel-2 data để test"""
+    # Create synthetic data: 5 time steps, 100x100 pixels
+    np.random.seed(42)
+    
+    time_steps = 5
+    y_size = 100
+    x_size = 100
+    
+    # Create bands
+    bands = {}
+    for band in ["B02", "B03", "B04", "B08", "B11"]:
+        # Random reflectance values
+        data = np.random.rand(time_steps, y_size, x_size) * 0.3 + 0.1
+        bands[band] = (["time", "y", "x"], data)
+    
+    # Create SCL (Scene Classification Layer)
+    # Mostly vegetation (4), with some clouds
+    scl_data = np.full((time_steps, y_size, x_size), 4, dtype=np.uint8)
+    
+    # Add clouds (class 9) in random locations
+    for t in range(time_steps):
+        # Random cloud patches
+        n_clouds = np.random.randint(5, 15)
+        for _ in range(n_clouds):
+            y_start = np.random.randint(0, y_size - 20)
+            x_start = np.random.randint(0, x_size - 20)
+            cloud_height = np.random.randint(10, 20)
+            cloud_width = np.random.randint(10, 20)
+            scl_data[t, y_start:y_start+cloud_height, x_start:x_start+cloud_width] = 9
+    
+    bands["SCL"] = (["time", "y", "x"], scl_data)
+    
+    # Create xarray Dataset
+    ds = xr.Dataset(
+        bands,
+        coords={
+            "time": np.arange(time_steps),
+            "y": np.arange(y_size),
+            "x": np.arange(x_size)
+        }
+    )
+    
+    return ds
+
+
+def test_available_methods():
+    """Test lấy danh sách methods"""
+    print("=" * 60)
+    print("TEST: Get Available Methods")
+    print("=" * 60)
+    
+    methods = get_available_methods()
+    print(f"\nFound {len(methods)} methods:")
+    for method, description in methods.items():
+        print(f"  - {method:20s}: {description}")
+    
+    print("\n✅ Test passed!")
+
+
+def test_single_method(method_name="classic"):
+    """Test một method cụ thể"""
+    print("\n" + "=" * 60)
+    print(f"TEST: Cloud Removal Method '{method_name}'")
+    print("=" * 60)
+    
+    # Create mock data
+    s2_data = create_mock_s2_data()
+    print(f"\nMock data created: {dict(s2_data.dims)}")
+    
+    # Process clouds
+    cleaned_data, metadata = process_cloud_removal(
+        s2_data=s2_data,
+        method=method_name,
+        verbose=True
+    )
+    
+    # Check results
+    print(f"\nMetadata:")
+    print(f"  - Method: {metadata['method']}")
+    print(f"  - Cloud coverage: {metadata['cloud_coverage_percent']:.1f}%")
+    print(f"  - Masked pixels: {metadata['masked_pixels']:,}/{metadata['total_pixels']:,}")
+    print(f"  - Steps applied: {', '.join(metadata['steps_applied'])}")
+    
+    # Verify no NaN remaining
+    nan_count = 0
+    for band in cleaned_data.data_vars:
+        if band != "SCL":
+            nan_count += np.isnan(cleaned_data[band].values).sum()
+    
+    print(f"\nRemaining NaN pixels: {nan_count}")
+    
+    if nan_count == 0:
+        print("✅ Test passed - no NaN remaining!")
+    else:
+        print(f"⚠️  Warning - {nan_count} NaN pixels remaining")
+
+
+def test_comparison():
+    """Test so sánh nhiều methods"""
+    print("\n" + "=" * 60)
+    print("TEST: Compare Multiple Methods")
+    print("=" * 60)
+    
+    # Create mock data
+    s2_data = create_mock_s2_data()
+    
+    # Compare methods
+    methods_to_test = ["classic", "temporal_only", "median_composite", "ml_knn"]
+    
+    print(f"\nComparing {len(methods_to_test)} methods...")
+    results = compare_methods(s2_data, methods=methods_to_test)
+    
+    # Print summary
+    print("\n" + "-" * 60)
+    print(f"{'Method':<20} {'Success':<10} {'NaN %':<10} {'Steps'}")
+    print("-" * 60)
+    
+    for method, result in results.items():
+        if result['success']:
+            nan_pct = result['remaining_nan_percent']
+            steps = ', '.join(result['metadata']['steps_applied'][:2])  # First 2 steps
+            print(f"{method:<20} {'✅':<10} {nan_pct:>6.2f}%   {steps}")
+        else:
+            print(f"{method:<20} {'❌':<10} {'ERROR':<10} {result['error']}")
+    
+    print("-" * 60)
+    print("\n✅ Comparison test completed!")
+
+
+def test_edge_cases():
+    """Test các trường hợp đặc biệt"""
+    print("\n" + "=" * 60)
+    print("TEST: Edge Cases")
+    print("=" * 60)
+    
+    # Case 1: No SCL band
+    print("\n1. Testing without SCL band...")
+    s2_data = create_mock_s2_data()
+    s2_data_no_scl = s2_data.drop_vars("SCL")
+    
+    cleaned, meta = process_cloud_removal(s2_data_no_scl, method="classic", verbose=False)
+    print(f"   Result: {meta.get('warning', 'OK')}")
+    
+    # Case 2: 100% cloud coverage
+    print("\n2. Testing with 100% cloud coverage...")
+    s2_data_full_cloud = create_mock_s2_data()
+    s2_data_full_cloud["SCL"][:] = 9  # All clouds
+    
+    cleaned, meta = process_cloud_removal(s2_data_full_cloud, method="classic", verbose=False)
+    print(f"   Cloud coverage: {meta['cloud_coverage_percent']:.1f}%")
+    
+    # Case 3: No clouds
+    print("\n3. Testing with no clouds...")
+    s2_data_clear = create_mock_s2_data()
+    s2_data_clear["SCL"][:] = 4  # All vegetation
+    
+    cleaned, meta = process_cloud_removal(s2_data_clear, method="classic", verbose=False)
+    print(f"   Cloud coverage: {meta['cloud_coverage_percent']:.1f}%")
+    
+    print("\n✅ Edge case tests passed!")
+
+
+if __name__ == "__main__":
+    print("\n" + "🌥️  CLOUD REMOVAL MODULE TESTS 🌥️ ".center(60, "="))
+    print()
+    
+    # Run tests
+    test_available_methods()
+    test_single_method("classic")
+    test_single_method("hybrid")
+    test_comparison()
+    test_edge_cases()
+    
+    print("\n" + "=" * 60)
+    print("ALL TESTS COMPLETED!")
+    print("=" * 60)
+    print("\nModule is ready to use. Available methods:")
+    for method, desc in get_available_methods().items():
+        print(f"  • {method}")

.temp/test_cloud_training.py

@@ -0,0 +1,28 @@
+"""
+Script test nhanh cho cloud removal training
+"""
+
+import sys
+from pathlib import Path
+
+# Add winter_dataset to path
+sys.path.insert(0, str(Path(__file__).parent / "winter_dataset"))
+
+from train_cloud_removal import train_cloud_removal_model
+
+if __name__ == "__main__":
+    print("\n🌥️  Starting Cloud Removal Training Test")
+    print("=" * 70)
+    
+    # Test with small dataset
+    model, train_losses, val_losses = train_cloud_removal_model(
+        data_dir="winter_dataset",
+        use_s1=True,  # Use S1 radar data
+        batch_size=4,  # Small batch for testing
+        num_epochs=5,  # Few epochs for quick test
+        learning_rate=1e-4
+    )
+    
+    print("\n✅ Training test completed!")
+    print(f"Final train loss: {train_losses[-1]:.6f}")
+    print(f"Final val loss: {val_losses[-1]:.6f}")

.temp/test_cloud_upload.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python3
+"""
+Test Cloud Removal Model Upload Feature
+"""
+
+import requests
+import json
+from pathlib import Path
+
+# API base URL
+BASE_URL = "http://localhost:8000"
+
+def test_upload_cloud_model(file_path):
+    """Test uploading a cloud removal model"""
+    print(f"\n{'='*60}")
+    print("TEST 1: Upload Cloud Removal Model")
+    print(f"{'='*60}")
+    
+    if not Path(file_path).exists():
+        print(f"❌ File not found: {file_path}")
+        print("   Create a dummy .pth file for testing:")
+        print(f"   touch {file_path}")
+        return None
+    
+    with open(file_path, 'rb') as f:
+        files = {'file': (Path(file_path).name, f, 'application/octet-stream')}
+        
+        print(f"📤 Uploading: {file_path}")
+        response = requests.post(f"{BASE_URL}/api/cloud-removal/upload", files=files)
+        
+        if response.status_code == 200:
+            result = response.json()
+            print(f"✅ Upload successful!")
+            print(f"   Filename: {result['filename']}")
+            print(f"   Size: {result['size_mb']} MB")
+            print(f"   Path: {result['path']}")
+            return result['filename']
+        else:
+            print(f"❌ Upload failed: {response.status_code}")
+            print(f"   {response.json().get('detail', 'Unknown error')}")
+            return None
+
+def test_list_cloud_models():
+    """Test listing cloud removal models"""
+    print(f"\n{'='*60}")
+    print("TEST 2: List Cloud Removal Models")
+    print(f"{'='*60}")
+    
+    response = requests.get(f"{BASE_URL}/api/cloud-removal/models")
+    
+    if response.status_code == 200:
+        data = response.json()
+        print(f"✅ Found {data['count']} models:")
+        for i, model in enumerate(data['models'], 1):
+            print(f"\n   {i}. {model['filename']}")
+            print(f"      Size: {model['size_mb']} MB")
+            print(f"      Created: {model['created']}")
+            if 'epoch' in model:
+                print(f"      Epoch: {model['epoch']}, Val Loss: {model['val_loss']:.4f}")
+        return data['models']
+    else:
+        print(f"❌ Failed to list models: {response.status_code}")
+        return []
+
+def test_prediction_with_cloud_model(model_filename, cloud_model_filename):
+    """Test prediction using uploaded cloud removal model"""
+    print(f"\n{'='*60}")
+    print("TEST 3: Prediction with Custom Cloud Removal Model")
+    print(f"{'='*60}")
+    
+    config = {
+        "model_filename": model_filename,
+        "min_lon": 105.80,
+        "min_lat": 10.00,
+        "max_lon": 105.82,
+        "max_lat": 10.02,
+        "start_date": "2024-01-15",
+        "end_date": "2024-01-17",
+        "max_scenes": 2,
+        "cloud_cover": 30,
+        "resolution": 20,
+        "use_gpu": False,
+        "export_ndvi": True,
+        "export_classification": True,
+        "cloud_removal_method": "deep",
+        "cloud_removal_model": cloud_model_filename
+    }
+    
+    print("📊 Prediction Config:")
+    print(json.dumps(config, indent=2))
+    
+    print(f"\n🚀 Starting prediction with cloud removal model: {cloud_model_filename}")
+    response = requests.post(
+        f"{BASE_URL}/api/predict/with-ndvi",
+        json=config,
+        headers={'Content-Type': 'application/json'}
+    )
+    
+    if response.status_code == 200:
+        result = response.json()
+        print(f"✅ Prediction started!")
+        print(f"   Message: {result.get('message')}")
+        return result
+    else:
+        print(f"❌ Prediction failed: {response.status_code}")
+        print(f"   {response.json().get('detail', 'Unknown error')}")
+        return None
+
+def test_delete_cloud_model(filename):
+    """Test deleting a cloud removal model"""
+    print(f"\n{'='*60}")
+    print("TEST 4: Delete Cloud Removal Model")
+    print(f"{'='*60}")
+    
+    print(f"🗑️  Deleting: {filename}")
+    response = requests.delete(f"{BASE_URL}/api/cloud-removal/models/{filename}")
+    
+    if response.status_code == 200:
+        result = response.json()
+        print(f"✅ {result['message']}")
+        return True
+    else:
+        print(f"❌ Delete failed: {response.status_code}")
+        return False
+
+def main():
+    print("="*60)
+    print("CLOUD REMOVAL MODEL UPLOAD - FEATURE TEST")
+    print("="*60)
+    
+    # Test file path (create a dummy file for testing)
+    test_file = "test_cloud_removal_model.pth"
+    
+    # Create dummy file if it doesn't exist
+    if not Path(test_file).exists():
+        print(f"\n📝 Creating dummy test file: {test_file}")
+        Path(test_file).write_bytes(b"dummy_pytorch_model_data")
+    
+    # Run tests
+    uploaded_filename = test_upload_cloud_model(test_file)
+    
+    if uploaded_filename:
+        models = test_list_cloud_models()
+        
+        # Test prediction (requires a real land classification model)
+        print(f"\n{'='*60}")
+        print("NOTE: Prediction test requires a trained land classification model")
+        print("      Skipping prediction test in this demo")
+        print(f"{'='*60}")
+        
+        # Cleanup - delete test model
+        if input("\nDelete test model? (y/n): ").lower() == 'y':
+            test_delete_cloud_model(uploaded_filename)
+    
+    # Cleanup dummy file
+    if Path(test_file).exists():
+        Path(test_file).unlink()
+        print(f"\n🗑️  Cleaned up dummy file: {test_file}")
+    
+    print(f"\n{'='*60}")
+    print("TESTS COMPLETED")
+    print(f"{'='*60}")
+
+if __name__ == "__main__":
+    main()

.temp/test_empty_points.py

@@ -0,0 +1,53 @@
+import geopandas as gpd
+import planetary_computer
+import pystac_client
+import odc.stac
+import numpy as np
+from shapely.geometry import Point, shape
+from pyproj import Transformer
+
+bbox = [105.5, 9.2, 106.3, 10.0]
+time_range = "2023-01-01/2023-04-30"
+catalog = pystac_client.Client.open(
+    "https://planetarycomputer.microsoft.com/api/stac/v1",
+    modifier=planetary_computer.sign_inplace,
+)
+items = list(catalog.search(collections=["sentinel-2-l2a"], bbox=bbox, datetime=time_range, query={"eo:cloud_cover": {"lt": 30}}).items())
+items = sorted(items, key=lambda x: x.properties["eo:cloud_cover"])
+
+gdf = gpd.read_file("train/ST_training_data_updated_1130points_new.shp")
+gdf = gdf.to_crs("EPSG:32648")
+
+# Find a point that fails. Let's just test a few points.
+for idx, row in gdf.head(20).iterrows():
+    x_coord = row['geometry'].x
+    y_coord = row['geometry'].y
+    
+    transformer = Transformer.from_crs("epsg:32648", "epsg:4326", always_xy=True)
+    lon, lat = transformer.transform(x_coord, y_coord)
+    point = Point(lon, lat)
+    
+    filtered = []
+    for item in items:
+        if shape(item.geometry).contains(point):
+            filtered.append(item)
+            
+    filtered = [planetary_computer.sign(item) for item in filtered]
+    if not filtered:
+        print(f"Point {idx}: NO ITEMS CONTAINS POINT!")
+        continue
+        
+    ds = odc.stac.load(
+        filtered,
+        bands=["B02"],
+        x=(x_coord - 80, x_coord + 80),
+        y=(y_coord - 80, y_coord + 80),
+        crs="EPSG:32648",
+        resolution=10,
+        patch_url=planetary_computer.sign,
+        fail_on_error=False
+    ).compute()
+    
+    sums = ds["B02"].sum(dim=["x", "y"]).values
+    non_zero = (sums > 0).sum()
+    print(f"Point {idx}: {len(filtered)} items, {non_zero} non-zero time steps")

.temp/test_extract_one.py

@@ -0,0 +1,37 @@
+import geopandas as gpd
+import planetary_computer
+import pystac_client
+import odc.stac
+import sys
+
+bbox = [105.5, 9.2, 106.3, 10.0]
+time_range = "2023-01-01/2023-04-30"
+catalog = pystac_client.Client.open("https://planetarycomputer.microsoft.com/api/stac/v1", modifier=planetary_computer.sign_inplace)
+search = catalog.search(collections=["sentinel-2-l2a"], bbox=bbox, datetime=time_range, query={"eo:cloud_cover": {"lt": 30}})
+items = list(search.items())
+items = sorted(items, key=lambda x: x.properties.get("eo:cloud_cover", 100))[:4]
+items = [planetary_computer.sign(item) for item in items]
+
+gdf = gpd.read_file("train/ST_training_data_updated_1130points_new.shp")
+gdf = gdf.to_crs("EPSG:32648")
+
+row = gdf.iloc[0]
+x, y_coord = row.geometry.x, row.geometry.y
+point_bbox = [x - 80, y_coord - 80, x + 80, y_coord + 80]
+
+patch_s2 = odc.stac.load(
+    items,
+    bands=["B02", "B03", "B04", "B08", "SCL"],
+    x=(x - 80, x + 80),
+    y=(y_coord - 80, y_coord + 80),
+    crs="EPSG:32648",
+    resolution=10,
+    patch_url=planetary_computer.sign,
+    fail_on_error=False
+).compute()
+
+print("patch_s2 vars:", patch_s2.data_vars)
+if patch_s2.dims['x'] < 16 or patch_s2.dims['y'] < 16:
+    print("Too small:", patch_s2.dims)
+else:
+    print("Success dimension:", patch_s2.dims)

.temp/test_feature_extractor.py

@@ -0,0 +1,171 @@
+"""
+Test FeatureExtractor và kiểm tra tích hợp với hệ thống
+"""
+
+import numpy as np
+import xarray as xr
+from feature_extractor import get_feature_extractor
+from pathlib import Path
+
+print("=" * 70)
+print("TESTING FEATURE EXTRACTOR MODULE")
+print("=" * 70)
+
+# Test 1: Simple mode
+print("\n[TEST 1] Simple Mode (3 features)")
+print("-" * 50)
+extractor_simple = get_feature_extractor(mode='simple')
+print(f"✓ Created extractor: {extractor_simple.mode}")
+print(f"✓ Expected features: {extractor_simple.config['n_features']}")
+print(f"✓ Feature names: {extractor_simple.get_feature_names()}")
+
+# Create dummy NDVI data
+ndvi_dummy = xr.DataArray(
+    np.random.rand(10, 10),
+    dims=['y', 'x'],
+    coords={'y': np.arange(10), 'x': np.arange(10)}
+)
+vh_dummy = xr.DataArray(
+    np.random.rand(10, 10) * -10,
+    dims=['y', 'x'],
+    coords={'y': np.arange(10), 'x': np.arange(10)}
+)
+vv_dummy = xr.DataArray(
+    np.random.rand(10, 10) * -8,
+    dims=['y', 'x'],
+    coords={'y': np.arange(10), 'x': np.arange(10)}
+)
+
+features_simple = extractor_simple.extract(
+    ndvi_data=ndvi_dummy,
+    vh_data=vh_dummy,
+    vv_data=vv_dummy
+)
+print(f"✓ Extracted features shape: {features_simple.shape}")
+assert features_simple.shape[1] == 3, "Expected 3 features"
+print("✅ Simple mode test PASSED\n")
+
+# Test 2: Extended mode
+print("[TEST 2] Extended Mode (15 features)")
+print("-" * 50)
+extractor_extended = get_feature_extractor(mode='extended')
+print(f"✓ Created extractor: {extractor_extended.mode}")
+print(f"✓ Expected features: {extractor_extended.config['n_features']}")
+print(f"✓ Feature names: {extractor_extended.get_feature_names()}")
+
+# Create dummy S2 dataset with time dimension
+s2_dummy = xr.Dataset({
+    'B02': xr.DataArray(np.random.rand(5, 10, 10), dims=['time', 'y', 'x']),
+    'B03': xr.DataArray(np.random.rand(5, 10, 10), dims=['time', 'y', 'x']),
+    'B04': xr.DataArray(np.random.rand(5, 10, 10), dims=['time', 'y', 'x']),
+    'B08': xr.DataArray(np.random.rand(5, 10, 10), dims=['time', 'y', 'x']),
+    'B11': xr.DataArray(np.random.rand(5, 10, 10), dims=['time', 'y', 'x'])
+})
+
+features_extended = extractor_extended.extract(
+    s2_data=s2_dummy,
+    vh_data=vh_dummy,
+    vv_data=vv_dummy
+)
+print(f"✓ Extracted features shape: {features_extended.shape}")
+assert features_extended.shape[1] == 15, "Expected 15 features"
+print("✅ Extended mode test PASSED\n")
+
+# Test 3: Temporal mode
+print("[TEST 3] Temporal Mode (39 features for 12 timesteps)")
+print("-" * 50)
+extractor_temporal = get_feature_extractor(mode='temporal')
+print(f"✓ Created extractor: {extractor_temporal.mode}")
+
+# Create dummy S2 dataset with 12 timesteps
+s2_dummy_12 = xr.Dataset({
+    'B02': xr.DataArray(np.random.rand(12, 10, 10), dims=['time', 'y', 'x']),
+    'B03': xr.DataArray(np.random.rand(12, 10, 10), dims=['time', 'y', 'x']),
+    'B04': xr.DataArray(np.random.rand(12, 10, 10), dims=['time', 'y', 'x']),
+    'B08': xr.DataArray(np.random.rand(12, 10, 10), dims=['time', 'y', 'x']),
+    'B11': xr.DataArray(np.random.rand(12, 10, 10), dims=['time', 'y', 'x'])
+})
+
+features_temporal = extractor_temporal.extract(
+    s2_data=s2_dummy_12,
+    vh_data=vh_dummy,
+    vv_data=vv_dummy
+)
+
+# For temporal mode: 12 timesteps * 3 indices + 3 radar = 39 features
+expected_features = 12 * 3 + 3
+print(f"✓ Extracted features shape: {features_temporal.shape}")
+print(f"✓ Expected: {expected_features} features (12 timesteps * 3 indices + 3 radar)")
+
+feature_names_temporal = extractor_temporal.get_feature_names(n_timesteps=12)
+print(f"✓ Feature names count: {len(feature_names_temporal)}")
+print(f"✓ First 5 features: {feature_names_temporal[:5]}")
+print(f"✓ Last 5 features: {feature_names_temporal[-5:]}")
+
+assert features_temporal.shape[1] == expected_features, f"Expected {expected_features} features"
+assert len(feature_names_temporal) == expected_features, f"Expected {expected_features} feature names"
+print("✅ Temporal mode test PASSED\n")
+
+# Test 4: Check model_odc.joblib metadata
+print("[TEST 4] Verify model_odc.joblib metadata")
+print("-" * 50)
+metadata_file = Path("model_train/model_odc_info.json")
+if metadata_file.exists():
+    import json
+    with open(metadata_file) as f:
+        metadata = json.load(f)
+    
+    print(f"✓ Metadata file exists: {metadata_file}")
+    print(f"✓ Feature mode: {metadata.get('feature_mode')}")
+    print(f"✓ Number of features: {metadata.get('n_features')}")
+    print(f"✓ Features list length: {len(metadata.get('features', []))}")
+    print(f"✓ First 5 features: {metadata.get('features', [])[:5]}")
+    
+    assert metadata.get('feature_mode') == 'temporal', "Expected temporal mode"
+    assert metadata.get('n_features') == 39, "Expected 39 features"
+    assert len(metadata.get('features', [])) == 39, "Expected 39 feature names"
+    
+    print("✅ model_odc.joblib metadata VERIFIED\n")
+else:
+    print("❌ model_odc_info.json not found. Run: python create_odc_metadata.py")
+
+# Test 5: Check ModelManager integration
+print("[TEST 5] Test ModelManager integration")
+print("-" * 50)
+try:
+    from model_manager import get_model_manager
+    
+    manager = get_model_manager()
+    print(f"✓ ModelManager initialized")
+    
+    # List models
+    models = manager.list_models()
+    print(f"✓ Found {len(models)} models")
+    
+    # Check if model_odc.joblib has metadata
+    odc_model = next((m for m in models if m['filename'] == 'model_odc.joblib'), None)
+    if odc_model:
+        print(f"✓ model_odc.joblib found in list")
+        print(f"  - Feature mode: {odc_model.get('feature_mode', 'N/A')}")
+        print(f"  - N features: {odc_model.get('n_features', 'N/A')}")
+        print("✅ ModelManager integration test PASSED\n")
+    else:
+        print("⚠️  model_odc.joblib not in model list")
+        
+except Exception as e:
+    print(f"❌ ModelManager test failed: {e}")
+
+# Summary
+print("=" * 70)
+print("TEST SUMMARY")
+print("=" * 70)
+print("✅ All feature extraction modes working correctly")
+print("✅ Feature dimensions match expectations")
+print("✅ Feature names generated correctly")
+print("✅ model_odc.joblib metadata verified")
+print("\nNext steps:")
+print("1. Update api_server.py with run_prediction from run_prediction_new.py")
+print("2. Test training with different feature_modes")
+print("3. Test prediction with models using different modes")
+print("\nSee UPDATE_SUMMARY.md for details.")
+print("=" * 70)

.temp/test_gdf.py

@@ -0,0 +1,3 @@
+import geopandas as gpd
+gdf = gpd.read_file("train/ST_training_data_updated_1130points_new.shp")
+print(gdf.head(1)['HT_code'])

.temp/test_gdf_cols.py

@@ -0,0 +1,3 @@
+import geopandas as gpd
+gdf = gpd.read_file("train/ST_training_data_updated_1130points_new.shp")
+print(gdf.columns)

.temp/test_gpu.py

@@ -0,0 +1,23 @@
+import numpy as np
+from xgboost import XGBClassifier
+from sklearn.datasets import make_classification
+from sklearn.metrics import accuracy_score
+
+print("🚀 Testing XGBoost with CUDA GPU...")
+try:
+    X, y = make_classification(n_samples=10000, n_features=20, n_classes=2, random_state=42)
+    model = XGBClassifier(
+        n_estimators=100,
+        max_depth=10,
+        tree_method="hist",
+        device="cuda",
+        random_state=42,
+        verbosity=1
+    )
+    print("Training model...")
+    model.fit(X, y)
+    y_pred = model.predict(X)
+    acc = accuracy_score(y, y_pred)
+    print(f"✅ Training successful! Accuracy: {acc*100:.2f}%")
+except Exception as e:
+    print(f"❌ Error during training: {e}")

.temp/test_keys.py

@@ -0,0 +1,3 @@
+import torch
+checkpoint = torch.load('cloud_removal_model/cloud_removal_unet_best.pth', map_location='cpu')
+print(list(checkpoint['model_state_dict'].keys())[:5])

.temp/test_model_manager.py

@@ -0,0 +1,101 @@
+"""
+Test script for Model Manager
+Kiểm tra các chức năng: list models, load models, validate models
+"""
+
+from model_manager import ModelManager, get_model_manager
+import json
+
+def test_model_manager():
+    print("="*70)
+    print("MODEL MANAGER TEST")
+    print("="*70)
+    
+    # Initialize ModelManager
+    model_manager = get_model_manager()
+    print("\n✅ ModelManager initialized")
+    
+    # Test 1: List all models
+    print("\n" + "="*70)
+    print("TEST 1: LIST ALL MODELS")
+    print("="*70)
+    
+    models = model_manager.list_models()
+    print(f"\n📦 Found {len(models)} models:")
+    
+    for idx, model in enumerate(models, 1):
+        print(f"\n[{idx}] {model['filename']}")
+        print(f"    Size: {model['size_mb']:.2f} MB")
+        print(f"    Modified: {model['modified']}")
+        
+        if model.get('has_metadata'):
+            print(f"    Type: {model.get('model_type', 'N/A')}")
+            print(f"    Features: {model.get('n_features', 'N/A')}")
+            print(f"    Accuracy: {model.get('test_accuracy', 'N/A')}")
+            print(f"    Feature list: {model.get('features', [])}")
+        else:
+            print(f"    ⚠️  No metadata")
+    
+    # Test 2: Load a model
+    if len(models) > 0:
+        print("\n" + "="*70)
+        print("TEST 2: LOAD MODEL")
+        print("="*70)
+        
+        test_model = models[0]['filename']
+        print(f"\n🔄 Loading model: {test_model}")
+        
+        try:
+            model, encoder, metadata = model_manager.load_model(test_model)
+            print(f"✅ Model loaded successfully!")
+            print(f"\n📊 Metadata:")
+            print(json.dumps(metadata, indent=2))
+            
+            # Test 3: Validate model
+            print("\n" + "="*70)
+            print("TEST 3: VALIDATE MODEL")
+            print("="*70)
+            
+            validation = model_manager.validate_model(test_model)
+            print(f"\n✅ Validation result:")
+            print(f"  Valid: {validation['valid']}")
+            if validation['errors']:
+                print(f"  Errors: {validation['errors']}")
+            if validation['warnings']:
+                print(f"  Warnings: {validation['warnings']}")
+            
+            # Test 4: Get required features
+            print("\n" + "="*70)
+            print("TEST 4: GET REQUIRED FEATURES")
+            print("="*70)
+            
+            features = model_manager.get_required_features(test_model)
+            print(f"\n📋 Required features for {test_model}:")
+            for feat in features:
+                print(f"  - {feat}")
+            
+        except Exception as e:
+            print(f"❌ Error loading model: {e}")
+            import traceback
+            traceback.print_exc()
+    
+    # Test 5: Get latest model
+    print("\n" + "="*70)
+    print("TEST 5: GET LATEST MODEL")
+    print("="*70)
+    
+    latest = model_manager.get_latest_model()
+    print(f"\n📌 Latest model: {latest}")
+    
+    latest_xgb = model_manager.get_latest_model(model_type='xgboost')
+    print(f"📌 Latest XGBoost model: {latest_xgb}")
+    
+    latest_cnn = model_manager.get_latest_model(model_type='cnn')
+    print(f"📌 Latest CNN model: {latest_cnn}")
+    
+    print("\n" + "="*70)
+    print("✅ ALL TESTS COMPLETED")
+    print("="*70)
+
+if __name__ == "__main__":
+    test_model_manager()

.temp/test_model_struct.py

@@ -0,0 +1,13 @@
+import torch
+from pathlib import Path
+model = torch.load('cloud_removal_model/cloud_removal_unet_best.pth', map_location='cpu')
+print(type(model))
+print("hasattr inc:", hasattr(model, 'inc'))
+if hasattr(model, 'inc'):
+    print("hasattr double_conv:", hasattr(model.inc, 'double_conv'))
+    if hasattr(model.inc, 'double_conv'):
+        print("in_channels:", model.inc.double_conv[0].in_channels)
+else:
+    for name, param in model.named_parameters():
+        print(name, param.shape)
+        break

.temp/test_odc_stac.py

@@ -0,0 +1,19 @@
+import geopandas as gpd
+import planetary_computer
+import pystac_client
+import odc.stac
+
+bbox = [105.5, 9.2, 106.3, 10.0]
+time_range = "2023-01-01/2023-01-30"
+catalog = pystac_client.Client.open("https://planetarycomputer.microsoft.com/api/stac/v1", modifier=planetary_computer.sign_inplace)
+items = list(catalog.search(collections=["sentinel-2-l2a"], bbox=bbox, datetime=time_range).items())[:1]
+items = [planetary_computer.sign(item) for item in items]
+
+x = 561609
+y = 1024183
+try:
+    ds = odc.stac.load(items, bands=["B02"], crs="EPSG:32648", resolution=10, x=(x-80, x+80), y=(y-80, y+80))
+    print("Success with x/y:", ds.dims)
+except Exception as e:
+    print("Error with x/y:", e)
+

.temp/test_overlap.py

@@ -0,0 +1,18 @@
+import geopandas as gpd
+import planetary_computer
+import pystac_client
+import odc.stac
+import numpy as np
+
+bbox = [105.5, 9.2, 106.3, 10.0]
+time_range = "2023-01-01/2023-04-30"
+catalog = pystac_client.Client.open("https://planetarycomputer.microsoft.com/api/stac/v1", modifier=planetary_computer.sign_inplace)
+# Get ALL items
+items = list(catalog.search(collections=["sentinel-2-l2a"], bbox=bbox, datetime=time_range, query={"eo:cloud_cover": {"lt": 30}}).items())
+items = [planetary_computer.sign(item) for item in items]
+print(f"Total items: {len(items)}")
+
+x = 561609
+y = 1024183
+ds = odc.stac.load(items, bands=["B02"], x=(x-80, x+80), y=(y-80, y+80), crs="EPSG:32648", resolution=10, patch_url=planetary_computer.sign).compute()
+print("Time dimension size:", ds.dims['time'])

.temp/test_overlap2.py

@@ -0,0 +1,20 @@
+import geopandas as gpd
+import planetary_computer
+import pystac_client
+import odc.stac
+import numpy as np
+
+bbox = [105.5, 9.2, 106.3, 10.0]
+time_range = "2023-01-01/2023-04-30"
+catalog = pystac_client.Client.open("https://planetarycomputer.microsoft.com/api/stac/v1", modifier=planetary_computer.sign_inplace)
+items = list(catalog.search(collections=["sentinel-2-l2a"], bbox=bbox, datetime=time_range, query={"eo:cloud_cover": {"lt": 30}}).items())
+items = [planetary_computer.sign(item) for item in items]
+
+x = 561609
+y = 1024183
+ds = odc.stac.load(items, bands=["B02"], x=(x-80, x+80), y=(y-80, y+80), crs="EPSG:32648", resolution=10, patch_url=planetary_computer.sign, fail_on_error=False).compute()
+print("Original time size:", len(ds.time))
+ds2 = ds.dropna(dim="time", how="all")
+print("After dropna time size:", len(ds2.time))
+print("B02 mean:", np.nanmean(ds2["B02"].values))
+print("B02 non-nan count:", np.sum(~np.isnan(ds2["B02"].values)))

.temp/test_overlap_issue.py

@@ -0,0 +1,25 @@
+import geopandas as gpd
+import planetary_computer
+import pystac_client
+import odc.stac
+import numpy as np
+
+bbox = [105.5, 9.2, 106.3, 10.0]
+time_range = "2023-01-01/2023-04-30"
+catalog = pystac_client.Client.open("https://planetarycomputer.microsoft.com/api/stac/v1", modifier=planetary_computer.sign_inplace)
+items = list(catalog.search(collections=["sentinel-2-l2a"], bbox=bbox, datetime=time_range, query={"eo:cloud_cover": {"lt": 30}}).items())
+items = [planetary_computer.sign(item) for item in items]
+
+x = 561609
+y = 1024183
+ds = odc.stac.load(items, bands=["B02", "B03", "B04", "B08", "SCL"], x=(x-80, x+80), y=(y-80, y+80), crs="EPSG:32648", resolution=10, patch_url=planetary_computer.sign, fail_on_error=False).compute()
+
+print("Original shape:", ds["B02"].shape)
+ds2 = ds.dropna(dim="time", how="all")
+print("After dropna time size:", len(ds2.time))
+if len(ds2.time) > 0:
+    ds2 = ds2.isel(time=slice(0, 4))
+    median = ds2["B02"].median(dim="time", skipna=True).values
+    print("Median shape:", median.shape)
+    print("Zeros in median:", np.sum(median == 0) / median.size)
+    print("NaNs in median:", np.sum(np.isnan(median)) / median.size)

.temp/test_planetary.py

@@ -0,0 +1,15 @@
+import sys
+# Thêm đường dẫn hiện tại vào PYTHONPATH để import được new_import_ODC nếu cần
+sys.path.append('.')
+import warnings
+warnings.filterwarnings('ignore')
+from new_import_ODC import load_sen1
+
+print("Testing load_sen1 with a short time range to speed up Dask compute...")
+bbox = [105.5, 9.2, 106.4, 10.0]
+time_range = "2023-01-01/2023-01-31" # Short time range for fast testing
+vh, vv = load_sen1(bbox, time_range)
+print("VH shape:", vh.shape)
+print("VV shape:", vv.shape)
+print("VH CRS:", vh.rio.crs)
+print("Success!")

.temp/test_planetary2.py

@@ -0,0 +1,49 @@
+import warnings
+warnings.filterwarnings('ignore')
+
+def load_sen1(bbox, time_range):
+    import pystac_client
+    import planetary_computer
+    import odc.stac
+    
+    catalog = pystac_client.Client.open(
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
+        modifier=planetary_computer.sign_inplace,
+    )
+
+    search = catalog.search(
+        collections=["sentinel-1-rtc"],
+        bbox=bbox,
+        datetime=time_range,
+    )
+    items = list(search.items())
+    print("Found items:", len(items))
+
+    ds_s1 = odc.stac.load(
+        items,
+        bands=["vv", "vh"],
+        bbox=bbox,
+        crs="EPSG:32648",
+        resolution=10,
+        chunks={"x": 2048, "y": 2048, "time": 1}
+    )
+
+    ds_median = ds_s1.median(dim="time").compute()
+    vv = ds_median["vv"]
+    vh = ds_median["vh"]
+
+    vv = vv.expand_dims(dim="band")
+    vh = vh.expand_dims(dim="band")
+
+    vv = vv.rio.write_crs("EPSG:32648")
+    vh = vh.rio.write_crs("EPSG:32648")
+
+    return vh, vv
+
+print("Testing load_sen1...")
+bbox = [105.5, 9.2, 106.4, 10.0]
+time_range = "2022-09-01/2023-10-01"
+vh, vv = load_sen1(bbox, time_range)
+print("VH shape:", vh.shape)
+print("VV shape:", vv.shape)
+print("Success!")

.temp/test_planetary_computer.py

@@ -0,0 +1,88 @@
+"""
+Test Microsoft Planetary Computer connectivity và token
+"""
+import planetary_computer
+from pystac_client import Client
+from datetime import datetime, timedelta
+
+print("=" * 70)
+print("🧪 TESTING MICROSOFT PLANETARY COMPUTER CONNECTION")
+print("=" * 70)
+
+# Test 1: Basic connection
+print("\n1️⃣ Testing basic connection...")
+try:
+    catalog = Client.open(
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
+        modifier=planetary_computer.sign_inplace,
+    )
+    print("✅ Successfully connected to Planetary Computer")
+    print(f"   Catalog ID: {catalog.id}")
+    print(f"   Title: {catalog.title}")
+except Exception as e:
+    print(f"❌ Connection failed: {e}")
+    exit(1)
+
+# Test 2: List collections
+print("\n2️⃣ Testing collections access...")
+try:
+    collections = list(catalog.get_collections())
+    print(f"✅ Found {len(collections)} collections")
+    sentinel_2 = [c for c in collections if 'sentinel-2' in c.id.lower()]
+    print(f"   Sentinel-2 collections: {[c.id for c in sentinel_2]}")
+except Exception as e:
+    print(f"❌ Collections access failed: {e}")
+
+# Test 3: Small search query (very conservative)
+print("\n3️⃣ Testing small search query...")
+try:
+    # Tiny bbox in Vietnam
+    bbox = [105.8, 10.0, 105.9, 10.1]  # ~10km x 10km area
+    end_date = datetime.now()
+    start_date = end_date - timedelta(days=7)  # Last 7 days only
+    
+    time_range = f"{start_date.strftime('%Y-%m-%d')}/{end_date.strftime('%Y-%m-%d')}"
+    
+    print(f"   Bbox: {bbox}")
+    print(f"   Time: {time_range}")
+    print(f"   Searching...")
+    
+    search = catalog.search(
+        collections=["sentinel-2-l2a"],
+        bbox=bbox,
+        datetime=time_range,
+        limit=5  # Only 5 items
+    )
+    
+    items = []
+    for i, item in enumerate(search.items()):
+        items.append(item)
+        if i >= 4:  # Stop at 5
+            break
+    
+    print(f"✅ Search successful! Found {len(items)} items")
+    if items:
+        first_item = items[0]
+        print(f"   First item: {first_item.id}")
+        print(f"   Date: {first_item.datetime}")
+        
+        # Test token signing
+        signed_item = planetary_computer.sign(first_item)
+        print(f"✅ SAS token signing works")
+        print(f"   Asset keys: {list(signed_item.assets.keys())[:5]}")
+        
+except Exception as e:
+    print(f"❌ Search failed: {e}")
+    import traceback
+    traceback.print_exc()
+
+print("\n" + "=" * 70)
+print("🏁 Test completed!")
+print("=" * 70)
+print("\n💡 Nếu test này PASS:")
+print("   → Planetary Computer hoạt động bình thường")
+print("   → Vấn đề là query quá lớn (bbox/time range/max_scenes)")
+print("\n💡 Nếu test này FAIL:")
+print("   → Kiểm tra internet connection")
+print("   → Thử lại sau (server có thể bị quá tải)")
+print("   → Xem xét dùng dữ liệu local")

.temp/test_raw.py

@@ -0,0 +1,17 @@
+import geopandas as gpd
+import planetary_computer
+import pystac_client
+import odc.stac
+import numpy as np
+
+bbox = [105.5, 9.2, 106.3, 10.0]
+time_range = "2023-01-01/2023-04-30"
+catalog = pystac_client.Client.open("https://planetarycomputer.microsoft.com/api/stac/v1", modifier=planetary_computer.sign_inplace)
+items = list(catalog.search(collections=["sentinel-2-l2a"], bbox=bbox, datetime=time_range, query={"eo:cloud_cover": {"lt": 30}}).items())[:4]
+items = [planetary_computer.sign(item) for item in items]
+
+x = 561609
+y = 1024183
+ds = odc.stac.load(items, bands=["B02", "B03", "B04", "B08"], x=(x-80, x+80), y=(y-80, y+80), crs="EPSG:32648", resolution=10, patch_url=planetary_computer.sign).compute()
+print("B04 nanmean:", np.nanmean(ds["B04"].values))
+print("B04 nanmax:", np.nanmax(ds["B04"].values))

.temp/test_shape.py

@@ -0,0 +1,41 @@
+import new_import_ODC
+importlib = __import__('importlib')
+importlib.reload(new_import_ODC)
+from new_import_ODC import *
+import numpy as np
+
+date_range = ("2022-09-01", "2022-10-01")
+longtitude_range = (105.86, 105.94)
+latitude_range = (9.65, 9.69)
+coordinates = (longtitude_range, latitude_range)
+
+print("Loading S2...")
+data = load_data(None, date_range, longtitude_range, latitude_range)
+result = mask_clean(data)
+ds1 = calculate_indices(result, index="NDVI", satellite_mission="s2")
+ndvi = ds1["NDVI"]
+time_split = [
+    slice("2022-09-01", "2023-01-01"),
+    slice("2023-01-01", "2023-05-01"),
+    slice("2023-05-01", "2023-07-01"),
+    slice("2023-07-01", "2022-10-01"),
+]
+fill_nan_ndvi = fill_nan(ndvi, time_split)
+average_ndvi = fill_nan_ndvi.resample(time="1M").mean().compute()
+
+print("Loading S1...")
+dsvh, dsvv = load_data_sen1(None, date_range, coordinates)
+average_vv = calculate_average(dsvv, time_pattern='1M')
+average_vh = calculate_average(dsvh, time_pattern='1M')
+
+train = load_train_data("train/ST_training_data_updated_1130points_new.shp")
+point = train.iloc[0]
+
+ndvi_val = average_ndvi.sel(x=point.geometry.x, y=point.geometry.y, method='nearest').values
+vh_val = average_vh.sel(x=point.geometry.x, y=point.geometry.y, method='nearest').values
+vv_val = average_vv.sel(x=point.geometry.x, y=point.geometry.y, method='nearest').values
+
+print("NDVI shape:", ndvi_val.shape, "ndim:", ndvi_val.ndim)
+print("VH shape:", vh_val.shape, "ndim:", vh_val.ndim)
+print("VV shape:", vv_val.shape, "ndim:", vv_val.ndim)
+

.temp/test_spatial_filter.py

@@ -0,0 +1,42 @@
+import geopandas as gpd
+import planetary_computer
+import pystac_client
+import odc.stac
+import numpy as np
+import time
+from shapely.geometry import Point, box, shape
+
+bbox = [105.5, 9.2, 106.3, 10.0]
+time_range = "2023-01-01/2023-04-30"
+catalog = pystac_client.Client.open("https://planetarycomputer.microsoft.com/api/stac/v1", modifier=planetary_computer.sign_inplace)
+items = list(catalog.search(collections=["sentinel-2-l2a"], bbox=bbox, datetime=time_range, query={"eo:cloud_cover": {"lt": 30}}).items())
+
+x = 561609
+y = 1024183
+
+start = time.time()
+# Filter items by spatial intersection
+from pyproj import Transformer
+# The items geometry are in EPSG:4326 (lon, lat)
+# Our x, y are in EPSG:32648
+transformer = Transformer.from_crs("epsg:32648", "epsg:4326", always_xy=True)
+lon, lat = transformer.transform(x, y)
+point = Point(lon, lat)
+
+filtered_items = []
+for item in items:
+    geom = shape(item.geometry)
+    if geom.contains(point):
+        filtered_items.append(item)
+
+filtered_items = sorted(filtered_items, key=lambda x: x.properties["eo:cloud_cover"])
+
+print("Original items:", len(items))
+print("Filtered items:", len(filtered_items))
+print("Time to filter:", time.time() - start)
+
+start = time.time()
+filtered_items = [planetary_computer.sign(item) for item in filtered_items]
+ds = odc.stac.load(filtered_items[:4], bands=["B02"], x=(x-80, x+80), y=(y-80, y+80), crs="EPSG:32648", resolution=10, patch_url=planetary_computer.sign, fail_on_error=False).compute()
+print("Time to load 4 items:", time.time() - start)
+print(ds["B02"].shape)

.temp/test_training_api.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+"""
+Test script to verify training API endpoints
+"""
+
+import requests
+import json
+
+API_BASE = "http://localhost:8000/api"
+
+def test_training_labels():
+    """Test /api/training/labels endpoint"""
+    print("=" * 70)
+    print("TEST 1: Getting training labels")
+    print("=" * 70)
+    
+    response = requests.get(f"{API_BASE}/training/labels")
+    if response.ok:
+        data = response.json()
+        print(f"✅ Success! Found {data['count']} labels:")
+        for label in data['labels']:
+            print(f"  {label['code']}: {label['name']}")
+    else:
+        print(f"❌ Error: {response.status_code}")
+    print()
+
+def test_training_files():
+    """Test /api/training/files endpoint"""
+    print("=" * 70)
+    print("TEST 2: Getting training files")
+    print("=" * 70)
+    
+    response = requests.get(f"{API_BASE}/training/files")
+    if response.ok:
+        data = response.json()
+        print(f"✅ Success! Found {data['count']} training files:")
+        for file in data['files']:
+            print(f"\n  📄 {file['filename']}")
+            print(f"     Size: {file['size_mb']} MB")
+            if 'point_count' in file:
+                print(f"     Points: {file['point_count']}")
+                print(f"     Label column: {file.get('label_column', 'N/A')}")
+                print(f"     Unique labels: {file.get('label_count', 0)}")
+    else:
+        print(f"❌ Error: {response.status_code}")
+    print()
+
+def test_shapefile_labels(filename="ST_training data_updated_1130points_new.shp"):
+    """Test /api/training/shapefile/{filename}/labels endpoint"""
+    print("=" * 70)
+    print(f"TEST 3: Getting labels from shapefile: {filename}")
+    print("=" * 70)
+    
+    response = requests.get(f"{API_BASE}/training/shapefile/{filename}/labels")
+    if response.ok:
+        data = response.json()
+        print(f"✅ Success!")
+        print(f"  Filename: {data['filename']}")
+        print(f"  Points: {data['point_count']}")
+        print(f"  Label column: {data['label_column']}")
+        print(f"  Unique labels: {data['label_count']}")
+        print(f"  Bbox: {data['bbox']}")
+        print(f"\n  Labels distribution:")
+        for label in data['labels']:
+            mapped = "✅" if label['mapped'] else "⚠️"
+            print(f"    {mapped} {label['name']}: {label['count']} points (code: {label['code']})")
+    else:
+        print(f"❌ Error: {response.status_code}")
+        print(response.text)
+    print()
+
+def test_config_presets():
+    """Test /api/config/presets endpoint"""
+    print("=" * 70)
+    print("TEST 4: Getting config presets")
+    print("=" * 70)
+    
+    response = requests.get(f"{API_BASE}/config/presets")
+    if response.ok:
+        data = response.json()
+        print(f"✅ Success! Found {len(data['presets'])} presets:")
+        for preset in data['presets']:
+            print(f"\n  📋 {preset['name']}")
+            config = preset['config']
+            print(f"     Bbox: [{config['min_lon']}, {config['min_lat']}, {config['max_lon']}, {config['max_lat']}]")
+            print(f"     Time: {config['start_date']} → {config['end_date']}")
+            print(f"     Resolution: {config['resolution']}m")
+    else:
+        print(f"❌ Error: {response.status_code}")
+    print()
+
+if __name__ == "__main__":
+    print("\n" + "=" * 70)
+    print("🧪 TESTING TRAINING API ENDPOINTS")
+    print("=" * 70 + "\n")
+    
+    try:
+        test_training_labels()
+        test_training_files()
+        test_shapefile_labels()
+        test_config_presets()
+        
+        print("=" * 70)
+        print("✅ ALL TESTS COMPLETED!")
+        print("=" * 70)
+        
+    except requests.exceptions.ConnectionError:
+        print("\n❌ Error: Cannot connect to API server")
+        print("Make sure the server is running: python api_server.py")
+    except Exception as e:
+        print(f"\n❌ Error: {e}")
+        import traceback
+        traceback.print_exc()

.temp/test_unet.py

@@ -0,0 +1,5 @@
+from train_cloud_removal import UNet
+model = UNet(in_channels=6, out_channels=4)
+print(hasattr(model, 'inc'))
+print(hasattr(model, 'conv1'))
+print(list(model.parameters())[0].shape)

.temp/test_y_mapped.py

@@ -0,0 +1,13 @@
+import numpy as np
+
+y = []
+# simulate appending 1130 labels
+for i in range(1130):
+    y.append(i % 5)
+
+y = np.array(y)
+unique_labels = sorted(list(np.unique(y)))
+label_map = {lbl: i for i, lbl in enumerate(unique_labels)}
+y_mapped = np.array([label_map[l] for l in y])
+
+print(len(y), len(y_mapped))

01.train_ODC.ipynb

@@ -1,947 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "id": "912ed572-1658-406b-976c-cd6de2d4e89e",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [
-    {
-     "ename": "ModuleNotFoundError",
-     "evalue": "No module named 'easi_tools'",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
-      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
-      "File \u001b[0;32m<timed exec>:4\u001b[0m\n",
-      "File \u001b[0;32m~/CSIROBoeingPhase5-Vietnam/new_import_ODC.py:23\u001b[0m\n\u001b[1;32m     21\u001b[0m easinotebooksrepo \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/home/jovyan/easi-notebooks\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m     22\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m easinotebooksrepo \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m sys\u001b[38;5;241m.\u001b[39mpath: sys\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mappend(easinotebooksrepo)\n\u001b[0;32m---> 23\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01measi_tools\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m EasiDefaults, xarray_object_size, notebook_utils, unset_cachingproxy\n\u001b[1;32m     24\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01measi_tools\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mload_s2l2a\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m load_s2l2a_with_offset\n\u001b[1;32m     25\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mdask\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdistributed\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m progress\n",
-      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'easi_tools'"
-     ]
-    }
-   ],
-   "source": [
-    "%%time\n",
-    "%matplotlib inline\n",
-    "\n",
-    "import importlib\n",
-    "import new_import_ODC  \n",
-    "\n",
-    "importlib.reload(new_import_ODC)\n",
-    "\n",
-    "from new_import_ODC import *\n",
-    "\n",
-    "print(\"✅ All modules loaded successfully\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "id": "d824dc4f-994b-4d1c-8d24-ce6674da141c",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "✅ AWS credentials loaded from environment variables\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/home/x79/miniconda/envs/env_01/lib/python3.10/site-packages/distributed/node.py:187: UserWarning: Port 8787 is already in use.\n",
-      "Perhaps you already have a cluster running?\n",
-      "Hosting the HTTP server on port 41709 instead\n",
-      "  warnings.warn(\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "✅ Dask cluster initialized\n",
-      "   Cluster: LocalCluster(9f2167a3, 'tcp://127.0.0.1:41233', workers=4, threads=24, memory=31.26 GiB)\n",
-      "✅ Datacube connected (metadata only)\n",
-      "\n",
-      "======================================================================\n",
-      "CPU times: user 4.98 s, sys: 831 ms, total: 5.81 s\n",
-      "Wall time: 7.69 s\n"
-     ]
-    }
-   ],
-   "source": [
-    "%%time\n",
-    "import os\n",
-    "import sys\n",
-    "\n",
-    "print(\"✅ AWS credentials loaded from environment variables\")\n",
-    "\n",
-    "# Cấu hình Dask local\n",
-    "from dask.distributed import Client, LocalCluster\n",
-    "\n",
-    "cluster = LocalCluster(n_workers=4)\n",
-    "client = Client(cluster)\n",
-    "print(\"✅ Dask cluster initialized\")\n",
-    "print(f\"   Cluster: {cluster}\")\n",
-    "\n",
-    "# Khai báo Datacube (chỉ để lấy metadata, không dùng load())\n",
-    "import datacube\n",
-    "try:\n",
-    "    dc = datacube.Datacube()\n",
-    "    print(\"✅ Datacube connected (metadata only)\")\n",
-    "except Exception as e:\n",
-    "    print(f\"⚠️  Datacube connection not critical: {e}\")\n",
-    "    dc = None\n",
-    "\n",
-    "print(\"\\n\" + \"=\"*70)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "id": "1e113730",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "======================================================================\n",
-      "GETTING SENTINEL-2 SCENE METADATA\n",
-      "======================================================================\n",
-      "\n",
-      "[1] Loading metadata from datacube...\n",
-      "    ✅ Found 40 scenes\n",
-      "\n",
-      "[2] Selected scene: S2A_48PWR_20231226_0_L2A\n",
-      "    Date: 2023-12-26 03:35:26.919000+00:00\n",
-      "\n",
-      "[3] Available bands:\n",
-      "    - nir: https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/48/P/WR/20\n",
-      "    - red: https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/48/P/WR/20\n",
-      "    - scl: https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/48/P/WR/20\n",
-      "    - blue: https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/48/P/WR/20\n",
-      "    - green: https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/48/P/WR/20\n",
-      "    - nir08: https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/48/P/WR/20\n",
-      "    - nir09: https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/48/P/WR/20\n",
-      "    - swir16: https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/48/P/WR/20\n",
-      "    - swir22: https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/48/P/WR/20\n",
-      "    - coastal: https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/48/P/WR/20\n",
-      "    - rededge1: https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/48/P/WR/20\n",
-      "    - rededge2: https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/48/P/WR/20\n",
-      "    - rededge3: https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/48/P/WR/20\n",
-      "======================================================================\n",
-      "CPU times: user 3.34 s, sys: 76.3 ms, total: 3.41 s\n",
-      "Wall time: 3.23 s\n"
-     ]
-    }
-   ],
-   "source": [
-    "%%time\n",
-    "# 🔧 Get Sentinel-2 scene metadata from datacube\n",
-    "print(\"=\"*70)\n",
-    "print(\"GETTING SENTINEL-2 SCENE METADATA\")\n",
-    "print(\"=\"*70)\n",
-    "\n",
-    "date_range = (\"2023-03-01\", \"2023-12-31\")\n",
-    "longtitude_range = (105.5, 106.4)\n",
-    "latitude_range = (9.2, 10.0)\n",
-    "\n",
-    "try:\n",
-    "    print(f\"\\n[1] Loading metadata from datacube...\")\n",
-    "    datasets = list(dc.find_datasets(product='s2_l2a', time=date_range))\n",
-    "    print(f\"    ✅ Found {len(datasets)} scenes\")\n",
-    "    \n",
-    "    if datasets:\n",
-    "        selected = datasets[0]\n",
-    "        print(f\"\\n[2] Selected scene: {selected.metadata.label}\")\n",
-    "        scene_datetime = selected.time.begin if hasattr(selected.time, 'begin') else selected.time\n",
-    "        print(f\"    Date: {scene_datetime}\")\n",
-    "        \n",
-    "        # Display measurement paths\n",
-    "        print(f\"\\n[3] Available bands:\")\n",
-    "        for name, measurement in selected.measurements.items():\n",
-    "            print(f\"    - {name}: {measurement['path'][:80]}\")\n",
-    "            \n",
-    "except Exception as e:\n",
-    "    print(f\"❌ Error: {e}\")\n",
-    "    import traceback\n",
-    "    traceback.print_exc()\n",
-    "\n",
-    "print(\"=\"*70)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "id": "3cd69645",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "======================================================================\n",
-      "CHECKING FOR CACHED DATASET\n",
-      "======================================================================\n",
-      "\n",
-      "⏳ Cache file not found: dataset_cache/sentinel2_timeseries_40scenes.nc\n",
-      "   Will download from S3 and save cache\n",
-      "   (Next run will use cache automatically)\n",
-      "======================================================================\n",
-      "CPU times: user 4.36 ms, sys: 3.7 ms, total: 8.06 ms\n",
-      "Wall time: 7.22 ms\n"
-     ]
-    }
-   ],
-   "source": [
-    "%%time\n",
-    "# 🔍 CHECK IF DATASET CACHE EXISTS (Skip download if available)\n",
-    "print(\"=\"*70)\n",
-    "print(\"CHECKING FOR CACHED DATASET\")\n",
-    "print(\"=\"*70)\n",
-    "\n",
-    "import os\n",
-    "import xarray as xr\n",
-    "\n",
-    "cache_dir = \"dataset_cache\"\n",
-    "cache_file = f\"{cache_dir}/sentinel2_timeseries_40scenes.nc\"\n",
-    "\n",
-    "use_cache = False\n",
-    "\n",
-    "if os.path.exists(cache_file):\n",
-    "    print(f\"\\n✅ Cache file found: {cache_file}\")\n",
-    "    \n",
-    "    # Get file info\n",
-    "    file_size_gb = os.path.getsize(cache_file) / (1024**3)\n",
-    "    print(f\"   File size: {file_size_gb:.2f} GB\")\n",
-    "    \n",
-    "    # Try to load\n",
-    "    try:\n",
-    "        print(f\"\\n🔄 Loading dataset from cache...\")\n",
-    "        data = xr.open_dataset(cache_file)\n",
-    "        \n",
-    "        print(f\"✅ Dataset loaded from cache!\")\n",
-    "        print(f\"   Total scenes: {len(data['time'])}\")\n",
-    "        print(f\"   Variables: {len(data.data_vars)}\")\n",
-    "        print(f\"   Dimensions: {dict(data.dims)}\")\n",
-    "        print(f\"\\n   ⏭️  Skipping S3 download (using cached data)\")\n",
-    "        \n",
-    "        use_cache = True\n",
-    "        \n",
-    "    except Exception as e:\n",
-    "        print(f\"❌ Error loading cache: {e}\")\n",
-    "        print(f\"   Will download fresh data from S3\")\n",
-    "        use_cache = False\n",
-    "else:\n",
-    "    print(f\"\\n⏳ Cache file not found: {cache_file}\")\n",
-    "    print(f\"   Will download from S3 and save cache\")\n",
-    "    print(f\"   (Next run will use cache automatically)\")\n",
-    "\n",
-    "print(\"=\"*70)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "id": "435f9f78-a9a4-4226-86ca-d4bec42d454e",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "======================================================================\n",
-      "LOADING SENTINEL-2 DATA FROM S3 COGs (RASTERIO) - OPTIMAL ACCURACY\n",
-      "======================================================================\n",
-      "\n",
-      "📥 Downloading from S3...\n",
-      "\n",
-      "📦 Found 40 available scenes\n",
-      "   Date range: 2023-03-01 to 2023-12-31\n",
-      "\n",
-      "[LOADING] Loading ALL 40 scenes with ALL available bands...\n",
-      "   (Keeping NATIVE resolution - NO upsampling/magnification)\n",
-      "   Available bands: ['nir', 'red', 'scl', 'blue', 'green', 'nir08', 'nir09', 'swir16', 'swir22', 'coastal', 'rededge1', 'rededge2', 'rededge3']\n",
-      "\n",
-      "   [ 1/1] S2A_48PWR_20231226_0_L2A (2023-12-26)\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "      ✅ 13 bands loaded\n",
-      "\n",
-      "✅ Successfully loaded 1 scenes!\n",
-      "\n",
-      "[RESOLUTION NORMALIZATION] Aligning all bands to native resolution (NO magnification)...\n",
-      "   Reference resolution: 10980×10980 pixels (native nir)\n",
-      "      Resampling scl: 5490×5490 → 10980×10980\n",
-      "      Resampling nir08: 5490×5490 → 10980×10980\n",
-      "      Resampling nir09: 1830×1830 → 10980×10980\n",
-      "      Resampling swir16: 5490×5490 → 10980×10980\n",
-      "      Resampling swir22: 5490×5490 → 10980×10980\n",
-      "      Resampling coastal: 1830×1830 → 10980×10980\n",
-      "      Resampling rededge1: 5490×5490 → 10980×10980\n",
-      "      Resampling rededge2: 5490×5490 → 10980×10980\n",
-      "      Resampling rededge3: 5490×5490 → 10980×10980\n",
-      "✅ Resolution normalization complete! (9 bands resampled)\n",
-      "\n",
-      "[SPECTRAL INDICES] Calculating spectral indices for each scene...\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "<timed exec>:169: RuntimeWarning: divide by zero encountered in divide\n",
-      "<timed exec>:169: RuntimeWarning: invalid value encountered in divide\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "✅ Calculated 2 spectral indices per scene\n",
-      "\n",
-      "[STACKING] Stacking all 1 scenes to create time-series...\n",
-      "\n",
-      "[TEMPORAL FEATURES] Computing temporal features from time-series...\n",
-      "✅ Added 6 temporal/aggregate features\n",
-      "\n",
-      "[CACHE] Saving dataset to cache...\n",
-      "✅ Dataset saved to cache: dataset_cache/sentinel2_timeseries_40scenes.nc\n",
-      "   Cache size: 6.40 GB\n",
-      "\n",
-      "✅ OPTIMAL Dataset with native resolution + temporal features created!\n",
-      "   ======================================================================\n",
-      "   🎬 Total scenes (time steps): 1\n",
-      "   📊 Total bands/variables: 21\n",
-      "   🖼️  Spatial size: 10980 × 10980 pixels (NATIVE resolution)\n",
-      "   📏 Native resolution: 10m (Sentinel-2 L2A)\n",
-      "   ⏰ Temporal range: 2023-12-26 to 2023-12-26\n",
-      "❌ Error: name 'notebook_utils' is not defined\n",
-      "======================================================================\n",
-      "CPU times: user 9min, sys: 4min 16s, total: 13min 16s\n",
-      "Wall time: 16min 42s\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Traceback (most recent call last):\n",
-      "  File \"<timed exec>\", line 262, in <module>\n",
-      "NameError: name 'notebook_utils' is not defined\n"
-     ]
-    }
-   ],
-   "source": [
-    "%%time\n",
-    "# 💾 LOAD SENTINEL-2 DATA DIRECTLY FROM S3 COGS USING RASTERIO - WITH TEMPORAL FEATURES\n",
-    "print(\"=\"*70)\n",
-    "print(\"LOADING SENTINEL-2 DATA FROM S3 COGs (RASTERIO) - OPTIMAL ACCURACY\")\n",
-    "print(\"=\"*70)\n",
-    "\n",
-    "try:\n",
-    "    import rasterio\n",
-    "    import xarray as xr\n",
-    "    import numpy as np\n",
-    "    from scipy import ndimage\n",
-    "    \n",
-    "    # ===== CHECK IF SHOULD SKIP DOWNLOAD =====\n",
-    "    if use_cache and data is not None:\n",
-    "        print(f\"\\n✅ Using cached dataset - skipping download!\")\n",
-    "        print(f\"   Variables: {len(data.data_vars)}\")\n",
-    "        print(f\"   Shape: {data.dims}\")\n",
-    "        display(data)\n",
-    "    \n",
-    "    else:\n",
-    "        # ===== DOWNLOAD FROM S3 =====\n",
-    "        print(f\"\\n📥 Downloading from S3...\")\n",
-    "        \n",
-    "        # Get all scenes from datacube metadata\n",
-    "        datasets = list(dc.find_datasets(\n",
-    "            product='s2_l2a',\n",
-    "            time=date_range\n",
-    "        ))\n",
-    "        \n",
-    "        if not datasets:\n",
-    "            raise ValueError(\"No datasets found for date range\")\n",
-    "        \n",
-    "        print(f\"\\n📦 Found {len(datasets)} available scenes\")\n",
-    "        print(f\"   Date range: {date_range[0]} to {date_range[1]}\")\n",
-    "        \n",
-    "        # ===== LOAD ALL SCENES WITH ALL AVAILABLE BANDS (NO MAGNIFICATION) =====\n",
-    "        print(f\"\\n[LOADING] Loading ALL {len(datasets)} scenes with ALL available bands...\")\n",
-    "        print(f\"   (Keeping NATIVE resolution - NO upsampling/magnification)\")\n",
-    "        \n",
-    "        # num_scenes = len(datasets)  # Load ALL scenes\n",
-    "        num_scenes = 1 # Load ALL scenes\n",
-    "        all_data_dict = {}\n",
-    "        failed_scenes = []\n",
-    "        scene_dates = []\n",
-    "        \n",
-    "        # Discover all available bands from first scene\n",
-    "        first_scene = datasets[0]\n",
-    "        all_available_bands = list(first_scene.measurements.keys())\n",
-    "        print(f\"   Available bands: {all_available_bands}\")\n",
-    "        \n",
-    "        for scene_idx in range(num_scenes):\n",
-    "            selected = datasets[scene_idx]\n",
-    "            scene_label = selected.metadata.label\n",
-    "            scene_datetime = selected.time.begin if hasattr(selected.time, 'begin') else selected.time\n",
-    "            scene_dates.append(scene_datetime)\n",
-    "            \n",
-    "            # Print progress every 5 scenes\n",
-    "            if scene_idx % 5 == 0 or scene_idx == 0 or scene_idx == num_scenes - 1:\n",
-    "                print(f\"\\n   [{scene_idx + 1:2d}/{num_scenes}] {scene_label} ({scene_datetime.date()})\")\n",
-    "            \n",
-    "            # Load ALL available bands from S3 COGs\n",
-    "            scene_data_dict = {}\n",
-    "            \n",
-    "            for band_name in all_available_bands:\n",
-    "                if band_name in selected.measurements:\n",
-    "                    band_path = selected.measurements[band_name]['path']\n",
-    "                    \n",
-    "                    try:\n",
-    "                        with rasterio.open(band_path) as src:\n",
-    "                            data_band = src.read(1)\n",
-    "                            scene_data_dict[band_name] = data_band\n",
-    "                    except Exception as e:\n",
-    "                        if scene_idx % 5 == 0:\n",
-    "                            print(f\"      ⚠️ Error loading {band_name}: {str(e)[:30]}\")\n",
-    "                        failed_scenes.append((scene_idx, scene_label, band_name, str(e)))\n",
-    "            \n",
-    "            if scene_data_dict:\n",
-    "                all_data_dict[scene_idx] = scene_data_dict\n",
-    "                if scene_idx % 5 == 0 or scene_idx == num_scenes - 1:\n",
-    "                    print(f\"      ✅ {len(scene_data_dict)} bands loaded\")\n",
-    "            else:\n",
-    "                failed_scenes.append((scene_idx, scene_label, \"all\", \"No bands loaded\"))\n",
-    "        \n",
-    "        if not all_data_dict:\n",
-    "            raise ValueError(\"Could not load any bands from any scene\")\n",
-    "        \n",
-    "        print(f\"\\n✅ Successfully loaded {len(all_data_dict)} scenes!\")\n",
-    "        if failed_scenes:\n",
-    "            print(f\"⚠️  Failed to load {len(failed_scenes)} band instances (will be skipped)\")\n",
-    "        \n",
-    "        # ===== NORMALIZE RESOLUTION (No upsampling - just match to highest) =====\n",
-    "        print(f\"\\n[RESOLUTION NORMALIZATION] Aligning all bands to native resolution (NO magnification)...\")\n",
-    "        \n",
-    "        # Find max resolution\n",
-    "        ref_resolution = None\n",
-    "        max_size = 0\n",
-    "        max_band = None\n",
-    "        \n",
-    "        for scene_idx in all_data_dict.keys():\n",
-    "            for band_name, data_band in all_data_dict[scene_idx].items():\n",
-    "                size = data_band.shape[0]\n",
-    "                if size > max_size:\n",
-    "                    max_size = size\n",
-    "                    ref_resolution = size\n",
-    "                    max_band = band_name\n",
-    "        \n",
-    "        print(f\"   Reference resolution: {max_size}×{max_size} pixels (native {max_band})\")\n",
-    "        \n",
-    "        # Resample all bands to match reference resolution (both up and down)\n",
-    "        resampled_count = 0\n",
-    "        for scene_idx in all_data_dict.keys():\n",
-    "            for band_name in list(all_data_dict[scene_idx].keys()):\n",
-    "                band_data_arr = all_data_dict[scene_idx][band_name]\n",
-    "                current_size = band_data_arr.shape[0]\n",
-    "                \n",
-    "                if current_size != ref_resolution:\n",
-    "                    scale_factor = ref_resolution / current_size\n",
-    "                    \n",
-    "                    # Resample to match reference resolution (both up and down)\n",
-    "                    if band_name == 'scl':\n",
-    "                        resampled_data = ndimage.zoom(band_data_arr, scale_factor, order=0)\n",
-    "                    else:\n",
-    "                        resampled_data = ndimage.zoom(band_data_arr, scale_factor, order=1)\n",
-    "                    \n",
-    "                    all_data_dict[scene_idx][band_name] = resampled_data\n",
-    "                    new_size = resampled_data.shape[0]\n",
-    "                    if scene_idx == 0:  # Print for first scene only to reduce clutter\n",
-    "                        print(f\"      Resampling {band_name}: {current_size}×{current_size} → {new_size}×{new_size}\")\n",
-    "                    resampled_count += 1\n",
-    "        \n",
-    "        print(f\"✅ Resolution normalization complete! ({resampled_count} bands resampled)\")\n",
-    "        \n",
-    "        # ===== CALCULATE SPECTRAL INDICES FOR EACH SCENE =====\n",
-    "        print(f\"\\n[SPECTRAL INDICES] Calculating spectral indices for each scene...\")\n",
-    "        \n",
-    "        indices_count = 0\n",
-    "        for scene_idx in all_data_dict.keys():\n",
-    "            scene_data = all_data_dict[scene_idx]\n",
-    "            \n",
-    "            try:\n",
-    "                # NDVI: (NIR - Red) / (NIR + Red)\n",
-    "                if 'nir' in scene_data and 'red' in scene_data:\n",
-    "                    nir = scene_data['nir'].astype(float)\n",
-    "                    red = scene_data['red'].astype(float)\n",
-    "                    ndvi = (nir - red) / (nir + red + 1e-8)\n",
-    "                    scene_data['ndvi'] = ndvi.astype(np.float32)\n",
-    "                    indices_count += 1\n",
-    "                \n",
-    "                # NDBI: (SWIR - NIR) / (SWIR + NIR)\n",
-    "                if 'b11' in scene_data and 'nir' in scene_data:\n",
-    "                    swir = scene_data['b11'].astype(float)\n",
-    "                    nir = scene_data['nir'].astype(float)\n",
-    "                    ndbi = (swir - nir) / (swir + nir + 1e-8)\n",
-    "                    scene_data['ndbi'] = ndbi.astype(np.float32)\n",
-    "                    indices_count += 1\n",
-    "                \n",
-    "                # NDWI: (NIR - SWIR) / (NIR + SWIR)\n",
-    "                if 'nir' in scene_data and 'b11' in scene_data:\n",
-    "                    nir = scene_data['nir'].astype(float)\n",
-    "                    swir = scene_data['b11'].astype(float)\n",
-    "                    ndwi = (nir - swir) / (nir + swir + 1e-8)\n",
-    "                    scene_data['ndwi'] = ndwi.astype(np.float32)\n",
-    "                    indices_count += 1\n",
-    "                \n",
-    "                # EVI: Enhanced Vegetation Index\n",
-    "                if 'nir' in scene_data and 'red' in scene_data and 'blue' in scene_data:\n",
-    "                    nir = scene_data['nir'].astype(float)\n",
-    "                    red = scene_data['red'].astype(float)\n",
-    "                    blue = scene_data['blue'].astype(float)\n",
-    "                    evi = 2.5 * (nir - red) / (nir + 6*red - 7.5*blue + 1)\n",
-    "                    scene_data['evi'] = evi.astype(np.float32)\n",
-    "                    indices_count += 1\n",
-    "                    \n",
-    "            except Exception as e:\n",
-    "                pass\n",
-    "        \n",
-    "        print(f\"✅ Calculated {indices_count} spectral indices per scene\")\n",
-    "        \n",
-    "        # ===== STACK SCENES ALONG TIME DIMENSION =====\n",
-    "        print(f\"\\n[STACKING] Stacking all {len(all_data_dict)} scenes to create time-series...\")\n",
-    "        \n",
-    "        data_vars = {}\n",
-    "        band_names = list(all_data_dict[0].keys())\n",
-    "        \n",
-    "        for band_name in band_names:\n",
-    "            band_data_list = []\n",
-    "            for scene_idx in sorted(all_data_dict.keys()):\n",
-    "                if band_name in all_data_dict[scene_idx]:\n",
-    "                    band_data_list.append(all_data_dict[scene_idx][band_name])\n",
-    "            \n",
-    "            if band_data_list:\n",
-    "                stacked = np.stack(band_data_list, axis=0)\n",
-    "                data_vars[band_name] = (['time', 'y', 'x'], stacked)\n",
-    "        \n",
-    "        # Create xarray Dataset with time dimension\n",
-    "        first_band_data = list(all_data_dict[0].values())[0]\n",
-    "        y_size, x_size = first_band_data.shape\n",
-    "        \n",
-    "        data = xr.Dataset(\n",
-    "            data_vars,\n",
-    "            coords={\n",
-    "                'time': np.arange(len(all_data_dict)),\n",
-    "                'x': np.arange(x_size),\n",
-    "                'y': np.arange(y_size)\n",
-    "            }\n",
-    "        )\n",
-    "        \n",
-    "        # ===== CALCULATE TEMPORAL FEATURES FOR ACCURACY =====\n",
-    "        print(f\"\\n[TEMPORAL FEATURES] Computing temporal features from time-series...\")\n",
-    "        \n",
-    "        temporal_features_added = 0\n",
-    "        \n",
-    "        # For NDVI: temporal statistics\n",
-    "        if 'ndvi' in data.data_vars:\n",
-    "            ndvi_ts = data['ndvi']\n",
-    "            \n",
-    "            # Min NDVI (vegetation stress indicator)\n",
-    "            data['ndvi_min'] = ndvi_ts.min(dim='time')\n",
-    "            temporal_features_added += 1\n",
-    "            \n",
-    "            # Max NDVI (peak vegetation)\n",
-    "            data['ndvi_max'] = ndvi_ts.max(dim='time')\n",
-    "            temporal_features_added += 1\n",
-    "            \n",
-    "            # Mean NDVI\n",
-    "            data['ndvi_mean'] = ndvi_ts.mean(dim='time')\n",
-    "            temporal_features_added += 1\n",
-    "            \n",
-    "            # NDVI range (variability)\n",
-    "            data['ndvi_range'] = data['ndvi_max'] - data['ndvi_min']\n",
-    "            temporal_features_added += 1\n",
-    "            \n",
-    "            # NDVI std (temporal consistency)\n",
-    "            data['ndvi_std'] = ndvi_ts.std(dim='time')\n",
-    "            temporal_features_added += 1\n",
-    "        \n",
-    "        # For all indices: mean values (aggregate features)\n",
-    "        for band_name in ['ndbi', 'ndwi', 'evi']:\n",
-    "            if band_name in data.data_vars:\n",
-    "                band_ts = data[band_name]\n",
-    "                data[f'{band_name}_mean'] = band_ts.mean(dim='time')\n",
-    "                temporal_features_added += 1\n",
-    "        \n",
-    "        print(f\"✅ Added {temporal_features_added} temporal/aggregate features\")\n",
-    "        \n",
-    "        # ===== SAVE TO CACHE =====\n",
-    "        print(f\"\\n[CACHE] Saving dataset to cache...\")\n",
-    "        try:\n",
-    "            data.to_netcdf(cache_file, engine='netcdf4')\n",
-    "            cache_size = os.path.getsize(cache_file) / (1024**3)\n",
-    "            print(f\"✅ Dataset saved to cache: {cache_file}\")\n",
-    "            print(f\"   Cache size: {cache_size:.2f} GB\")\n",
-    "        except Exception as e:\n",
-    "            print(f\"⚠️  Error saving cache: {e}\")\n",
-    "        \n",
-    "        print(f\"\\n✅ OPTIMAL Dataset with native resolution + temporal features created!\")\n",
-    "        print(f\"   {'='*70}\")\n",
-    "        print(f\"   🎬 Total scenes (time steps): {len(all_data_dict)}\")\n",
-    "        print(f\"   📊 Total bands/variables: {len(data.data_vars)}\")\n",
-    "        print(f\"   🖼️  Spatial size: {x_size} × {y_size} pixels (NATIVE resolution)\")\n",
-    "        print(f\"   📏 Native resolution: 10m (Sentinel-2 L2A)\")\n",
-    "        print(f\"   ⏰ Temporal range: {scene_dates[0].date()} to {scene_dates[-1].date()}\")\n",
-    "        print(f\"   💾 Total dataset size: {notebook_utils.xarray_object_size(data)}\")\n",
-    "        print(f\"   💿 Cached at: {cache_file}\")\n",
-    "        print(f\"   {'='*70}\")\n",
-    "        \n",
-    "        print(f\"\\n   Dataset dimensions:\")\n",
-    "        for dim, size in data.dims.items():\n",
-    "            print(f\"     {dim}: {size}\")\n",
-    "        \n",
-    "        print(f\"\\n   Variables ({len(data.data_vars)}):\")\n",
-    "        spatial_vars = []\n",
-    "        temporal_vars = []\n",
-    "        for var_name in sorted(data.data_vars):\n",
-    "            if len(data[var_name].shape) == 3:\n",
-    "                spatial_vars.append(f\"{var_name} {data[var_name].shape}\")\n",
-    "            else:\n",
-    "                temporal_vars.append(f\"{var_name} {data[var_name].shape}\")\n",
-    "        \n",
-    "        print(f\"   Spatial time-series ({len(spatial_vars)}):\")\n",
-    "        for v in spatial_vars:\n",
-    "            print(f\"     - {v}\")\n",
-    "        print(f\"   Temporal aggregates ({len(temporal_vars)}):\")\n",
-    "        for v in temporal_vars:\n",
-    "            print(f\"     - {v}\")\n",
-    "        \n",
-    "        print(f\"   {'='*70}\")\n",
-    "        \n",
-    "        display(data)\n",
-    "    \n",
-    "    # ===== EXTRACT NDVI FOR TRAINING =====\n",
-    "    print(f\"\\n[NDVI EXTRACTION] Extracting NDVI for model training...\")\n",
-    "    if 'ndvi_mean' in data.data_vars:\n",
-    "        # Use mean NDVI across time\n",
-    "        ndvi = data['ndvi_mean']\n",
-    "        print(f\"✅ NDVI extracted (mean across time)\")\n",
-    "        print(f\"   Shape: {ndvi.shape}\")\n",
-    "    elif 'ndvi' in data.data_vars:\n",
-    "        # Use first time step if mean not available\n",
-    "        ndvi = data['ndvi'].isel(time=0)\n",
-    "        print(f\"✅ NDVI extracted (first time step)\")\n",
-    "        print(f\"   Shape: {ndvi.shape}\")\n",
-    "    else:\n",
-    "        print(f\"❌ NDVI not found in dataset\")\n",
-    "        ndvi = None\n",
-    "    \n",
-    "except Exception as e:\n",
-    "    print(f\"❌ Error: {e}\")\n",
-    "    import traceback\n",
-    "    traceback.print_exc()\n",
-    "    data = None\n",
-    "    ndvi = None\n",
-    "\n",
-    "print(\"=\"*70)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "id": "d2585562-88aa-4c7d-bf70-1f6affcf65d4",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "======================================================================\n",
-      "TRAINING DATA SETUP\n",
-      "======================================================================\n",
-      "\n",
-      "[1] Loading training data: train/ST_training data_updated_1130points_new.shp\n",
-      "    ❌ Error: name 'load_train_data' is not defined\n",
-      "\n",
-      "[2] Label mapping:\n",
-      "    0: Lua tom\n",
-      "    1: Lua\n",
-      "    2: CHN\n",
-      "    3: CLN\n",
-      "    4: TS\n",
-      "    5: Song\n",
-      "    6: Dat xay dung\n",
-      "    7: Rung\n",
-      "\n",
-      "======================================================================\n"
-     ]
-    }
-   ],
-   "source": [
-    "# 🎯 LOAD TRAINING DATA & EXTRACT FEATURES\n",
-    "print(\"=\"*70)\n",
-    "print(\"TRAINING DATA SETUP\")\n",
-    "print(\"=\"*70)\n",
-    "\n",
-    "# Load training points\n",
-    "train_path = \"train/ST_training data_updated_1130points_new.shp\"\n",
-    "print(f\"\\n[1] Loading training data: {train_path}\")\n",
-    "\n",
-    "try:\n",
-    "    train = load_train_data(train_path)\n",
-    "    print(f\"    ✅ Loaded {len(train)} training points\")\n",
-    "    print(f\"    Columns: {list(train.columns)}\")\n",
-    "    train.head()\n",
-    "except Exception as e:\n",
-    "    print(f\"    ❌ Error: {e}\")\n",
-    "    train = None\n",
-    "\n",
-    "# Label mapping\n",
-    "label_mapping = {\n",
-    "    \"Lua tom\": \"0\",\n",
-    "    \"Lua\": \"1\",\n",
-    "    \"CHN\": \"2\",\n",
-    "    \"CLN\": \"3\",\n",
-    "    \"TS\": \"4\",\n",
-    "    \"Song\": \"5\",\n",
-    "    \"Dat xay dung\": \"6\",\n",
-    "    \"Rung\": \"7\",\n",
-    "}\n",
-    "\n",
-    "print(f\"\\n[2] Label mapping:\")\n",
-    "for label, code in label_mapping.items():\n",
-    "    print(f\"    {code}: {label}\")\n",
-    "\n",
-    "print(\"\\n\" + \"=\"*70)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 7,
-   "id": "2e955884-d4af-422d-a8e6-d436199540e0",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "======================================================================\n",
-      "MODEL TRAINING\n",
-      "======================================================================\n",
-      "❌ Missing training data or NDVI\n",
-      "======================================================================\n",
-      "CPU times: user 700 μs, sys: 0 ns, total: 700 μs\n",
-      "Wall time: 638 μs\n"
-     ]
-    }
-   ],
-   "source": [
-    "%%time\n",
-    "# 🤖 RANDOM FOREST MODEL TRAINING\n",
-    "print(\"=\"*70)\n",
-    "print(\"MODEL TRAINING\")\n",
-    "print(\"=\"*70)\n",
-    "\n",
-    "if train is not None and ndvi is not None:\n",
-    "    print(\"\\n[1] Extracting features from NDVI...\")\n",
-    "    try:\n",
-    "        # Extract NDVI values at training point locations\n",
-    "        X = []\n",
-    "        y = []\n",
-    "        \n",
-    "        for idx, point in train.iterrows():\n",
-    "            try:\n",
-    "                # Get NDVI value at point location (nearest neighbor)\n",
-    "                ndvi_val = float(ndvi.sel(x=point.geometry.x, y=point.geometry.y, method='nearest').values)\n",
-    "                label = label_mapping[point.Hientrang]\n",
-    "                \n",
-    "                X.append([ndvi_val])\n",
-    "                y.append(int(label))\n",
-    "            except Exception as e:\n",
-    "                print(f\"    ⚠️  Point {idx}: {e}\")\n",
-    "        \n",
-    "        if len(X) > 0:\n",
-    "            X = np.array(X)\n",
-    "            y = np.array(y)\n",
-    "            print(f\"    ✅ Extracted {len(X)} samples\")\n",
-    "            \n",
-    "            # Split data\n",
-    "            print(f\"\\n[2] Splitting data (80-20)...\")\n",
-    "            from sklearn.model_selection import train_test_split\n",
-    "            X_train, X_test, y_train, y_test = train_test_split(\n",
-    "                X, y, test_size=0.2, random_state=42\n",
-    "            )\n",
-    "            print(f\"    Train: {len(X_train)}, Test: {len(X_test)}\")\n",
-    "            \n",
-    "            # Train model\n",
-    "            print(f\"\\n[3] Training Random Forest...\")\n",
-    "            from sklearn.ensemble import RandomForestClassifier\n",
-    "            from sklearn.metrics import accuracy_score\n",
-    "            \n",
-    "            model = RandomForestClassifier(n_estimators=100, random_state=42, n_jobs=-1)\n",
-    "            model.fit(X_train, y_train)\n",
-    "            \n",
-    "            # Evaluate\n",
-    "            y_pred = model.predict(X_test)\n",
-    "            accuracy = accuracy_score(y_test, y_pred)\n",
-    "            print(f\"    ✅ Model trained!\")\n",
-    "            print(f\"    Accuracy: {accuracy*100:.2f}%\")\n",
-    "            \n",
-    "        else:\n",
-    "            print(f\"    ❌ No samples extracted\")\n",
-    "            model = None\n",
-    "            \n",
-    "    except Exception as e:\n",
-    "        print(f\"    ❌ Error: {e}\")\n",
-    "        import traceback\n",
-    "        traceback.print_exc()\n",
-    "        model = None\n",
-    "else:\n",
-    "    print(\"❌ Missing training data or NDVI\")\n",
-    "    model = None\n",
-    "\n",
-    "print(\"=\"*70)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 8,
-   "id": "f1a14379-ed6e-4897-9ca4-2669743fab40",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "======================================================================\n",
-      "MODEL SAVING\n",
-      "======================================================================\n",
-      "❌ No model to save\n",
-      "======================================================================\n"
-     ]
-    }
-   ],
-   "source": [
-    "# 💾 SAVE MODEL\n",
-    "print(\"=\"*70)\n",
-    "print(\"MODEL SAVING\")\n",
-    "print(\"=\"*70)\n",
-    "\n",
-    "if model is not None:\n",
-    "    print(\"\\n🔄 Saving trained model...\")\n",
-    "    try:\n",
-    "        save_model(\"model_rasterio.joblib\", model)\n",
-    "        print(\"✅ Model saved to model_train/model_rasterio.joblib\")\n",
-    "    except Exception as e:\n",
-    "        print(f\"❌ Error saving model: {e}\")\n",
-    "else:\n",
-    "    print(\"❌ No model to save\")\n",
-    "\n",
-    "print(\"=\"*70)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "33dd516d-9824-499e-96b9-5cd9224c194c",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "======================================================================\n",
-      "CLEANUP\n",
-      "======================================================================\n",
-      "\n",
-      "🔄 Closing Dask client and cluster...\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "✅ Cleanup complete\n",
-      "\n",
-      "======================================================================\n",
-      "✅ PIPELINE COMPLETE\n",
-      "======================================================================\n"
-     ]
-    },
-    {
-     "ename": "",
-     "evalue": "",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[1;31mThe Kernel crashed while executing code in the current cell or a previous cell. \n",
-      "\u001b[1;31mPlease review the code in the cell(s) to identify a possible cause of the failure. \n",
-      "\u001b[1;31mClick <a href='https://aka.ms/vscodeJupyterKernelCrash'>here</a> for more info. \n",
-      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
-     ]
-    }
-   ],
-   "source": [
-    "# 🛑 CLEANUP\n",
-    "print(\"=\"*70)\n",
-    "print(\"CLEANUP\")\n",
-    "print(\"=\"*70)\n",
-    "\n",
-    "print(\"\\n🔄 Closing Dask client and cluster...\")\n",
-    "try:\n",
-    "    client.close()\n",
-    "    cluster.close()\n",
-    "    print(\"✅ Cleanup complete\")\n",
-    "except Exception as e:\n",
-    "    print(f\"⚠️  Error during cleanup: {e}\")\n",
-    "\n",
-    "print(\"\\n\" + \"=\"*70)\n",
-    "print(\"✅ PIPELINE COMPLETE\")\n",
-    "print(\"=\"*70)"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "env_01",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.18"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}