refactor: reorganize project structure by moving core modules and update import paths in API server

update attribute table

.temp/01.train_ODC_XGBoost.py

@@ -0,0 +1,171 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+get_ipython().run_cell_magic('time', '', '%matplotlib inline\n\nimport importlib\nimport new_import_ODC  \n\nimportlib.reload(new_import_ODC)\n\nfrom new_import_ODC import *\n')
+
+
+# In[2]:
+
+
+get_ipython().run_cell_magic('time', '', '# Cấu hình Daskgateway\ncluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1, 10))\n# Khai báo 1 Datacube là dc\ndc = None\n\n# Cấu hình truy cập dịch vụ S3\nconfigure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n\nclient\n')
+
+
+# In[3]:
+
+
+## cấu hình thời gian lấy ảnh và tọa độ
+date_range = ("2022-09-01", "2022-10-01")
+longtitude_range = (105.86, 105.94)
+latitude_range = (9.65, 9.69)
+
+coordinates = (longtitude_range, latitude_range)
+
+
+# In[4]:
+
+
+## truy vấn ảnh vệ tinh sen2
+data = load_data(None, date_range, longtitude_range, latitude_range)
+notebook_utils.heading(notebook_utils.xarray_object_size(data))
+display(data)
+
+
+# In[5]:
+
+
+get_ipython().run_cell_magic('time', '', '# Tiến hành loại bỏ các vị trí bị mây ảnh hưởng\nresult = mask_clean(data)\n# progress(result)\n')
+
+
+# In[6]:
+
+
+# Tiến hành tính toán NDVI
+ds1 = calculate_indices(result, index="NDVI", satellite_mission="s2")
+ndvi = ds1["NDVI"]
+display(ndvi)
+
+
+# In[7]:
+
+
+## Hiển thị ảnh NDVI chưa điền các giá trị mây (chưa fill nan)
+plt.imshow(ndvi.isel(time=0))
+
+
+# In[8]:
+
+
+# Thiết lập giá trị trung bình mùa vụ để xử lý các điểm ảnh bị mây dựa vào sự thay đổi theo mùa
+time_split = [
+    slice("2022-09-01", "2023-01-01"),
+    slice("2023-01-01", "2023-05-01"),
+    slice("2023-05-01", "2023-07-01"),
+    slice("2023-07-01", "2022-10-01"),
+]
+
+# Điền mây ở các vị trí mang giá trị nan (fill nan)
+fill_nan_ndvi = fill_nan(ndvi, time_split)
+
+# In kết quả ảnh NDVI đã điền mây (đã fill nan)
+plt.imshow(fill_nan_ndvi.isel(time=0))
+
+
+# In[9]:
+
+
+get_ipython().run_cell_magic('time', '', '## tính ndvi theo tháng\naverage_ndvi = fill_nan_ndvi.resample(time="1M").mean().persist()\n# progress(average_ndvi)\n\n# compute average_ndvi\naverage_ndvi = average_ndvi.compute()\n')
+
+
+# In[10]:
+
+
+#Load dữ liệu ảnh Sentinel 1
+dsvh, dsvv = load_data_sen1(None, date_range, coordinates)
+average_vv = calculate_average(dsvv, time_pattern='1M')
+average_vh = calculate_average(dsvh, time_pattern='1M')
+
+
+# In[11]:
+
+
+## cấu hình bộ dữ liệu điểm huấn luyện mô hình (train file)
+train_path = "train/ST_training_data_updated_1130points_new.shp"  # đường dẫn shp file train
+
+## load dữ liệu điểm huấn luyện mô hình (train file)
+train = load_train_data(train_path)
+train.head()
+
+# cấu hình nhãn dữ liệu 
+label_mapping = {
+    "Lua tom": "0",
+    "Lua": "1",
+    "CHN": "2",
+    "CLN": "3",
+    "TS": "4",
+    "Song": "5",
+    "Dat xay dung": "6",
+    "Rung": "7",
+}
+
+# xây dựng tập dữ liệu (dataset) chứa dữ liệu VH, VV, NDVI
+datasets = get_data_sen1_and_sen2(train, average_ndvi, average_vh, average_vv)
+
+# chia tập dữ liệu thành các phần theo tỉ lệ 80(80-20)-20 tương ứng với tập train, validate, test
+X_train, X_val, X_test, y_train, y_val, y_test = split_train_data(
+    train, label_mapping, datasets
+)
+
+
+# In[ ]:
+
+
+get_ipython().run_cell_magic('time', '', '# Import XGBoost\nimport xgboost as xgb\nfrom sklearn.metrics import accuracy_score\nimport numpy as np\n\n# Convert to numpy arrays\nX_train_np = np.asarray(X_train, dtype=np.float32)\nX_val_np = np.asarray(X_val, dtype=np.float32)\ny_train_np = np.asarray(y_train, dtype=np.int32)\ny_val_np = np.asarray(y_val, dtype=np.int32)\n\nprint("🚀 Training XGBoost model...")\nprint(f"   Train samples: {len(X_train_np)}")\nprint(f"   Val samples: {len(X_val_np)}")\nprint(f"   Features: {X_train_np.shape[1]}")\nprint(f"   Classes: 8\\n")\n\n# XGBoost parameters\nparams = {\n    \'objective\': \'multi:softmax\',  # Multi-class classification\n    \'num_class\': 8,                 # 8 land use classes\n    \'max_depth\': 6,                 # Maximum tree depth\n    \'learning_rate\': 0.1,           # Learning rate\n    \'n_estimators\': 200,            # Number of trees\n    \'subsample\': 0.8,               # Subsample ratio\n    \'colsample_bytree\': 0.8,        # Feature sampling ratio\n    \'random_state\': 42,\n    \'n_jobs\': -1,                   # Use all CPU cores\n    \'eval_metric\': \'mlogloss\'       # Multi-class log loss\n}\n\n# Train XGBoost model\nmodel = xgb.XGBClassifier(**params)\n\nmodel.fit(\n    X_train_np, y_train_np,\n    eval_set=[(X_train_np, y_train_np), (X_val_np, y_val_np)],\n    verbose=True\n)\n\n# Validation accuracy\ny_val_pred = model.predict(X_val_np)\nval_accuracy = accuracy_score(y_val_np, y_val_pred)\nprint(f"\\n✅ Training completed!")\nprint(f"   Validation Accuracy: {val_accuracy:.4f} ({val_accuracy*100:.2f}%)")\n')
+
+
+# In[ ]:
+
+
+get_ipython().run_cell_magic('time', '', '# Evaluate on test set\nX_test_np = np.asarray(X_test, dtype=np.float32)\ny_test_np = np.asarray(y_test, dtype=np.int32)\n\nprint("📊 Evaluating XGBoost model on test set...\\n")\n\n# Predictions\ny_pred_test = model.predict(X_test_np)\n\n# Metrics\nfrom sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, confusion_matrix\n\ntest_accuracy = accuracy_score(y_test_np, y_pred_test)\nprecision = precision_score(y_test_np, y_pred_test, average=\'weighted\', zero_division=0)\nrecall = recall_score(y_test_np, y_pred_test, average=\'weighted\', zero_division=0)\nf1 = f1_score(y_test_np, y_pred_test, average=\'weighted\', zero_division=0)\n\nprint(f"📈 Test Results:")\nprint(f"   Accuracy:  {test_accuracy:.4f} ({test_accuracy*100:.2f}%)")\nprint(f"   Precision: {precision:.4f}")\nprint(f"   Recall:    {recall:.4f}")\nprint(f"   F1-Score:  {f1:.4f}\\n")\n\n# Confusion Matrix\nfrom sklearn.metrics import ConfusionMatrixDisplay\nimport matplotlib.pyplot as plt\n\n# Create figure first\nfig, ax = plt.subplots(figsize=(10, 8))\n\nclass_names = list(label_mapping.keys())\ncm = confusion_matrix(y_test_np, y_pred_test)\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=class_names)\ndisp.plot(cmap=\'Blues\', ax=ax)\nplt.xticks(rotation=45, ha=\'right\')\nplt.title(\'XGBoost Confusion Matrix\')\nplt.tight_layout()\nplt.show()\n')
+
+
+# In[ ]:
+
+
+# Lưu mô hình huấn luyện
+import json
+import joblib
+
+# Save XGBoost model
+model_path = "model_xgboost.joblib"
+joblib.dump(model, model_path)
+print(f"✅ Model saved to {model_path}")
+
+# Save model info
+info = {
+    "model_type": "XGBoost",
+    "num_classes": 8,
+    "classes": list(label_mapping.keys()),
+    "num_features": X_train_np.shape[1],
+    "params": params,
+    "accuracy": float(test_accuracy),
+    "precision": float(precision),
+    "recall": float(recall),
+    "f1_score": float(f1),
+}
+
+with open("model_xgboost_info.json", "w") as f:
+    json.dump(info, f, indent=2)
+
+print(f"✅ Model info saved to model_xgboost_info.json")
+
+
+# In[15]:
+
+
+# đóng client, cluster
+# client.close()
+# cluster.close()
+

.temp/01.train_ODC_local_with_Mic_supplyer.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[6]:
+
+
+get_ipython().run_cell_magic('time', '', '%matplotlib inline\n\n# Import Microsoft Planetary Computer libraries\nimport planetary_computer\nfrom pystac_client import Client\nfrom odc.stac import load as stac_load\n\n# Standard imports\nimport xarray as xr\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score, classification_report, confusion_matrix, ConfusionMatrixDisplay\nimport geopandas as gpd\n\n# XGBoost for GPU training\nimport xgboost as xgb\n\nfrom xgboost import XGBClassifier\n\nprint(f"   XGBoost version: {xgb.__version__}")\n\nprint("✅ All modules loaded successfully")\n')
+
+
+# In[7]:
+
+
+get_ipython().run_cell_magic('time', '', '# Kết nối tới Microsoft Planetary Computer STAC\nfrom pystac_client import Client\n\n# KHÔNG dùng modifier ở catalog level để tránh items bị convert thành dict\ncatalog = Client.open(\n    "https://planetarycomputer.microsoft.com/api/stac/v1"\n)\nprint("✅ Connected to Microsoft Planetary Computer")\n\nprint("\\n" + "="*70)\n')
+
+
+# In[8]:
+
+
+get_ipython().run_cell_magic('time', '', '# 🌍 Định nghĩa khu vực và thời gian\nprint("="*70)\nprint("CONFIGURATION")\nprint("="*70)\n\n# Khu vực quan tâm (Vietnam - Mekong Delta) - GIẢM DIỆN TÍCH ~40%\nbbox = [105.6, 9.3, 106.2, 9.8]  # [min_lon, min_lat, max_lon, max_lat]\n\n# GIẢM THỜI GIAN xuống 3 tháng để giảm kích thước dữ liệu cho PC\ntime_range = "2023-03-01/2023-05-31"  # 3 tháng (mùa khô)\n\nprint(f"\\n📍 Area of Interest:")\nprint(f"   Longitude: {bbox[0]} to {bbox[2]}")\nprint(f"   Latitude: {bbox[1]} to {bbox[3]}")\nprint(f"\\n📅 Time Range: {time_range}")\nprint(f"   ⚠️  Optimized for personal computer (3 months, reduced area)")\nprint(f"\\n🗺️  CRS: EPSG:32648")\nprint(f"   Resolution: 20m (reduced from 10m for smaller data size)")\n\nprint("="*70)\n')
+
+
+# In[9]:
+
+
+get_ipython().run_cell_magic('time', '', '# 📡 LOAD SENTINEL-2 FROM MICROSOFT PLANETARY COMPUTER\nprint("="*70)\nprint("LOADING SENTINEL-2 L2A")\nprint("="*70)\n\nprint("\\n🔍 Searching for Sentinel-2 scenes...")\nquery_s2 = catalog.search(\n    collections=["sentinel-2-l2a"],\n    bbox=bbox,\n    datetime=time_range,\n    query={"eo:cloud_cover": {"lt": 30}}  # Cloud cover < 30% (giảm từ 50%)\n)\n\nitems_s2 = list(query_s2.item_collection())\nprint(f"✅ Found {len(items_s2)} Sentinel-2 scenes")\n\n# GIỚI HẠN SỐ LƯỢNG SCENES cho PC cá nhân\nmax_scenes = 12  # Giảm xuống 12 scenes để tối ưu cho PC\nif len(items_s2) > max_scenes:\n    print(f"⚠️  Limiting to {max_scenes} scenes for personal computer")\n    # Chọn scenes đều đặn trong khoảng thời gian\n    step = len(items_s2) // max_scenes\n    items_s2 = items_s2[::step][:max_scenes]\n    print(f"   Selected {len(items_s2)} scenes evenly distributed")\n\nif len(items_s2) > 0:\n    # Show first few scenes\n    print(f"\\n📋 Sample scenes:")\n    for i, item in enumerate(items_s2[:5]):\n        date = item.datetime.strftime("%Y-%m-%d")\n        cloud = item.properties.get("eo:cloud_cover", "N/A")\n        print(f"   [{i+1}] {date} - Cloud: {cloud}%")\n    \n    # Re-sign items to ensure fresh URLs (keep as pystac objects)\n    print(f"\\n🔑 Signing STAC items...")\n    items_s2 = [planetary_computer.sign(item) for item in items_s2]\n    \n    # Load Sentinel-2 data (without Dask chunks)\n    print(f"\\n⏳ Loading Sentinel-2 data...")\n    ds_s2 = stac_load(\n        items_s2,\n        bands=["B04", "B08", "SCL"],  # Red (B04), NIR (B08), Scene Classification (SCL)\n        crs="EPSG:32648",\n        resolution=20,  # 20m resolution (4x smaller data than 10m)\n        bbox=bbox,\n        patch_url=planetary_computer.sign,  # Re-sign URLs during loading\n        fail_on_error=False,  # Skip problematic tiles instead of crashing\n    )\n    \n    # Rename bands to simpler names\n    ds_s2 = ds_s2.rename({"B04": "red", "B08": "nir", "SCL": "scl"})\n    \n    print(f"\\n✅ Sentinel-2 loaded!")\n    print(f"   Shape: {dict(ds_s2.dims)}")\n    print(f"   Variables: {list(ds_s2.data_vars)}")\n    display(ds_s2)\nelse:\n    print(f"❌ No Sentinel-2 scenes found")\n\n    ds_s2 = Noneprint("="*70)\n')
+
+
+# In[10]:
+
+
+get_ipython().run_cell_magic('time', '', '# 📡 LOAD SENTINEL-1 FROM MICROSOFT PLANETARY COMPUTER\nprint("="*70)\nprint("LOADING SENTINEL-1 RTC")\nprint("="*70)\n\nprint("\\n🔍 Searching for Sentinel-1 scenes...")\nquery_s1 = catalog.search(\n    collections=["sentinel-1-rtc"],\n    bbox=bbox,\n    datetime=time_range,\n)\n\nitems_s1 = list(query_s1.item_collection())\nprint(f"✅ Found {len(items_s1)} Sentinel-1 scenes")\n\n# GIỚI HẠN SỐ LƯỢNG SCENES cho PC cá nhân\nmax_scenes = 12  # Giảm xuống 12 scenes để tối ưu cho PC\nif len(items_s1) > max_scenes:\n    print(f"⚠️  Limiting to {max_scenes} scenes for personal computer")\n    # Chọn scenes đều đặn trong khoảng thời gian\n    step = len(items_s1) // max_scenes\n    items_s1 = items_s1[::step][:max_scenes]\n    print(f"   Selected {len(items_s1)} scenes evenly distributed")\n\nif len(items_s1) > 0:\n    # Show first few scenes\n    print(f"\\n📋 Sample scenes:")\n    for i, item in enumerate(items_s1[:5]):\n        date = item.datetime.strftime("%Y-%m-%d")\n        orbit = item.properties.get("sat:orbit_state", "N/A")\n        print(f"   [{i+1}] {date} - Orbit: {orbit}")\n    \n    # Re-sign items to ensure fresh URLs (keep as pystac objects)\n    print(f"\\n🔑 Signing STAC items...")\n    items_s1 = [planetary_computer.sign(item) for item in items_s1]\n    \n    # Load Sentinel-1 data (without Dask chunks)\n    print(f"\\n⏳ Loading Sentinel-1 data...")\n    ds_s1 = stac_load(\n        items_s1,\n        bands=["vv", "vh"],  # VV and VH polarizations\n        crs="EPSG:32648",\n        resolution=20,  # 20m resolution (4x smaller data than 10m)\n        bbox=bbox,\n        patch_url=planetary_computer.sign,  # Re-sign URLs during loading\n        fail_on_error=False,  # Skip problematic tiles instead of crashing\n    )\n    \n    # Convert to dB (Microsoft S1 is in linear power)\n    print(f"\\n🔄 Converting to dB...")\n    ds_s1[\'vv_db\'] = 10 * np.log10(ds_s1[\'vv\'].where(ds_s1[\'vv\'] > 0))\n    ds_s1[\'vh_db\'] = 10 * np.log10(ds_s1[\'vh\'].where(ds_s1[\'vh\'] > 0))\n    \n    print(f"\\n✅ Sentinel-1 loaded!")\n    print(f"   Shape: {dict(ds_s1.dims)}")\n    print(f"   Variables: {list(ds_s1.data_vars)}")\n    display(ds_s1)\nelse:\n    print(f"❌ No Sentinel-1 scenes found")\n\n    ds_s1 = Noneprint("="*70)\n')
+
+
+# In[11]:
+
+
+get_ipython().run_cell_magic('time', '', '# 🌿 CALCULATE NDVI AND PROCESS DATA\nprint("="*70)\nprint("DATA PROCESSING")\nprint("="*70)\n\nif ds_s2 is not None:\n    print("\\n[1] Calculating NDVI...")\n    # NDVI = (NIR - Red) / (NIR + Red)\n    ndvi = (ds_s2[\'nir\'] - ds_s2[\'red\']) / (ds_s2[\'nir\'] + ds_s2[\'red\'] + 1e-8)\n    \n    print(f"✅ NDVI calculated")\n    print(f"   Shape: {ndvi.shape}")\n    print(f"   Time steps: {len(ndvi.time)}")\n    \n    # Cloud masking using SCL band\n    print(f"\\n[2] Applying cloud mask...")\n    # SCL values: 1=defective, 3=cloud shadow, 8=cloud medium, 9=cloud high, 10=cirrus\n    cloud_mask = ds_s2[\'scl\'].isin([1, 3, 8, 9, 10])\n    ndvi_masked = ndvi.where(~cloud_mask)\n    \n    print(f"✅ Cloud mask applied")\n    \n    # Temporal aggregation (mean over time)\n    print(f"\\n[3] Computing mean NDVI across time...")\n    ndvi_mean = ndvi_masked.mean(dim=\'time\')\n    \n    # Data already in memory, no need to compute() again\n    print(f"✅ Mean NDVI computed")\n    print(f"   Shape: {ndvi_mean.shape}")\n    \nelse:\n    print("❌ No Sentinel-2 data to process")\n    ndvi_mean = None\n\nprint("="*70)\n')
+
+
+# In[ ]:
+
+
+get_ipython().run_cell_magic('time', '', '# 🎯 EXTRACT TRAINING DATA FEATURES\nprint("="*70)\nprint("FEATURE EXTRACTION")\nprint("="*70)\n\n# Check if required data is available\nif \'ndvi_mean\' not in globals() or \'ds_s1\' not in globals():\n    print("❌ Error: Please run Cell 6 (DATA PROCESSING) first!")\n    print("   Required variables: ndvi_mean, ds_s1")\n    raise RuntimeError("Missing required data. Run cells in order: Cell 4 → Cell 5 → Cell 6 → Cell 7")\n\n# Load training shapefile\nimport geopandas as gpd\n\ntrain_path = \'train/ST_training data_updated_1130points_new.shp\'\nprint(f"\\n[1] Loading training data from: {train_path}")\ntrain_gdf = gpd.read_file(train_path)\n\n# Ensure CRS matches\nif train_gdf.crs != \'EPSG:32648\':\n    print(f"   Reprojecting from {train_gdf.crs} to EPSG:32648...")\n    train_gdf = train_gdf.to_crs(\'EPSG:32648\')\n\nprint(f"✅ Loaded {len(train_gdf)} training points")\nprint(f"   Available columns: {list(train_gdf.columns)}")\n\n# Auto-detect label column (look for common names)\nlabel_column = None\nfor col in [\'HT_code\', \'Ma_LU\', \'LU2022\', \'class\', \'Class\', \'CLASS\', \'label\', \'Label\', \'LABEL\', \'LU_CODE\', \'LU_code\']:\n    if col in train_gdf.columns:\n        label_column = col\n        break\n\nif label_column is None:\n    print(f"❌ Cannot find label column. Available columns: {list(train_gdf.columns)}")\n    print(f"   Please check your shapefile and update the code.")\nelse:\n    print(f"   Using label column: \'{label_column}\'")\n    print(f"   Classes: {sorted(train_gdf[label_column].unique())}")\n    \n    # Extract features at each training point\n    print(f"\\n[2] Extracting features at training points...")\n    \n    features = []\n    labels = []\n    skipped = 0\n    \n    for idx, row in train_gdf.iterrows():\n        point = row.geometrychro\n        x_coord = point.x\n        y_coord = point.y\n        label = row[label_column]\n        \n        # Extract NDVI at this location\n        if ndvi_mean is not None and ds_s1 is not None:\n            try:\n                ndvi_val = ndvi_mean.sel(x=x_coord, y=y_coord, method=\'nearest\').values\n                \n                # Extract Sentinel-1 VH/VV at this location (mean across time)\n                # Data already in memory, no need to compute()\n                vh_val = ds_s1[\'vh_db\'].sel(x=x_coord, y=y_coord, method=\'nearest\').mean(dim=\'time\').values\n                vv_val = ds_s1[\'vv_db\'].sel(x=x_coord, y=y_coord, method=\'nearest\').mean(dim=\'time\').values\n                \n                # Create feature vector: [NDVI, VH_dB, VV_dB]\n                feature_vec = [ndvi_val, vh_val, vv_val]\n                \n                # Only add if all features are valid (not NaN)\n                if not np.isnan(feature_vec).any():\n                    features.append(feature_vec)\n                    labels.append(label)\n                else:\n                    skipped += 1\n            except Exception as e:\n                # Skip points outside the data extent\n                skipped += 1\n                continue\n    \n    features = np.array(features)\n    labels = np.array(labels)\n    \n    print(f"✅ Extracted features for {len(features)} valid points")\n    print(f"   Skipped {skipped} points (outside extent or NaN values)")\n    print(f"   Feature shape: {features.shape}")\n    print(f"   Feature names: [\'NDVI_mean\', \'VH_dB_mean\', \'VV_dB_mean\']")\n    print(f"\\n   Class distribution:")\n    unique, counts = np.unique(labels, return_counts=True)\n    for cls, cnt in zip(unique, counts):\n        print(f"      Class {cls}: {cnt} samples ({cnt/len(labels)*100:.1f}%)")\n\nprint("="*70)\n')
+
+
+# In[21]:
+
+
+get_ipython().run_cell_magic('time', '', '# 🤖 TRAIN XGBOOST MODEL ON GPU (RTX 4060)\nprint("="*70)\nprint("MODEL TRAINING - GPU ACCELERATED")\nprint("="*70)\n\nfrom xgboost import XGBClassifier\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import classification_report, confusion_matrix, ConfusionMatrixDisplay\nimport matplotlib.pyplot as plt\n\n# Encode labels to ensure they are 0, 1, 2, ... n-1\nprint("\\n[1] Encoding labels...")\nlabel_encoder = LabelEncoder()\nlabels_encoded = label_encoder.fit_transform(labels)\nprint(f"✅ Original classes: {label_encoder.classes_}")\nprint(f"   Encoded as: {np.unique(labels_encoded)}")\n\n# Split data\nprint("\\n[2] Splitting data (80% train, 20% test)...")\nX_train, X_test, y_train, y_test = train_test_split(\n    features, labels_encoded, test_size=0.2, random_state=42, stratify=labels_encoded\n)\nprint(f"✅ Training samples: {len(X_train)}")\nprint(f"   Testing samples: {len(X_test)}")\n\n# Train XGBoost on GPU\nprint("\\n[3] Training XGBoost classifier on RTX 4060 GPU...")\nprint("   GPU Settings: device=\'cuda:0\'")\n\nxgb_model = XGBClassifier(\n    n_estimators=100,\n    max_depth=20,\n    learning_rate=0.1,\n    device=\'cuda:0\',           # Use GPU (updated from deprecated gpu_id)\n    tree_method=\'hist\',        # Use hist with device for GPU training\n    random_state=42,\n    eval_metric=\'mlogloss\',    # Multi-class log loss\n    verbosity=1                # Show GPU training progress\n)\n\nxgb_model.fit(X_train, y_train)\nprint(f"✅ Model trained on GPU")\n\n# Evaluate\nprint("\\n[4] Evaluating model...")\ntrain_score = xgb_model.score(X_train, y_train)\ntest_score = xgb_model.score(X_test, y_test)\nprint(f"✅ Training accuracy: {train_score:.4f}")\nprint(f"   Testing accuracy: {test_score:.4f}")\n\n# Classification report\nprint("\\n[5] Classification Report:")\ny_pred = xgb_model.predict(X_test)\nprint(classification_report(y_test, y_pred, target_names=[str(c) for c in label_encoder.classes_]))\n\n# Confusion matrix\nprint("\\n[6] Confusion Matrix:")\nfig, ax = plt.subplots(figsize=(10, 8))\ncm = confusion_matrix(y_test, y_pred)\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=label_encoder.classes_)\ndisp.plot(ax=ax, cmap=\'Blues\', values_format=\'d\')\nplt.title(\'Confusion Matrix - XGBoost GPU Model (RTX 4060)\')\nplt.tight_layout()\nplt.show()\n\nprint("="*70)\n')
+
+
+# In[23]:
+
+
+get_ipython().run_cell_magic('time', '', '# 💾 SAVE MODEL AND CLEANUP\nprint("="*70)\nprint("SAVING MODEL & CLEANUP")\nprint("="*70)\n\nimport joblib\nfrom datetime import datetime\n\n# Save model and label encoder\nmodel_filename = f"model_train/model_xgboost_gpu_{datetime.now().strftime(\'%Y%m%d_%H%M%S\')}.joblib"\nprint(f"\\n[1] Saving model to: {model_filename}")\njoblib.dump({\'model\': xgb_model, \'label_encoder\': label_encoder}, model_filename)\nprint(f"✅ Model and label encoder saved")\n\n# Save model info\ninfo = {\n    "timestamp": datetime.now().isoformat(),\n    "data_source": "Microsoft Planetary Computer STAC",\n    "collections": ["sentinel-2-l2a", "sentinel-1-rtc"],\n    "features": ["NDVI_mean", "VH_dB_mean", "VV_dB_mean"],\n    "training_samples": len(X_train),\n    "testing_samples": len(X_test),\n    "train_accuracy": float(train_score),\n    "test_accuracy": float(test_score),\n    "model_type": "XGBClassifier",\n    "device": "cuda:0",\n    "gpu_device": "RTX 4060",\n    "tree_method": "hist",\n    "n_estimators": 100,\n    "max_depth": 20,\n    "learning_rate": 0.1\n}\n\nimport json\ninfo_filename = model_filename.replace(\'.joblib\', \'_info.json\')\nwith open(info_filename, \'w\') as f:\n    json.dump(info, f, indent=2)\nprint(f"✅ Model info saved to: {info_filename}")\n\n# No cleanup needed (Dask removed)\nprint("\\n[2] Cleanup complete")\n\nprint("="*70)\n\nprint("\\n" + "="*70)\n\nprint("🎉 TRAINING COMPLETE!")\n')
+

.temp/02.predict_ODC.py

@@ -0,0 +1,191 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+get_ipython().run_cell_magic('time', '', '%matplotlib inline\n\nimport importlib\nimport new_import_ODC  \n\nimportlib.reload(new_import_ODC)\n\nfrom new_import_ODC import *\n')
+
+
+# In[2]:
+
+
+get_ipython().run_cell_magic('time', '', '# Dask gateway\ncluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))\ndc = datacube.Datacube()\n\n# Configure s3 access\nconfigure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n\nclient\n')
+
+
+# In[3]:
+
+
+## cấu hình thời gian lấy ảnh và tọa độ
+date_range = ('2022-09-01', '2023-10-01')
+longtitude_range = (105.86575, 105.94120)
+latitude_range = (9.65070, 9.69850)
+
+
+# In[4]:
+
+
+## truy vấn ảnh vệ tinh sen2
+data = load_data(dc, date_range, longtitude_range, latitude_range)
+notebook_utils.heading(notebook_utils.xarray_object_size(data))
+display(data)
+
+
+# In[5]:
+
+
+get_ipython().run_cell_magic('time', '', '# Tiến hành loại bỏ các vị trí bị mây ảnh hưởng\nresult = mask_clean(data)\nprogress(result)\n')
+
+
+# In[6]:
+
+
+# Tiến hành tính toán NDVI
+ds1 = calculate_indices(result, index='NDVI', satellite_mission='s2')
+ndvi = ds1["NDVI"]
+display(ndvi)
+
+
+# In[7]:
+
+
+## ảnh NDVI chưa điền mây (fill nan)
+plt.imshow(ndvi.isel(time=50))
+
+
+# In[8]:
+
+
+# đặt thời gian các mùa
+time_split = [slice('2022-09-01', '2023-01-01'), 
+               slice('2023-01-01', '2023-05-01'),
+               slice('2023-05-01', '2023-07-01'),
+               slice('2023-07-01', '2023-10-01')]
+
+# Điền mây ở các vị trí mang giá trị nan (fill nan)
+fill_nan_ndvi = fill_nan(ndvi, time_split)
+
+# In kết quả ảnh ndvi đã điền mây (đã fill nan)
+plt.imshow(fill_nan_ndvi.isel(time=50))
+
+
+# In[9]:
+
+
+get_ipython().run_cell_magic('time', '', "## tính ndvi theo tháng\naverage_ndvi = fill_nan_ndvi.resample(time='1M').mean().persist()\nprogress(average_ndvi)\n")
+
+
+# In[10]:
+
+
+# compute average_ndvi
+average_ndvi = average_ndvi.compute()
+
+
+# In[11]:
+
+
+# load dữ liệu sen1
+coordinates = (longtitude_range, latitude_range)
+dsvh, dsvv = load_data_sen1(dc, date_range, coordinates)
+average_vv = calculate_average(dsvv, time_pattern='1M')
+average_vh = calculate_average(dsvh, time_pattern='1M')
+
+
+# In[12]:
+
+
+# load model RF
+loaded_model = joblib.load(os.path.join("model_train", "model_odc.joblib"))
+
+# dự đoán
+data_array = predict(loaded_model, data.rio.crs, average_ndvi, average_vh, average_vv)
+
+
+# In[13]:
+
+
+# cấu hình màu cho các loại đất
+colors = [
+    "#abcee9",
+    "#ffef44",
+    "#c4ff9e",
+    "#ffd6a8",
+    "#93ddda",
+    "#1aeef7",
+    "#ffa7f2",
+    "#33ee33"
+]
+labels = [
+    "Lúa tôm",
+    "Lúa",
+    "CHN",
+    "CLN",
+    "TS",
+    "Sông",
+    "Đất xây dựng",
+    "Rừng"
+]
+# hiển thị phân loại sử dụng đất
+cmap = ListedColormap(colors)
+img = data_array.plot(cmap=cmap, add_colorbar=False)
+cbar = plt.colorbar(img)
+cbar.ax.set_yticklabels(labels)
+plt.title("Phân loại sử dụng đất")
+plt.axis('off')
+plt.show()
+
+
+# In[14]:
+
+
+## cấu hình shapefile ranh giới thuận hòa và vh vv file
+thuanhoa_path = "ThuanHoa/region/ST_ThuanHoa_Boundaryofficially.shp"
+
+# cắt theo ranh giới xã thuận hòa
+region_result = cut_according_shp(thuanhoa_path, average_ndvi, data_array)
+
+
+# In[15]:
+
+
+# hiển thị kết quả phân loại sử dụng đất
+colorval = list(range(len(colors)))
+options = {
+    'title': 'Phân loại sử dụng đất',
+    'cmap': colors,
+    'clim': (0, 8),
+    'aspect': 'equal',
+    'colorbar_opts': {
+        'major_label_overrides': dict(zip(colorval, labels)),
+        'major_label_text_align': 'left',
+        'ticker': FixedTicker(ticks=colorval),
+    },
+}
+                                      
+region_result.hvplot(
+    rasterize = True,                  # Use Datashader, particularly useful for dask arrays
+    aggregator = reductions.mode(),    # Datashader selects mode value, requires 'hv.Image'
+).options(opts.Image(**options))
+
+
+# In[16]:
+
+
+# Lưu lại kết quả
+region_result.rio.to_raster("KetQuaPhanLoaiDatODC.tif")
+
+
+# In[17]:
+
+
+# đóng client, cluster
+client.close()
+cluster.close()
+
+
+# In[ ]:
+
+
+
+

.temp/03.compare_ODC.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+# Khai báo các thư viện cần thiết
+from new_import_ODC import *
+
+# Khai báo đường dẫn đến kết quả phân loại và dữ liệu của địa phương
+KD_path = "ThuanHoa/KhoanhDat/ThuanHoa_TKDD2022.shp"
+KetQuaPhanLoaiDat = "KetQuaPhanLoaiDatODC.tif"
+
+
+# In[2]:
+
+
+# khai báo các loại đất từ dữ liệu kiểm kê ứng với các hiện trạng được phân loại từ viễn thám
+CODE_MAP = {
+    "BHK": 2,
+    "CLN": 3,
+    "DGD": 6,
+    "DGT": 6,
+    "DNL": 6,
+    "DRA": 6,
+    "DSH": 6,
+    "DTL": 5,
+    "DTS": 6,
+    "DYT": 6,
+    "LUC": 1,
+    "NKH": 3,
+    "NTD": 6,
+    "NTS": 4,
+    "ONT": 6,
+    "SKC": 6,
+    "SKX": 6,
+    "SON": 5,
+    "TMD": 6,
+    "TON": 6,
+    "TSC": 6,
+}
+
+# Khai báo các nhãn phân loại đất ứng với 3 loại đất chính
+HT_MAP = {
+    "NN": {"name": "Đất Nông Nghiệp", "data": [1, 2, 3, 4]},
+    "PNN": {"name": "Đất Phi Nông Nghiệp", "data": [6]},
+    "TQ": {"name": "Đất Thổ Quả", "data": [15]},
+}
+
+
+# In[3]:
+
+
+# Tiến hành chồng lắp
+result = compare(KD_path, KetQuaPhanLoaiDat, CODE_MAP, HT_MAP)
+
+
+# In[4]:
+
+
+# cấu hình màu cho các loại sử dụng đất
+colors = [
+    "#abcee9",
+    "#ffffc0",
+    "#c4ff9e",
+    "#ffd6a8",
+    "#93ddda",
+    "#1aeef7",
+    "#ffa7f2",
+    "#33ee33",
+]
+labels = ["Lúa tôm", "Lúa", "CHN", "CLN", "TS", "Sông", "Đất xây dựng", "Rừng"]
+
+
+# In[5]:
+
+
+# Lưu kết quả
+save_result(result, HT_MAP)
+
+
+# In[6]:
+
+
+# hiển thị kết quả
+xx = []
+
+for k, v in result.items():
+    rs = merge_arrays(v, nodata=np.nan)
+    xx.append(rs.squeeze(drop=True))
+xx = xr.concat(xx, pd.Index([HT_MAP[x]["name"] for x in HT_MAP], name="name"))
+
+colorval = list(range(len(colors)))
+options = {
+    "cmap": colors,
+    "clim": (0, 8),
+    "aspect": "equal",
+    "height": 400,
+    "colorbar_opts": {
+        "major_label_overrides": dict(zip(colorval, labels)),
+        "major_label_text_align": "left",
+        "ticker": FixedTicker(ticks=colorval),
+    },
+}
+
+xx.hvplot(
+    groupby="name",
+    rasterize=True,  # Use Datashader, particularly useful for dask arrays
+    aggregator=reductions.mode(),  # Datashader selects mode value, requires 'hv.Image'
+).options(opts.Image(**options))
+
+
+# In[ ]:
+
+
+
+

.temp/check_X.py

@@ -0,0 +1,18 @@
+import joblib
+import numpy as np
+
+cache_file = "dataset_cache/training_data_2d.joblib"
+data = joblib.load(cache_file)
+X = np.array(data['X'])
+y = np.array(data['y'])
+
+print("X shape:", X.shape)
+print("X mean:", np.mean(X))
+print("X std:", np.std(X))
+print("X min:", np.min(X))
+print("X max:", np.max(X))
+print("Any NaN:", np.isnan(X).any())
+
+for i in range(6):
+    print(f"Channel {i} mean: {np.mean(X[:, i, :, :]):.4f}, min: {np.min(X[:, i, :, :]):.4f}, max: {np.max(X[:, i, :, :]):.4f}")
+

.temp/check_missing.py

@@ -0,0 +1,10 @@
+import joblib
+import geopandas as gpd
+from shapely.geometry import Point
+
+data = joblib.load('dataset_cache/training_data_2d.joblib')
+X, y = data['X'], data['y']
+print(f"X shape: {X.shape}, y shape: {y.shape}")
+
+gdf = gpd.read_file("train/ST_training_data_updated_1130points_new.shp")
+print("Total points:", len(gdf))

.temp/check_xgboost_data.py

@@ -0,0 +1,7 @@
+import joblib
+import numpy as np
+
+data = joblib.load('dataset_cache/training_data.joblib')
+X, y = data['X'], data['y']
+print(f"X shape: {X.shape}")
+print(f"y shape: {y.shape}")

.temp/check_zeros.py

@@ -0,0 +1,8 @@
+import joblib
+import numpy as np
+cache_file = "dataset_cache/training_data_2d.joblib"
+data = joblib.load(cache_file)
+X = np.array(data['X'])
+b2 = X[:, 0, :, :]
+print("Zeros in B2:", np.sum(b2 == 0) / b2.size)
+print("X shape:", X.shape)

.temp/fix_all.py

@@ -0,0 +1,25 @@
+import glob, json
+
+changed_files = []
+for file_path in glob.glob('*.ipynb'):
+    with open(file_path, 'r', encoding='utf-8') as f:
+        nb = json.load(f)
+        
+    changed = False
+    for cell in nb.get('cells', []):
+        if cell.get('cell_type') == 'code':
+            source = cell.get('source', [])
+            for i, line in enumerate(source):
+                if 'time=50' in line:
+                    source[i] = line.replace('time=50', 'time=0')
+                    changed = True
+                if 'load_data_sen1(dc,' in line:
+                    source[i] = line.replace('load_data_sen1(dc,', 'load_data_sen1(None,')
+                    changed = True
+                    
+    if changed:
+        with open(file_path, 'w', encoding='utf-8') as f:
+            json.dump(nb, f, indent=1)
+        changed_files.append(file_path)
+
+print('Fixed issues in:', changed_files)

.temp/fix_cache.py

@@ -0,0 +1,19 @@
+import joblib
+import geopandas as gpd
+import numpy as np
+
+cache_file = "dataset_cache/training_data_2d.joblib"
+data = joblib.load(cache_file)
+X = data['X']
+print("X shape:", len(X))
+
+gdf = gpd.read_file("train/ST_training_data_updated_1130points_new.shp")
+gdf = gdf.to_crs("EPSG:32648")
+print("gdf length:", len(gdf))
+
+if len(X) == len(gdf):
+    y = [(row['HT_code'] - 1) for idx, row in gdf.iterrows()]
+    joblib.dump({'X': X, 'y': y}, cache_file)
+    print("Fixed y in cache! Saved.")
+else:
+    print("Lengths do not match, cannot fix automatically.")

.temp/fix_new_train.py

@@ -0,0 +1,20 @@
+import json
+
+def fix_import(file_path):
+    with open(file_path, 'r', encoding='utf-8') as f:
+        nb = json.load(f)
+    changed = False
+    for cell in nb.get('cells', []):
+        if cell.get('cell_type') == 'code':
+            source = cell.get('source', [])
+            if isinstance(source, list):
+                for i, line in enumerate(source):
+                    if "from new_import import *" in line:
+                        source[i] = line.replace("from new_import import *", "from new_import_ODC import *")
+                        changed = True
+    if changed:
+        with open(file_path, 'w', encoding='utf-8') as f:
+            json.dump(nb, f, indent=1)
+        print(f"Fixed {file_path}")
+
+fix_import('new_train.ipynb')

.temp/fix_train.py

@@ -0,0 +1,20 @@
+import re
+
+filepath = "01.train_ODC.py"
+with open(filepath, 'r', encoding='utf-8') as f:
+    content = f.read()
+
+# Find the run_cell_magic line
+pattern = re.compile(r"get_ipython\(\)\.run_cell_magic\('time', '', '(# 🤖 LAND USE CLASSIFICATION MODEL TRAINING.*?)(?=\n')\n'", re.DOTALL)
+
+def repl(match):
+    # Get the inner string and escape all actual newlines with \n
+    inner = match.group(1)
+    inner = inner.replace('\n', '\\n')
+    return f"get_ipython().run_cell_magic('time', '', '{inner}')"
+
+content = pattern.sub(repl, content)
+
+with open(filepath, 'w', encoding='utf-8') as f:
+    f.write(content)
+print("Fixed 01.train_ODC.py syntax")

.temp/fix_typo.py

@@ -0,0 +1,22 @@
+import json
+
+def fix_filename(file_path):
+    with open(file_path, 'r', encoding='utf-8') as f:
+        nb = json.load(f)
+    changed = False
+    for cell in nb.get('cells', []):
+        if cell.get('cell_type') == 'code':
+            source = cell.get('source', [])
+            if isinstance(source, list):
+                for i, line in enumerate(source):
+                    if "ST_training data_updated_1130points.shp" in line:
+                        source[i] = line.replace("ST_training data_updated_1130points.shp", "ST_training_data_updated_1130points.shp")
+                        changed = True
+    if changed:
+        with open(file_path, 'w', encoding='utf-8') as f:
+            json.dump(nb, f, indent=1)
+        print(f"Fixed typo in {file_path}")
+
+import glob
+for nb in glob.glob("*.ipynb"):
+    fix_filename(nb)

.temp/generate_colab_notebook.py

@@ -0,0 +1,58 @@
+import nbformat as nbf
+
+nb = nbf.v4.new_notebook()
+
+text_1 = """# Script Tải Dữ liệu Vệ tinh (Cache) qua Google Colab
+Mục đích của Notebook này là mượn sức mạnh đường truyền và RAM của Google Colab để tải 270 ảnh Sentinel-2 & Sentinel-1 từ Microsoft Planetary Computer. Sau khi xử lý nội suy, nó sẽ sinh ra một file cache `.joblib` duy nhất chứa toàn bộ mảng dữ liệu. 
+Bạn chỉ cần tải file `.joblib` đó về máy là xong!"""
+
+code_1 = """!pip install planetary-computer pystac-client odc-stac geopandas rasterio xarray joblib scikit-learn xgboost lightgbm"""
+
+code_2 = """from google.colab import drive
+drive.mount('/content/drive')"""
+
+text_2 = """## Hướng dẫn:
+1. Nén toàn bộ thư mục `remote-sensing` ở máy tính của bạn thành file `remote-sensing.zip`.
+2. Upload file `remote-sensing.zip` đó lên Google Drive (để ngay ngoài cùng).
+3. Chạy ô lệnh bên dưới để giải nén và chuyển vào thư mục dự án."""
+
+code_3 = """import os
+import shutil
+
+# Giải nén dự án từ Google Drive
+!unzip -q /content/drive/MyDrive/remote-sensing.zip -d /content/
+os.chdir('/content/remote-sensing')
+!ls -la"""
+
+text_3 = """## Bắt đầu tải và Cache Dữ Liệu
+Chạy một mô hình CPU đơn giản (Decision Tree) để ép hệ thống gọi hàm `FeatureExtractor`. Hàm này sẽ làm mọi việc nặng nhọc: tìm ảnh, ghép mây, tính trung vị và lưu kết quả vào thư mục `dataset_cache/`."""
+
+code_4 = """# Lệnh này sẽ mất khoảng 5-15 phút để tải toàn bộ ảnh từ Microsoft
+!python train_land_decision_tree_gpu.py"""
+
+text_4 = """## Hoàn tất
+Bạn hãy kiểm tra xem file `.joblib` lớn (khoảng 40-60MB) đã xuất hiện chưa. Nếu rồi, hãy lưu ngược nó lại Google Drive để tải về máy!"""
+
+code_5 = """# Xem file cache đã được tạo thành công chưa
+!ls -lh dataset_cache/
+
+# Copy toàn bộ thư mục cache sang Google Drive để tải về máy dễ dàng
+!cp -r dataset_cache/ /content/drive/MyDrive/dataset_cache_finished/
+print("Hoàn thành! Bạn hãy mở Google Drive của mình, tìm thư mục 'dataset_cache_finished' và tải file .joblib mới nhất về máy tính.")"""
+
+nb['cells'] = [
+    nbf.v4.new_markdown_cell(text_1),
+    nbf.v4.new_code_cell(code_1),
+    nbf.v4.new_code_cell(code_2),
+    nbf.v4.new_markdown_cell(text_2),
+    nbf.v4.new_code_cell(code_3),
+    nbf.v4.new_markdown_cell(text_3),
+    nbf.v4.new_code_cell(code_4),
+    nbf.v4.new_markdown_cell(text_4),
+    nbf.v4.new_code_cell(code_5)
+]
+
+with open('Download_Cache_Colab.ipynb', 'w') as f:
+    nbf.write(nb, f)
+
+print("Created Download_Cache_Colab.ipynb")

.temp/generate_report_only.py

@@ -0,0 +1,97 @@
+import os
+import glob
+import json
+from tabulate import tabulate
+
+print("📊 BẢNG SO SÁNH KẾT QUẢ CÁC MÔ HÌNH\n")
+
+# 1. Phân loại đất
+print("### 1. Nhóm Phân loại Lớp phủ (Land Classification)")
+land_data = []
+
+if os.path.exists("model_xgboost_info.json"):
+    with open("model_xgboost_info.json", 'r') as f:
+        data = json.load(f)
+        params = data.get('params', {})
+        param_str = f"estimators:{params.get('n_estimators')}, depth:{params.get('max_depth')}" if params else "N/A"
+        land_data.append([
+            data.get('model_type', 'XGBoost'),
+            data.get('accuracy', ''),
+            data.get('precision', ''),
+            data.get('recall', ''),
+            data.get('f1_score', ''),
+            param_str
+        ])
+
+for info_file in glob.glob("model_train/*_info.json"):
+    with open(info_file, 'r') as f:
+        data = json.load(f)
+        # Support both 'accuracy' and 'test_accuracy'
+        acc = data.get('accuracy', data.get('test_accuracy', ''))
+        
+        f1 = data.get('f1_score', '')
+        precision = data.get('precision', '')
+        recall = data.get('recall', '')
+        
+        clf_rep = data.get('classification_report')
+        if isinstance(clf_rep, dict) and 'macro avg' in clf_rep:
+            if not f1:
+                f1 = clf_rep['macro avg'].get('f1-score', '')
+            if not precision:
+                precision = clf_rep['macro avg'].get('precision', '')
+            if not recall:
+                recall = clf_rep['macro avg'].get('recall', '')
+            
+        if not acc and not f1:
+            continue
+            
+        params = data.get('params', {})
+        param_str = f"estimators:{params.get('n_estimators')}, depth:{params.get('max_depth')}" if params else "N/A"
+        
+        if data.get('model_type') == 'RandomForest_RealData':
+            param_str = "estimators:100, depth:15"
+            
+        land_data.append([
+            data.get('model_type', ''),
+            acc,
+            precision,
+            recall,
+            f1,
+            param_str
+        ])
+
+if land_data:
+    print(tabulate(land_data, headers=["Model", "Accuracy", "Precision", "Recall", "F1-Score", "Parameters"], tablefmt="github"))
+print("\n")
+
+# 2. Xóa mây
+print("### 2. Nhóm Xóa mây (Cloud Removal)")
+cloud_data = []
+for info_file in glob.glob("cloud_removal_model/*_info.json"):
+    with open(info_file, 'r') as f:
+        data = json.load(f)
+        cloud_data.append([
+            data.get('model_type', ''),
+            data.get('epoch', ''),
+            data.get('train_loss', ''),
+            data.get('val_loss', '')
+        ])
+if cloud_data:
+    print(tabulate(cloud_data, headers=["Model", "Epochs", "Train Loss", "Val Loss"], tablefmt="github"))
+print("\n")
+
+# 3. Dự báo NDVI
+print("### 3. Nhóm Dự báo Thực vật (NDVI Forecasting)")
+ndvi_data = []
+for info_file in glob.glob("ndvi_forecast_model/*_info.json"):
+    with open(info_file, 'r') as f:
+        data = json.load(f)
+        ndvi_data.append([
+            data.get('model_type', ''),
+            data.get('rmse', ''),
+            data.get('mae', ''),
+            data.get('epoch', 'N/A')
+        ])
+if ndvi_data:
+    print(tabulate(ndvi_data, headers=["Model", "RMSE", "MAE", "Epochs"], tablefmt="github"))
+print("\n")

.temp/inspect_cells_train.py

@@ -0,0 +1,7 @@
+import json
+nb = json.load(open('01.train_ODC.ipynb'))
+for idx, cell in enumerate(nb['cells']):
+    if cell['cell_type'] == 'code':
+        print(f"Cell {idx}:")
+        print("".join(cell['source'][:3]))
+        print("-" * 20)

.temp/log_cnn.txt

@@ -0,0 +1,18 @@
+🚀 BẮT ĐẦU HUẤN LUYỆN MÔ HÌNH CNN (GPU & CACHE)
+Initializing FeatureExtractor (mode=extended)...
+📦 Đang load cache: training_data_507bd2ba4ec0d3fe107839cbf73a7a7d.joblib...
+✅ Loaded 632 samples từ cache!
+   ⚡ Đã bỏ qua download위성 data (tiết kiệm thời gian)
+[CACHE HIT] Using cached dataset with 632 samples
+Training CNN model...
+Building CNN model on cuda...
+Training CNN model with PyTorch...
+CNN Epoch 10/15, Loss: 1.2171
+Evaluating model...
+Generating classification report...
+Saving model...
+[MODEL MANAGER] Saving model to: model_train/model_cnn_auto.joblib
+[MODEL MANAGER] Saving metadata to: model_train/model_cnn_auto_info.json
+[MODEL MANAGER] Model saved successfully!
+Training complete!
+✅ Hoàn thành! Accuracy: 0.5748

.temp/log_dt.txt

@@ -0,0 +1,15 @@
+🚀 BẮT ĐẦU HUẤN LUYỆN MÔ HÌNH DECISION TREE (GPU & CACHE)
+Initializing FeatureExtractor (mode=extended)...
+📦 Đang load cache: training_data_507bd2ba4ec0d3fe107839cbf73a7a7d.joblib...
+✅ Loaded 632 samples từ cache!
+   ⚡ Đã bỏ qua download위성 data (tiết kiệm thời gian)
+[CACHE HIT] Using cached dataset with 632 samples
+Training DECISION_TREE model...
+Evaluating model...
+Generating classification report...
+Saving model...
+[MODEL MANAGER] Saving model to: model_train/model_decision_tree_auto.joblib
+[MODEL MANAGER] Saving metadata to: model_train/model_decision_tree_auto_info.json
+[MODEL MANAGER] Model saved successfully!
+Training complete!
+✅ Hoàn thành! Accuracy: 0.5906

.temp/log_mobilenet.txt

@@ -0,0 +1,30 @@
+🚀 BẮT ĐẦU HUẤN LUYỆN MÔ HÌNH MOBILENET-LRASPP (GPU & CACHE)
+Initializing FeatureExtractor (mode=extended)...
+📦 Đang load cache: training_data_507bd2ba4ec0d3fe107839cbf73a7a7d.joblib...
+✅ Loaded 632 samples từ cache!
+   ⚡ Đã bỏ qua download위성 data (tiết kiệm thời gian)
+[CACHE HIT] Using cached dataset with 632 samples
+Training MOBILENET-LRASPP model...
+Building MobileNetV3 + LR-ASPP model on cuda...
+[MOBILENET] Class distribution: [ 48  89   3  86  74  38 117  50]
+[MOBILENET] Class weights: [0.37418982 0.20181024 5.98703525 0.20885013 0.24271772 0.47266082
+ 0.15351377 0.35922223]
+Training MobileNetV3 + LR-ASPP model with PyTorch...
+MobileNet Epoch 5/25, Train Loss: 1.0614, Val Loss: 1.3584, Val Acc: 40.16%, LR: 0.000800
+[MOBILENET] Epoch 5/25 - Train Loss: 1.0614, Val Loss: 1.3584, Val Acc: 40.16%
+MobileNet Epoch 10/25, Train Loss: 0.9054, Val Loss: 0.8582, Val Acc: 59.06%, LR: 0.000800
+[MOBILENET] Epoch 10/25 - Train Loss: 0.9054, Val Loss: 0.8582, Val Acc: 59.06%
+MobileNet Epoch 15/25, Train Loss: 0.7728, Val Loss: 0.8231, Val Acc: 61.42%, LR: 0.000800
+[MOBILENET] Epoch 15/25 - Train Loss: 0.7728, Val Loss: 0.8231, Val Acc: 61.42%
+MobileNet Epoch 20/25, Train Loss: 0.7125, Val Loss: 0.8783, Val Acc: 59.84%, LR: 0.000400
+[MOBILENET] Epoch 20/25 - Train Loss: 0.7125, Val Loss: 0.8783, Val Acc: 59.84%
+[MOBILENET] Early stopping at epoch 24 (best val loss: 0.8029)
+MobileNet early stopped at epoch 24
+Evaluating model...
+Generating classification report...
+Saving model...
+[MODEL MANAGER] Saving model to: model_train/model_mobilenet-lraspp_auto.joblib
+[MODEL MANAGER] Saving metadata to: model_train/model_mobilenet-lraspp_auto_info.json
+[MODEL MANAGER] Model saved successfully!
+Training complete!
+✅ Hoàn thành! Accuracy: 0.5669

.temp/log_rf.txt

@@ -0,0 +1,15 @@
+🚀 BẮT ĐẦU HUẤN LUYỆN MÔ HÌNH RANDOM FOREST (GPU & CACHE)
+Initializing FeatureExtractor (mode=extended)...
+📦 Đang load cache: training_data_507bd2ba4ec0d3fe107839cbf73a7a7d.joblib...
+✅ Loaded 632 samples từ cache!
+   ⚡ Đã bỏ qua download위성 data (tiết kiệm thời gian)
+[CACHE HIT] Using cached dataset with 632 samples
+Training RANDOM_FOREST model...
+Evaluating model...
+Generating classification report...
+Saving model...
+[MODEL MANAGER] Saving model to: model_train/model_random_forest_auto.joblib
+[MODEL MANAGER] Saving metadata to: model_train/model_random_forest_auto_info.json
+[MODEL MANAGER] Model saved successfully!
+Training complete!
+✅ Hoàn thành! Accuracy: 0.6142

.temp/log_svm.txt

@@ -0,0 +1,15 @@
+🚀 BẮT ĐẦU HUẤN LUYỆN MÔ HÌNH SVM (GPU & CACHE)
+Initializing FeatureExtractor (mode=extended)...
+📦 Đang load cache: training_data_507bd2ba4ec0d3fe107839cbf73a7a7d.joblib...
+✅ Loaded 632 samples từ cache!
+   ⚡ Đã bỏ qua download위성 data (tiết kiệm thời gian)
+[CACHE HIT] Using cached dataset with 632 samples
+Training SVM model...
+Evaluating model...
+Generating classification report...
+Saving model...
+[MODEL MANAGER] Saving model to: model_train/model_svm_auto.joblib
+[MODEL MANAGER] Saving metadata to: model_train/model_svm_auto_info.json
+[MODEL MANAGER] Model saved successfully!
+Training complete!
+✅ Hoàn thành! Accuracy: 0.5827

.temp/log_swin.txt

@@ -0,0 +1,30 @@
+🚀 BẮT ĐẦU HUẤN LUYỆN MÔ HÌNH SWIN-UNET (GPU & CACHE)
+Initializing FeatureExtractor (mode=extended)...
+📦 Đang load cache: training_data_507bd2ba4ec0d3fe107839cbf73a7a7d.joblib...
+✅ Loaded 632 samples từ cache!
+   ⚡ Đã bỏ qua download위성 data (tiết kiệm thời gian)
+[CACHE HIT] Using cached dataset with 632 samples
+Training SWIN-UNET model...
+Building Swin-UNet model on cuda...
+[SWIN-UNET] Class distribution: [ 48  89   3  86  74  38 117  50]
+[SWIN-UNET] Class weights: [0.37418982 0.20181024 5.98703525 0.20885013 0.24271772 0.47266082
+ 0.15351377 0.35922223]
+Training Swin-UNet model with PyTorch (with class weights)...
+Swin-UNet Epoch 5/40, Train Loss: 1.4096, Val Loss: 1.2804, Val Acc: 48.03%, LR: 0.000293
+[SWIN-UNET] Epoch 5/40 - Train Loss: 1.4096, Val Loss: 1.2804, Val Acc: 48.03%
+Swin-UNet Epoch 10/40, Train Loss: 1.1129, Val Loss: 1.2746, Val Acc: 57.48%, LR: 0.000271
+[SWIN-UNET] Epoch 10/40 - Train Loss: 1.1129, Val Loss: 1.2746, Val Acc: 57.48%
+Swin-UNet Epoch 15/40, Train Loss: 1.0479, Val Loss: 1.3126, Val Acc: 50.39%, LR: 0.000238
+[SWIN-UNET] Epoch 15/40 - Train Loss: 1.0479, Val Loss: 1.3126, Val Acc: 50.39%
+Swin-UNet Epoch 20/40, Train Loss: 0.9548, Val Loss: 1.1118, Val Acc: 52.76%, LR: 0.000196
+[SWIN-UNET] Epoch 20/40 - Train Loss: 0.9548, Val Loss: 1.1118, Val Acc: 52.76%
+[SWIN-UNET] Early stopping at epoch 22 (best val loss: 1.1096)
+Swin-UNet early stopped at epoch 22
+Evaluating model...
+Generating classification report...
+Saving model...
+[MODEL MANAGER] Saving model to: model_train/model_swin-unet_auto.joblib
+[MODEL MANAGER] Saving metadata to: model_train/model_swin-unet_auto_info.json
+[MODEL MANAGER] Model saved successfully!
+Training complete!
+✅ Hoàn thành! Accuracy: 0.5354

.temp/log_train_2d_final.txt

@@ -0,0 +1,38 @@
+🚀 BẮT ĐẦU PIPELINE 2D PATCH-BASED & CLOUD REMOVAL
+Loading 2D patches from dataset_cache/training_data_2d.joblib...
+Training 2D CNN with Data Augmentation...
+Epoch 1/150 - Loss: 2.2272 - Test Acc: 0.0752 🌟
+Epoch 2/150 - Loss: 2.0843 - Test Acc: 0.0796 🌟
+Epoch 4/150 - Loss: 2.0350 - Test Acc: 0.1372 🌟
+Epoch 8/150 - Loss: 2.0447 - Test Acc: 0.1637 🌟
+Epoch 10/150 - Loss: 2.0397 - Test Acc: 0.1372
+Epoch 12/150 - Loss: 2.0353 - Test Acc: 0.2168 🌟
+Epoch 20/150 - Loss: 2.0139 - Test Acc: 0.1372
+Traceback (most recent call last):
+  File "/home/x79/remote-sensing/train_land_2d_patch.py", line 298, in <module>
+    main()
+  File "/home/x79/remote-sensing/train_land_2d_patch.py", line 294, in main
+    train_2d_model(X, y)
+  File "/home/x79/remote-sensing/train_land_2d_patch.py", line 219, in train_2d_model
+    for batch_X, batch_y in train_loader:
+  File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 725, in __next__
+    data = self._next_data()
+  File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 785, in _next_data
+    data = self._dataset_fetcher.fetch(index)  # may raise StopIteration
+  File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 54, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 54, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/home/x79/remote-sensing/train_land_2d_patch.py", line 192, in __getitem__
+    x = transform(x)
+  File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torchvision/transforms/transforms.py", line 95, in __call__
+    img = t(img)
+  File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torchvision/transforms/transforms.py", line 752, in forward
+    return F.vflip(img)
+  File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torchvision/transforms/functional.py", line 757, in vflip
+    def vflip(img: Tensor) -> Tensor:
+KeyboardInterrupt

.temp/log_train_v4.txt

@@ -0,0 +1,39 @@
+🚀 V4: TÍCH HỢP RADAR SENTINEL-1 (32-CHANNELS FUSION)
+============================================================
+Clean FUSION data: (252, 32, 16, 16), 7 classes, [31, 38, 32, 49, 23, 75, 4]
+
+============================================================
+32-CHANNELS FUSION CNN
+============================================================
+  Ep 1 Fusion-Acc=0.1961 🌟
+  Ep 3 Fusion-Acc=0.2745 🌟
+  Ep 4 Fusion-Acc=0.4706 🌟
+  Ep 5 Fusion-Acc=0.5490 🌟
+  Ep 6 Fusion-Acc=0.7059 🌟
+  Ep 7 Fusion-Acc=0.7451 🌟
+  Ep 8 Fusion-Acc=0.8431 🌟
+  Ep 15 Fusion-Acc=0.8627 🌟
+
+✅ CNN Fusion best: 0.8627
+
+============================================================
+HYBRID FUSION: CNN embed + S1/S2 Rich features + XGBoost
+============================================================
+Extracted 2182 fusion features per sample
+  Final Feature Vector: (252, 2694)
+  ✅ Hybrid Fusion Acc: 0.8627
+    Fold 1: 0.9412
+    Fold 2: 0.9020
+    Fold 3: 0.9200
+    Fold 4: 0.8800
+    Fold 5: 0.9000
+  ✅ CV Mean: 0.9086 ± 0.0206
+
+============================================================
+📊 FINAL RESULTS V4 (WITH RADAR)
+============================================================
+  ✅ Hybrid Fusion CV: 0.9086
+  📈 CNN Fusion (32ch): 0.8627
+  📈 Hybrid Fusion (CNN+XGB): 0.8627
+
+🏆 BEST: 0.9086

.temp/log_train_v5.txt

@@ -0,0 +1,18 @@
+
+============================================================
+HYBRID FUSION ENSEMBLE: CNN embed + S1/S2 Rich features + XGB/LGBM/ETC
+============================================================
+  Final Feature Vector: (443, 2694)
+    Fold 1: 0.8876
+    Fold 2: 0.9438
+    Fold 3: 0.9438
+    Fold 4: 0.9659
+    Fold 5: 0.9432
+  ✅ Ensemble CV Mean: 0.9369 ± 0.0261
+
+============================================================
+📊 FINAL RESULTS V5 (ENSEMBLE + RADAR)
+============================================================
+  ✅ Hybrid Fusion Ensemble CV: 0.9369
+
+🏆 BEST: 0.9369

.temp/log_train_v6.txt

@@ -0,0 +1,21 @@
+🚀 V6: EXHAUSTIVE HYPERPARAMETER TUNING
+============================================================
+Data: (443, 32, 16, 16), 7 classes, dist=[65, 55, 48, 72, 74, 124, 5]
+
+--- Training Multi-Seed CNN Ensemble ---
+  Seed 42: CNN Acc = 0.8989
+  Seed 123: CNN Acc = 0.8652
+  Seed 777: CNN Acc = 0.8876
+  Multi-seed CNN embedding: (443, 1536)
+  Total features: (443, 3940)
+
+============================================================
+🔬 EXHAUSTIVE HYPERPARAMETER SEARCH
+============================================================
+  🏆 XGB-deep: 0.9526 ± 0.0110  (folds: ['0.955', '0.933', '0.966', '0.955', '0.955'])
+  ✅ XGB-shallow: 0.9436 ± 0.0173  (folds: ['0.944', '0.910', '0.955', '0.955', '0.955'])
+  🏆 XGB-balanced: 0.9504 ± 0.0113  (folds: ['0.944', '0.933', '0.955', '0.955', '0.966'])
+  ✅ LGBM-tuned: 0.9458 ± 0.0149  (folds: ['0.955', '0.921', '0.966', '0.943', '0.943'])
+  ✅ LGBM-conservative: 0.9481 ± 0.0152  (folds: ['0.944', '0.921', '0.966', '0.955', '0.955'])
+  🏆 ETC-deep: 0.9572 ± 0.0082  (folds: ['0.944', '0.955', '0.955', '0.966', '0.966'])
+  🏆 RF-tuned: 0.9549 ± 0.0099  (folds: ['0.944', '0.955', '0.944', '0.966', '0.966'])

.temp/log_tune_swin_unet.txt

@@ -0,0 +1,157 @@
+🚀 BẮT ĐẦU TÌM KIẾM SIÊU THAM SỐ CHO SWIN-UNET
+Loading data from dataset_cache/training_data_507bd2ba4ec0d3fe107839cbf73a7a7d.joblib...
+Using device: cuda
+
+[1/48] Training with params: {'embed_dim': 64, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.5984
+🌟 NEW BEST ACCURACY: 0.5984
+
+[2/48] Training with params: {'embed_dim': 64, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.6063
+🌟 NEW BEST ACCURACY: 0.6063
+
+[3/48] Training with params: {'embed_dim': 64, 'lr': 0.001, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.6142
+🌟 NEW BEST ACCURACY: 0.6142
+
+[4/48] Training with params: {'embed_dim': 64, 'lr': 0.001, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.6772
+🌟 NEW BEST ACCURACY: 0.6772
+
+[5/48] Training with params: {'embed_dim': 64, 'lr': 0.0005, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.5827
+
+[6/48] Training with params: {'embed_dim': 64, 'lr': 0.0005, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.5984
+
+[7/48] Training with params: {'embed_dim': 64, 'lr': 0.0005, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.5669
+
+[8/48] Training with params: {'embed_dim': 64, 'lr': 0.0005, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.6142
+
+[9/48] Training with params: {'embed_dim': 64, 'lr': 0.0001, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.6142
+
+[10/48] Training with params: {'embed_dim': 64, 'lr': 0.0001, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.5118
+
+[11/48] Training with params: {'embed_dim': 64, 'lr': 0.0001, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.5591
+
+[12/48] Training with params: {'embed_dim': 64, 'lr': 0.0001, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.6063
+
+[13/48] Training with params: {'embed_dim': 128, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.7008
+🌟 NEW BEST ACCURACY: 0.7008
+
+[14/48] Training with params: {'embed_dim': 128, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.6142
+
+[15/48] Training with params: {'embed_dim': 128, 'lr': 0.001, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.6772
+
+[16/48] Training with params: {'embed_dim': 128, 'lr': 0.001, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.6063
+
+[17/48] Training with params: {'embed_dim': 128, 'lr': 0.0005, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.5906
+
+[18/48] Training with params: {'embed_dim': 128, 'lr': 0.0005, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.6457
+
+[19/48] Training with params: {'embed_dim': 128, 'lr': 0.0005, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.5906
+
+[20/48] Training with params: {'embed_dim': 128, 'lr': 0.0005, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.5906
+
+[21/48] Training with params: {'embed_dim': 128, 'lr': 0.0001, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.6220
+
+[22/48] Training with params: {'embed_dim': 128, 'lr': 0.0001, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.6457
+
+[23/48] Training with params: {'embed_dim': 128, 'lr': 0.0001, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.5984
+
+[24/48] Training with params: {'embed_dim': 128, 'lr': 0.0001, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.6063
+
+[25/48] Training with params: {'embed_dim': 256, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.7087
+🌟 NEW BEST ACCURACY: 0.7087
+
+[26/48] Training with params: {'embed_dim': 256, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.7323
+🌟 NEW BEST ACCURACY: 0.7323
+
+[27/48] Training with params: {'embed_dim': 256, 'lr': 0.001, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.6457
+
+[28/48] Training with params: {'embed_dim': 256, 'lr': 0.001, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.6142
+
+[29/48] Training with params: {'embed_dim': 256, 'lr': 0.0005, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.5984
+
+[30/48] Training with params: {'embed_dim': 256, 'lr': 0.0005, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.5906
+
+[31/48] Training with params: {'embed_dim': 256, 'lr': 0.0005, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.6142
+
+[32/48] Training with params: {'embed_dim': 256, 'lr': 0.0005, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.7008
+
+[33/48] Training with params: {'embed_dim': 256, 'lr': 0.0001, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.6299
+
+[34/48] Training with params: {'embed_dim': 256, 'lr': 0.0001, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.6299
+
+[35/48] Training with params: {'embed_dim': 256, 'lr': 0.0001, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.6378
+
+[36/48] Training with params: {'embed_dim': 256, 'lr': 0.0001, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.6457
+
+[37/48] Training with params: {'embed_dim': 512, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.6142
+
+[38/48] Training with params: {'embed_dim': 512, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.6457
+
+[39/48] Training with params: {'embed_dim': 512, 'lr': 0.001, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.6378
+
+[40/48] Training with params: {'embed_dim': 512, 'lr': 0.001, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.6299
+
+[41/48] Training with params: {'embed_dim': 512, 'lr': 0.0005, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.6378
+
+[42/48] Training with params: {'embed_dim': 512, 'lr': 0.0005, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.6220
+
+[43/48] Training with params: {'embed_dim': 512, 'lr': 0.0005, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.6142
+
+[44/48] Training with params: {'embed_dim': 512, 'lr': 0.0005, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.7008
+
+[45/48] Training with params: {'embed_dim': 512, 'lr': 0.0001, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.6457
+
+[46/48] Training with params: {'embed_dim': 512, 'lr': 0.0001, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.7323
+
+[47/48] Training with params: {'embed_dim': 512, 'lr': 0.0001, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.6693
+
+[48/48] Training with params: {'embed_dim': 512, 'lr': 0.0001, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.6457
+
+✅ Đã lưu mô hình tốt nhất (Acc: 0.7323) vào land_classification_model/model_swin-unet_optimized_95.joblib
+Cấu hình tốt nhất: {'embed_dim': 256, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 500}

.temp/log_ultimate_95.txt

@@ -0,0 +1,117 @@
+🚀 CHIẾN LƯỢC TOÀN DIỆN ĐẠT >95% ACCURACY
+============================================================
+Loaded data: X=(706, 24, 16, 16), y=(706,)
+Labels unique: [-1  0  1  2  3  4  5  6]
+After cleanup: X=(652, 24, 16, 16), y=(652,) (removed 54 bad samples)
+Remapped labels: [0 1 2 3 4 5 6]
+  Class 0: 65 samples
+  Class 1: 52 samples
+  Class 2: 48 samples
+  Class 3: 72 samples
+  Class 4: 108 samples
+  Class 5: 219 samples
+  Class 6: 88 samples
+
+============================================================
+STRATEGY 5: Flat pixel features + XGBoost (sanity check)
+============================================================
+  Flat features: (652, 6144)
+  ✅ Flat XGBoost acc: 0.7328
+
+============================================================
+STRATEGY 1: Lightweight CNN (no upsampling)
+============================================================
+Device: cuda
+  Epoch 1/300 Loss=1.8379 Acc=0.0763 🌟
+  Epoch 2/300 Loss=1.5825 Acc=0.2824 🌟
+  Epoch 3/300 Loss=1.4412 Acc=0.5649 🌟
+  Epoch 4/300 Loss=1.3274 Acc=0.6336 🌟
+  Epoch 5/300 Loss=1.2893 Acc=0.6870 🌟
+  Epoch 8/300 Loss=1.2140 Acc=0.7099 🌟
+  Epoch 11/300 Loss=1.2224 Acc=0.7252 🌟
+  Epoch 14/300 Loss=1.1087 Acc=0.7328 🌟
+  Epoch 16/300 Loss=1.1081 Acc=0.7710 🌟
+  Epoch 18/300 Loss=1.1847 Acc=0.7939 🌟
+  Epoch 20/300 Loss=1.0316 Acc=0.7786 (patience=2)
+  Epoch 24/300 Loss=1.0465 Acc=0.8092 🌟
+  Epoch 26/300 Loss=0.9583 Acc=0.8397 🌟
+  Epoch 40/300 Loss=0.9361 Acc=0.8626 🌟
+  Epoch 60/300 Loss=0.9807 Acc=0.8092 (patience=20)
+  Epoch 80/300 Loss=0.9459 Acc=0.8015 (patience=40)
+  Epoch 100/300 Loss=0.8443 Acc=0.7939 (patience=60)
+  Early stop at epoch 100
+  ✅ LightCNN best acc: 0.8626
+
+============================================================
+STRATEGY 2: Hybrid CNN embeddings + XGBoost
+============================================================
+  CNN embeddings: (652, 256)
+Extracted 316 rich features per sample
+  Combined features: (652, 572)
+  ✅ Hybrid XGBoost acc: 0.8244
+
+============================================================
+STRATEGY 3: Rich Features + Stacking Ensemble
+============================================================
+Extracted 316 rich features per sample
+  XGBoost: 0.7939
+  LightGBM: 0.7786
+  ExtraTrees: 0.7863
+  RandomForest: 0.7710
+  GBM: 0.7710
+[06:55:44] WARNING: /__w/xgboost/xgboost/src/learner.cc:782: 
+Parameters: { "use_label_encoder" } are not used.
+[06:55:44] WARNING: /__w/xgboost/xgboost/src/learner.cc:782: 
+Parameters: { "use_label_encoder" } are not used.
+
+
+[06:55:44] WARNING: /__w/xgboost/xgboost/src/learner.cc:782: 
+Parameters: { "use_label_encoder" } are not used.
+[06:55:44] WARNING: /__w/xgboost/xgboost/src/learner.cc:782: 
+Parameters: { "use_label_encoder" } are not used.
+
+
+[06:55:44] WARNING: /__w/xgboost/xgboost/src/learner.cc:782: 
+Parameters: { "use_label_encoder" } are not used.
+
+[06:56:17] WARNING: /__w/xgboost/xgboost/src/common/error_msg.cc:62: Falling back to prediction using DMatrix due to mismatched devices. This might lead to higher memory usage and slower performance. XGBoost is running on: cuda:0, while the input data is on: cpu.
+Potential solutions:
+- Use a data structure that matches the device ordinal in the booster.
+- Set the device for booster before call to inplace_predict.
+
+This warning will only be shown once.
+
+  Stacking Ensemble: 0.7710
+  Voting Ensemble: 0.7786
+  ✅ Best ensemble: XGBoost = 0.7939
+Extracted 316 rich features per sample
+
+============================================================
+STRATEGY 4: 5-Fold Stratified Cross-Validation
+============================================================
+  Fold 1: 0.7939
+  Fold 2: 0.8244
+  Fold 3: 0.7769
+  Fold 4: 0.8154
+  Fold 5: 0.7615
+  ✅ CV Mean: 0.7944 ± 0.0234
+
+============================================================
+📊 TỔNG KẾT KẾT QUẢ
+============================================================
+  📈 LightCNN: 0.8626
+  📈 Hybrid CNN+XGBoost: 0.8244
+  📈 CV Mean (XGBoost rich): 0.7944
+  📈 Ensemble XGBoost: 0.7939
+  📈 Ensemble ExtraTrees: 0.7863
+  📈 Ensemble LightGBM: 0.7786
+  📈 Ensemble Voting: 0.7786
+  📈 Ensemble RandomForest: 0.7710
+  📈 Ensemble GBM: 0.7710
+  📈 Ensemble Stacking: 0.7710
+  📈 Flat XGBoost (baseline): 0.7328
+
+🏆 BEST: LightCNN = 0.8626
+
+✅ Kết quả đã được lưu vào model_train/ultimate_results.json
+⚠️ Chưa đạt 95%. Best = 0.8626. Cần thêm dữ liệu hoặc feature engineering.

.temp/log_ultimate_v2.txt

@@ -0,0 +1,69 @@
+🚀 CHIẾN LƯỢC V2: TOÀN DIỆN ĐẠT >95%
+============================================================
+Clean data: (652, 24, 16, 16), 7 classes
+
+============================================================
+CNN + TTA (Test-Time Augmentation)
+============================================================
+  Ep 1 Loss=1.6500 TTA-Acc=0.1450 🌟
+  Ep 2 Loss=1.4633 TTA-Acc=0.3511 🌟
+  Ep 3 Loss=1.3316 TTA-Acc=0.6336 🌟
+  Ep 4 Loss=1.2101 TTA-Acc=0.7252 🌟
+  Ep 6 Loss=1.2056 TTA-Acc=0.8015 🌟
+  Ep 13 Loss=1.1216 TTA-Acc=0.8244 🌟
+  Ep 23 Loss=0.9355 TTA-Acc=0.8321 🌟
+  Ep 30 Loss=0.9346 TTA-Acc=0.8092 (pat=7)
+  Ep 60 Loss=0.9658 TTA-Acc=0.7634 (pat=37)
+  Ep 69 Loss=0.8988 TTA-Acc=0.8397 🌟
+  Ep 74 Loss=0.9159 TTA-Acc=0.8550 🌟
+  Ep 90 Loss=0.8761 TTA-Acc=0.8168 (pat=16)
+  Ep 120 Loss=0.9473 TTA-Acc=0.8092 (pat=46)
+  Ep 139 Loss=0.9317 TTA-Acc=0.8626 🌟
+  Ep 150 Loss=0.7716 TTA-Acc=0.8397 (pat=11)
+  Ep 175 Loss=0.7432 TTA-Acc=0.8702 🌟
+  Ep 180 Loss=0.8290 TTA-Acc=0.8702 (pat=5)
+  Ep 210 Loss=0.8060 TTA-Acc=0.8626 (pat=35)
+  Ep 240 Loss=0.6980 TTA-Acc=0.8473 (pat=65)
+  Early stop ep 255
+  ✅ CNN+TTA best: 0.8702
+
+============================================================
+RICH FEATURES V2 + ENSEMBLE
+============================================================
+Extracted 1713 features per sample
+  XGB: 0.7557
+  LGBM: 0.7634
+  ET: 0.7481
+  RF: 0.7557
+  Voting: 0.7634
+
+  5-Fold CV:
+    Fold 1: 0.7557
+    Fold 2: 0.7939
+    Fold 3: 0.7769
+    Fold 4: 0.8308
+    Fold 5: 0.8000
+  CV: 0.7915 ± 0.0249
+
+============================================================
+HYBRID V2: CNN embed + Rich features + XGBoost
+============================================================
+Extracted 1713 features per sample
+  Combined: (652, 2097)
+  ✅ Hybrid V2: 0.8244
+  CV: 0.9142 ± 0.0194
+
+============================================================
+📊 KẾT QUẢ TỔNG HỢP V2
+============================================================
+  ✅ Hybrid CV: 0.9142
+  📈 CNN+TTA: 0.8702
+  📈 Hybrid V2: 0.8244
+  📈 Ens CV: 0.7915
+  📈 Ens_LGBM: 0.7634
+  📈 Ens_Vote: 0.7634
+  📈 Ens_XGB: 0.7557
+  📈 Ens_RF: 0.7557
+  📈 Ens_ET: 0.7481
+
+🏆 BEST: Hybrid CV = 0.9142

.temp/log_ultimate_v3.txt

@@ -0,0 +1,27 @@
+🚀 V3: MULTI-SEED ENSEMBLE + T0-ONLY + SELF-TRAINING
+============================================================
+Clean: (652, 24, 16, 16), 7 classes, [65, 52, 48, 72, 108, 219, 88]
+
+============================================================
+MULTI-SEED CNN ENSEMBLE (10 models)
+============================================================
+  Seed 0: 0.8473
+  Seed 1: 0.8702
+  Seed 2: 0.8550
+  Seed 3: 0.8550
+  Seed 4: 0.8702
+  Seed 5: 0.8626
+  Seed 6: 0.8702
+  Seed 7: 0.8702
+  Seed 8: 0.8702
+  Seed 9: 0.8702
+  ✅ 10-Model Ensemble TTA: 0.8473
+
+============================================================
+TIMESTEP-0-ONLY XGBoost (cleanest data)
+============================================================
+  T0 valid: 539/652
+  Features: (539, 1638)
+  XGB t0: 0.6852
+  LGBM t0: 0.6296
+  ET t0: 0.6852

.temp/log_xgboost.txt

@@ -0,0 +1,15 @@
+🚀 BẮT ĐẦU HUẤN LUYỆN MÔ HÌNH XGBOOST (GPU & CACHE)
+Initializing FeatureExtractor (mode=extended)...
+📦 Đang load cache: training_data_507bd2ba4ec0d3fe107839cbf73a7a7d.joblib...
+✅ Loaded 632 samples từ cache!
+   ⚡ Đã bỏ qua download위성 data (tiết kiệm thời gian)
+[CACHE HIT] Using cached dataset with 632 samples
+Training XGBOOST model...
+Evaluating model...
+Generating classification report...
+Saving model...
+[MODEL MANAGER] Saving model to: model_train/model_xgboost_auto.joblib
+[MODEL MANAGER] Saving metadata to: model_train/model_xgboost_auto_info.json
+[MODEL MANAGER] Model saved successfully!
+Training complete!
+✅ Hoàn thành! Accuracy: 0.6378

.temp/new_import_ODC.py

@@ -1,8 +1,16 @@
+TEST_MODE = True
+RESOLUTION = 1000 if TEST_MODE else 10
 import matplotlib.pyplot as plt
 
 # Common imports and settings
 import os, sys
 os.environ['USE_PYGEOS'] = '0'
+os.environ["GDAL_HTTP_MAX_RETRY"] = "5"
+os.environ["GDAL_HTTP_RETRY_DELAY"] = "2"
+os.environ["GDAL_HTTP_CONNECTION_TIMEOUT"] = "10"
+os.environ["GDAL_HTTP_TIMEOUT"] = "30"
+os.environ["CPL_VSIL_CURL_ALLOWED_EXTENSIONS"] = ".tif,.tiff"
+os.environ["GDAL_DISABLE_READDIR_ON_OPEN"] = "YES"
 from IPython.display import Markdown
 import pandas as pd
 pd.set_option("display.max_rows", None)
@@ -13,15 +21,13 @@ import datacube
 from datacube.utils.rio import configure_s3_access
 from datacube.utils import masking
 from datacube.utils.cog import write_cog
-# https://github.com/GeoscienceAustralia/dea-notebooks/tree/develop/Tools
-from dea_tools.plotting import display_map, rgb
-from dea_tools.datahandling import mostcommon_crs
+# removed deafrica_tools imports to avoid ipyleaflet error
 
 # EASI defaults
-easinotebooksrepo = '/home/jovyan/easi-notebooks'
+easinotebooksrepo = '/home/x79/CSIROBoeingPhase4-Vietnam'
 if easinotebooksrepo not in sys.path: sys.path.append(easinotebooksrepo)
 from easi_tools import EasiDefaults, xarray_object_size, notebook_utils, unset_cachingproxy
-from easi_tools.load_s2l2a import load_s2l2a_with_offset
+# from easi_tools.load_s2l2a import load_s2l2a_with_offset
 from dask.distributed import progress
 
 # Data tools
@@ -31,7 +37,7 @@ from datetime import datetime
 # Datacube
 from datacube.utils import masking  # https://github.com/opendatacube/datacube-core/blob/develop/datacube/utils/masking.py
 from odc.algo import enum_to_bool   # https://github.com/opendatacube/odc-algo/blob/main/odc/algo/_masking.py
-from odc.algo import xr_reproject   # https://github.com/opendatacube/odc-algo/blob/main/odc/algo/_warp.py
+# removed xr_reproject
 from datacube.utils.geometry import GeoBox, box  # https://github.com/opendatacube/datacube-core/blob/develop/datacube/utils/geometry/_base.py
 
 # Holoviews, Datashader and Bokeh
@@ -51,13 +57,12 @@ import rioxarray
 hv.extension('bokeh', logo=False)
 
 from deafrica_tools.bandindices import calculate_indices
-from sklearn.ensemble import RandomForestClassifier
+from xgboost import XGBClassifier
 from sklearn.model_selection import train_test_split
 from sklearn.metrics import accuracy_score, classification_report
 from sklearn.preprocessing import LabelEncoder
 
 from sklearn.pipeline import Pipeline
-from sklearn.ensemble import RandomForestClassifier
 from sklearn.impute import SimpleImputer
 from sklearn.preprocessing import StandardScaler
 from sklearn.model_selection import GridSearchCV
@@ -82,57 +87,90 @@ import joblib
 
 
 def load_data(dc, date_range, longtitude_range, latitude_range):
-    product = 's2_l2a'
-    query = {
-        'product': product,                     # Product name
-        'x': longtitude_range,    # "x" axis bounds
-        'y': latitude_range,      # "y" axis bounds
-        'time': date_range,           # Any parsable date strings
-    }
-    native_crs = notebook_utils.mostcommon_crs(dc, query)
-    print(f'Most common native CRS: {native_crs}')
-    measurements = ['red', 'nir', 'scl']
+    import os, hashlib
+    cache_dir = "dataset_cache"
+    os.makedirs(cache_dir, exist_ok=True)
+    key_str = f"s2_{date_range}_{longtitude_range}_{latitude_range}"
+    cache_key = hashlib.md5(key_str.encode()).hexdigest() + ".nc"
+    cache_path = os.path.join(cache_dir, cache_key)
+    
+    if os.path.exists(cache_path):
+        print(f"✅ Loading cached S2 data from {cache_path}")
+        return xr.open_dataset(cache_path, engine='netcdf4')
 
-    load_params = {
-        'measurements': measurements,                   # Selected measurement or alias names
-        'output_crs': native_crs,                       # Target EPSG code
-        'resolution': (-10, 10),                        # Target resolution
-        'group_by': 'solar_day',                        # Scene grouping
-        'dask_chunks': {'x': 2048, 'y': 2048},          # Dask chunks
-    }
-    data = load_s2l2a_with_offset(
-        dc,
-        query | load_params   # Combine the two dicts that contain our search and load parameters
+    product = 's2_l2a'
+    bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
+    
+    import pystac_client
+    import planetary_computer
+    import odc.stac
+    
+    catalog = pystac_client.Client.open(
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
+        modifier=planetary_computer.sign_inplace,
     )
+    search = catalog.search(
+        collections=["sentinel-2-l2a"],
+        bbox=bbox,
+        datetime=f"{date_range[0]}/{date_range[1]}",
+    )
+    items = list(search.items())
+    
+    data = odc.stac.load(
+        items,
+        bands=["red", "nir", "SCL"],
+        bbox=bbox,
+        crs="EPSG:32648",
+        resolution=RESOLUTION,
+        chunks={"x": 2048, "y": 2048, "time": 1},
+        groupby="solar_day"
+    )
+    if "SCL" in data.data_vars:
+        data = data.rename({"SCL": "scl"})
+        
+    print(f"💾 Caching S2 data to {cache_path}")
+    data = data.compute()
+    data.to_netcdf(cache_path, engine='netcdf4')
+    
     return data
 
 
 def mask_clean(data):
-    flag_name = 'scl'
-    flag_desc = masking.describe_variable_flags(data[flag_name])  # Pandas dataframe
-    display(flag_desc)
-    display(flag_desc.loc['qa'].values[1])
-    # Create a "data quality" Mask layer
-    flags_def = flag_desc.loc['qa'].values[1]
-    good_pixel_flags = [flags_def[str(i)] for i in [2, 4, 5, 6]]  # To pass strings to enum_to_bool()
-
-    # enum_to_bool calculates the pixel-wise "or" of each set of pixels given by good_pixel_flags
-    # 1 = good data
-    # 0 = "bad" data
-    good_pixel_mask = enum_to_bool(data[flag_name], good_pixel_flags)
+    # For Sentinel-2 L2A SCL: 
+    # 2: Dark Area Pixels, 4: Vegetation, 5: Not Vegetated, 6: Water
+    good_pixel_mask = data['scl'].isin([2, 4, 5, 6])
     data_layer_names = [x for x in data.data_vars if x != 'scl']
-    # Apply good pixel mask to blue, green, red and nir.
+    # Apply good pixel mask
     result = data[data_layer_names].where(good_pixel_mask).persist()
     return result
 
 
 def fill_nan(ndvi, time_split):
+    if len(ndvi.time) == 0:
+        return ndvi
+    
+    # If the total time duration is less than 90 days, skip seasonal splitting
+    try:
+        total_days = (ndvi.time[-1] - ndvi.time[0]).dt.days.item()
+        if total_days < 90:
+            return ndvi.bfill(dim="time").ffill(dim="time")
+    except Exception:
+        pass
+        
     rs = []
     for times in time_split:
-        tmp = ndvi.sel(time=times)
-        fill_ds = tmp.sel(time=times).bfill(dim='time')
-        fill_ds = fill_ds.sel(time=times).ffill(dim='time')   
-        rs.append(fill_ds)
+        try:
+            tmp = ndvi.sel(time=times)
+            if len(tmp.time) == 0:
+                continue
+            fill_ds = tmp.bfill(dim='time').ffill(dim='time')   
+            rs.append(fill_ds)
+        except Exception:
+            continue
+            
+    if len(rs) == 0:
+        return ndvi.bfill(dim="time").ffill(dim="time")
+        
     merged_ndvi = xr.concat([i for i in rs], dim="time")
     fill_m = merged_ndvi.bfill(dim="time")
     fill_m = fill_m.ffill(dim="time")
@@ -144,10 +182,68 @@ def load_train_data(train_path):
     return train
 
 
-def load_sen1(name_vh, name_vv):
-    dsvv = rioxarray.open_rasterio(name_vv)
-    dsvh = rioxarray.open_rasterio(name_vh)
-    return dsvh, dsvv
+def load_sen1(bbox, time_range):
+    import os, hashlib
+    cache_dir = "dataset_cache"
+    os.makedirs(cache_dir, exist_ok=True)
+    key_str = f"s1_vh_vv_{bbox}_{time_range}"
+    cache_key_vh = hashlib.md5((key_str + "vh").encode()).hexdigest() + ".nc"
+    cache_key_vv = hashlib.md5((key_str + "vv").encode()).hexdigest() + ".nc"
+    cache_path_vh = os.path.join(cache_dir, cache_key_vh)
+    cache_path_vv = os.path.join(cache_dir, cache_key_vv)
+    
+    if os.path.exists(cache_path_vh) and os.path.exists(cache_path_vv):
+        print(f"✅ Loading cached S1 data from {cache_path_vh} and {cache_path_vv}")
+        ds_vh = xr.open_dataset(cache_path_vh, engine='netcdf4')
+        ds_vv = xr.open_dataset(cache_path_vv, engine='netcdf4')
+        return ds_vh[list(ds_vh.data_vars)[0]], ds_vv[list(ds_vv.data_vars)[0]]
+
+    import pystac_client
+    import planetary_computer
+    import odc.stac
+    
+    # Kết nối STAC Client
+    catalog = pystac_client.Client.open(
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
+        modifier=planetary_computer.sign_inplace,
+    )
+
+    # Tìm kiếm Items
+    search = catalog.search(
+        collections=["sentinel-1-rtc"],
+        bbox=bbox,
+        datetime=time_range,
+    )
+    items = list(search.items())
+
+    # Tải dữ liệu thành xarray Dataset
+    ds_s1 = odc.stac.load(
+        items,
+        bands=["vv", "vh"],
+        bbox=bbox,
+        crs="EPSG:32648",
+        resolution=RESOLUTION,
+        chunks={"x": 2048, "y": 2048, "time": 1}
+    )
+
+    # Tính giá trị trung vị theo thời gian
+    ds_median = ds_s1.median(dim="time").compute()
+    vv = ds_median["vv"]
+    vh = ds_median["vh"]
+
+    # Thêm chiều 'band' để giống hệt rioxarray
+    vv = vv.expand_dims(dim="band")
+    vh = vh.expand_dims(dim="band")
+
+    # Phục hồi metadata về toạ độ
+    vv = vv.rio.write_crs("EPSG:32648")
+    vh = vh.rio.write_crs("EPSG:32648")
+
+    print(f"💾 Caching S1 data to {cache_dir}")
+    vh.to_netcdf(cache_path_vh, engine='netcdf4')
+    vv.to_netcdf(cache_path_vv, engine='netcdf4')
+
+    return vh, vv
 
 
 def get_data_sen1_and_sen2(train, average_ndvi, dsvh, dsvv):
@@ -192,7 +288,7 @@ def train_with_rf(X_train, X_val, y_train, y_val):
     # Takes 1-2 minutes to complete
 
     # Tạo RandomForestClassifier mặc định để sử dụng làm mô hình ban đầu trong pipeline
-    base_model = RandomForestClassifier(random_state=42, n_jobs=-1)
+    base_model = XGBClassifier(tree_method="hist", device="cuda", random_state=42, n_jobs=-1)
 
     # Tạo pipeline
     pipeline = Pipeline([
@@ -204,7 +300,7 @@ def train_with_rf(X_train, X_val, y_train, y_val):
     param_grid = {
         'classifier__n_estimators': [100, 300, 500, 700, 1000],
         'classifier__max_depth': [6, 8, 10, 15, 20],
-        'classifier__criterion': ['gini', 'entropy'],
+        'classifier__learning_rate': [0.01, 0.1, 0.2],
     }
 
     # Sử dụng GridSearchCV để tìm bộ tham số tốt nhất
@@ -267,6 +363,10 @@ def save_model(name_file, model, metadata=None, label_encoder=None):
     
     
 def predict(model, data_crs, ndvi, vh, vv):
+    # Unpack model if it is wrapped in a dictionary (from ModelManager)
+    if isinstance(model, dict) and 'model' in model:
+        model = model['model']
+        
     data_predict = []
     for i in range(ndvi.shape[1]):
         ndvi_tmp = ndvi.isel(y=i).values
@@ -360,25 +460,59 @@ def save_result(result, HT_MAP):
         # plt.show()
         
 def load_data_sen1(dc, date_range, coordinates):
+    import os, hashlib
     longtitude_range, latitude_range = coordinates
-    data_sen1 = dc.load(
-        product="sentinel1_grd_gamma0_10m",
-        x=longtitude_range,
-        y=latitude_range,
-        time=date_range,
-        measurements=["vv", "vh"],
-        output_crs="EPSG:32648",
-        resolution=(-10,10),
-        dask_chunks={"x":2048, "y":2048},
-        skip_broken_datasets=True,
-        group_by='solar_day'
+    bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
+    
+    cache_dir = "dataset_cache"
+    os.makedirs(cache_dir, exist_ok=True)
+    key_str = f"data_sen1_{date_range}_{bbox}"
+    cache_key_vh = hashlib.md5((key_str + "vh").encode()).hexdigest() + ".nc"
+    cache_key_vv = hashlib.md5((key_str + "vv").encode()).hexdigest() + ".nc"
+    cache_path_vh = os.path.join(cache_dir, cache_key_vh)
+    cache_path_vv = os.path.join(cache_dir, cache_key_vv)
+    
+    if os.path.exists(cache_path_vh) and os.path.exists(cache_path_vv):
+        print(f"✅ Loading cached S1 (coord) data")
+        ds_vh = xr.open_dataset(cache_path_vh, engine='netcdf4')
+        ds_vv = xr.open_dataset(cache_path_vv, engine='netcdf4')
+        var_vh = [v for v in ds_vh.data_vars if v != 'spatial_ref'][0]
+        var_vv = [v for v in ds_vv.data_vars if v != 'spatial_ref'][0]
+        return ds_vh[var_vh], ds_vv[var_vv]
+    
+    import pystac_client
+    import planetary_computer
+    import odc.stac
+    
+    catalog = pystac_client.Client.open(
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
+        modifier=planetary_computer.sign_inplace,
+    )
+    search = catalog.search(
+        collections=["sentinel-1-rtc"],
+        bbox=bbox,
+        datetime=f"{date_range[0]}/{date_range[1]}",
+    )
+    items = list(search.items())
+    
+    data_sen1 = odc.stac.load(
+        items,
+        bands=["vv", "vh"],
+        bbox=bbox,
+        crs="EPSG:32648",
+        resolution=RESOLUTION,
+        chunks={"x": 2048, "y": 2048, "time": 1},
+        groupby="solar_day"
     )
     
-    notebook_utils.heading(notebook_utils.xarray_object_size(data_sen1))
-    display(data_sen1)
+    data_sen1 = data_sen1.compute()
     dsvh = data_sen1.vh
     dsvv = data_sen1.vv
     
+    print(f"💾 Caching S1 (coord) data")
+    dsvh.to_netcdf(cache_path_vh, engine='netcdf4')
+    dsvv.to_netcdf(cache_path_vv, engine='netcdf4')
+    
     return dsvh, dsvv
     
 def calculate_average(data, time_pattern='1M'):
@@ -386,54 +520,66 @@ def calculate_average(data, time_pattern='1M'):
 
 
 def load_data_sen2(dc, date_range, coordinates):
+    import os, hashlib
     longtitude_range, latitude_range = coordinates
-    product = 's2_l2a'
-    query = {
-        'product': product,                     # Product name
-        'x': longtitude_range,    # "x" axis bounds
-        'y': latitude_range,      # "y" axis bounds
-        'time': date_range,           # Any parsable date strings
-    }
-    native_crs = notebook_utils.mostcommon_crs(dc, query)
-    print(f'Most common native CRS: {native_crs}')
+    bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
     
-    # measurements = ['red','green', 'blue', 'nir', 'scl']
-    measurements = ['red', 'nir', 'scl']
-
-    load_params = {
-        'measurements': measurements,                   # Selected measurement or alias names
-        'output_crs': native_crs,                       # Target EPSG code
-        'resolution': (-10, 10),                        # Target resolution
-        'group_by': 'solar_day',                        # Scene grouping
-        'dask_chunks': {'x': 2048, 'y': 2048},          # Dask chunks
-    }
-    data = load_s2l2a_with_offset(
-        dc,
-        query | load_params   # Combine the two dicts that contain our search and load parameters
+    cache_dir = "dataset_cache"
+    os.makedirs(cache_dir, exist_ok=True)
+    key_str = f"data_sen2_{date_range}_{bbox}"
+    cache_key = hashlib.md5(key_str.encode()).hexdigest() + ".nc"
+    cache_path = os.path.join(cache_dir, cache_key)
+    
+    if os.path.exists(cache_path):
+        print(f"✅ Loading cached S2 (coord) data from {cache_path}")
+        return xr.open_dataset(cache_path, engine='netcdf4')
+    
+    import pystac_client
+    import planetary_computer
+    import odc.stac
+    
+    catalog = pystac_client.Client.open(
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
+        modifier=planetary_computer.sign_inplace,
     )
+    search = catalog.search(
+        collections=["sentinel-2-l2a"],
+        bbox=bbox,
+        datetime=f"{date_range[0]}/{date_range[1]}",
+    )
+    items = list(search.items())
+    
+    data = odc.stac.load(
+        items,
+        bands=["red", "nir", "SCL"],
+        bbox=bbox,
+        crs="EPSG:32648",
+        resolution=RESOLUTION,
+        chunks={"x": 2048, "y": 2048, "time": 1},
+        groupby="solar_day"
+    )
+    if "SCL" in data.data_vars:
+        data = data.rename({"SCL": "scl"})
+        
+    data = data.compute()
+    print(f"💾 Caching S2 (coord) data to {cache_path}")
+    data.to_netcdf(cache_path, engine='netcdf4')
+    
     return data
 
 def mask_cloud(data):
-    flag_name = 'scl'
-    flag_desc = masking.describe_variable_flags(data[flag_name])  # Pandas dataframe
-    display(flag_desc.loc['qa'].values[1])
-    # Create a "data quality" Mask layer
-    flags_def = flag_desc.loc['qa'].values[1]
-    good_pixel_flags = [flags_def[str(i)] for i in [2, 4, 5, 6]]  # To pass strings to enum_to_bool()
-
-    # enum_to_bool calculates the pixel-wise "or" of each set of pixels given by good_pixel_flags
-    # 1 = good data
-    # 0 = "bad" data
-    good_pixel_mask = enum_to_bool(data[flag_name], good_pixel_flags)
+    # For Sentinel-2 L2A SCL: 
+    # 2: Dark Area Pixels, 4: Vegetation, 5: Not Vegetated, 6: Water
+    good_pixel_mask = data['scl'].isin([2, 4, 5, 6])
     data_layer_names = [x for x in data.data_vars if x != 'scl']
-    # Apply good pixel mask to blue, green, red and nir.
+    # Apply good pixel mask
     result = data[data_layer_names].where(good_pixel_mask).persist()
     return result
 
 def find_best_model(dataset):
     X_train, X_val, y_train, y_val = dataset
     # Tạo RandomForestClassifier mặc định để sử dụng làm mô hình ban đầu trong pipeline
-    base_model = RandomForestClassifier(random_state=42, n_jobs=-1)
+    base_model = XGBClassifier(tree_method="hist", device="cuda", random_state=42, n_jobs=-1)
 
     # Tạo pipeline
     pipeline = Pipeline([
@@ -445,7 +591,7 @@ def find_best_model(dataset):
     param_grid = {
         'classifier__n_estimators': [100, 300, 500, 700, 1000],
         'classifier__max_depth': [6, 8, 10, 15, 20],
-        'classifier__criterion': ['gini', 'entropy'],
+        'classifier__learning_rate': [0.01, 0.1, 0.2],
     }
 
     # Sử dụng GridSearchCV để tìm bộ tham số tốt nhất

.temp/patch_caching.py

@@ -0,0 +1,231 @@
+import re
+import os
+
+with open('/home/x79/remote-sensing/new_import_ODC.py', 'r', encoding='utf-8') as f:
+    content = f.read()
+
+# 1. load_data
+load_data_replacement = """def load_data(dc, date_range, longtitude_range, latitude_range):
+    import os, hashlib
+    cache_dir = "dataset_cache"
+    os.makedirs(cache_dir, exist_ok=True)
+    key_str = f"s2_{date_range}_{longtitude_range}_{latitude_range}"
+    cache_key = hashlib.md5(key_str.encode()).hexdigest() + ".nc"
+    cache_path = os.path.join(cache_dir, cache_key)
+    
+    if os.path.exists(cache_path):
+        print(f"✅ Loading cached S2 data from {cache_path}")
+        return xr.open_dataset(cache_path)
+
+    product = 's2_l2a'
+    bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
+    
+    import pystac_client
+    import planetary_computer
+    import odc.stac
+    
+    catalog = pystac_client.Client.open(
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
+        modifier=planetary_computer.sign_inplace,
+    )
+    search = catalog.search(
+        collections=["sentinel-2-l2a"],
+        bbox=bbox,
+        datetime=f"{date_range[0]}/{date_range[1]}",
+    )
+    items = list(search.items())
+    
+    data = odc.stac.load(
+        items,
+        bands=["red", "nir", "SCL"],
+        bbox=bbox,
+        crs="EPSG:32648",
+        resolution=RESOLUTION,
+        chunks={"x": 2048, "y": 2048, "time": 1},
+        groupby="solar_day"
+    )
+    if "SCL" in data.data_vars:
+        data = data.rename({"SCL": "scl"})
+        
+    print(f"💾 Caching S2 data to {cache_path}")
+    data = data.compute()
+    data.to_netcdf(cache_path)
+    
+    return data"""
+content = re.sub(r'def load_data\(dc, date_range, longtitude_range, latitude_range\):.*?return data', load_data_replacement, content, flags=re.DOTALL)
+
+
+# 2. load_sen1
+load_sen1_replacement = """def load_sen1(bbox, time_range):
+    import os, hashlib
+    cache_dir = "dataset_cache"
+    os.makedirs(cache_dir, exist_ok=True)
+    key_str = f"s1_vh_vv_{bbox}_{time_range}"
+    cache_key_vh = hashlib.md5((key_str + "vh").encode()).hexdigest() + ".nc"
+    cache_key_vv = hashlib.md5((key_str + "vv").encode()).hexdigest() + ".nc"
+    cache_path_vh = os.path.join(cache_dir, cache_key_vh)
+    cache_path_vv = os.path.join(cache_dir, cache_key_vv)
+    
+    if os.path.exists(cache_path_vh) and os.path.exists(cache_path_vv):
+        print(f"✅ Loading cached S1 data from {cache_path_vh} and {cache_path_vv}")
+        return xr.open_dataarray(cache_path_vh), xr.open_dataarray(cache_path_vv)
+
+    import pystac_client
+    import planetary_computer
+    import odc.stac
+    
+    # Kết nối STAC Client
+    catalog = pystac_client.Client.open(
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
+        modifier=planetary_computer.sign_inplace,
+    )
+
+    # Tìm kiếm Items
+    search = catalog.search(
+        collections=["sentinel-1-rtc"],
+        bbox=bbox,
+        datetime=time_range,
+    )
+    items = list(search.items())
+
+    # Tải dữ liệu thành xarray Dataset
+    ds_s1 = odc.stac.load(
+        items,
+        bands=["vv", "vh"],
+        bbox=bbox,
+        crs="EPSG:32648",
+        resolution=RESOLUTION,
+        chunks={"x": 2048, "y": 2048, "time": 1}
+    )
+
+    # Tính giá trị trung vị theo thời gian
+    ds_median = ds_s1.median(dim="time").compute()
+    vv = ds_median["vv"]
+    vh = ds_median["vh"]
+
+    # Thêm chiều 'band' để giống hệt rioxarray
+    vv = vv.expand_dims(dim="band")
+    vh = vh.expand_dims(dim="band")
+
+    # Phục hồi metadata về toạ độ
+    vv = vv.rio.write_crs("EPSG:32648")
+    vh = vh.rio.write_crs("EPSG:32648")
+
+    print(f"💾 Caching S1 data to {cache_dir}")
+    vh.to_netcdf(cache_path_vh)
+    vv.to_netcdf(cache_path_vv)
+
+    return vh, vv"""
+content = re.sub(r'def load_sen1\(bbox, time_range\):.*?return vh, vv', load_sen1_replacement, content, flags=re.DOTALL)
+
+
+# 3. load_data_sen1
+load_data_sen1_replacement = """def load_data_sen1(dc, date_range, coordinates):
+    import os, hashlib
+    longtitude_range, latitude_range = coordinates
+    bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
+    
+    cache_dir = "dataset_cache"
+    os.makedirs(cache_dir, exist_ok=True)
+    key_str = f"data_sen1_{date_range}_{bbox}"
+    cache_key_vh = hashlib.md5((key_str + "vh").encode()).hexdigest() + ".nc"
+    cache_key_vv = hashlib.md5((key_str + "vv").encode()).hexdigest() + ".nc"
+    cache_path_vh = os.path.join(cache_dir, cache_key_vh)
+    cache_path_vv = os.path.join(cache_dir, cache_key_vv)
+    
+    if os.path.exists(cache_path_vh) and os.path.exists(cache_path_vv):
+        print(f"✅ Loading cached S1 (coord) data")
+        return xr.open_dataarray(cache_path_vh), xr.open_dataarray(cache_path_vv)
+    
+    import pystac_client
+    import planetary_computer
+    import odc.stac
+    
+    catalog = pystac_client.Client.open(
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
+        modifier=planetary_computer.sign_inplace,
+    )
+    search = catalog.search(
+        collections=["sentinel-1-rtc"],
+        bbox=bbox,
+        datetime=f"{date_range[0]}/{date_range[1]}",
+    )
+    items = list(search.items())
+    
+    data_sen1 = odc.stac.load(
+        items,
+        bands=["vv", "vh"],
+        bbox=bbox,
+        crs="EPSG:32648",
+        resolution=RESOLUTION,
+        chunks={"x": 2048, "y": 2048, "time": 1},
+        groupby="solar_day"
+    )
+    
+    data_sen1 = data_sen1.compute()
+    dsvh = data_sen1.vh
+    dsvv = data_sen1.vv
+    
+    print(f"💾 Caching S1 (coord) data")
+    dsvh.to_netcdf(cache_path_vh)
+    dsvv.to_netcdf(cache_path_vv)
+    
+    return dsvh, dsvv"""
+content = re.sub(r'def load_data_sen1\(dc, date_range, coordinates\):.*?return dsvh, dsvv', load_data_sen1_replacement, content, flags=re.DOTALL)
+
+
+# 4. load_data_sen2
+load_data_sen2_replacement = """def load_data_sen2(dc, date_range, coordinates):
+    import os, hashlib
+    longtitude_range, latitude_range = coordinates
+    bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
+    
+    cache_dir = "dataset_cache"
+    os.makedirs(cache_dir, exist_ok=True)
+    key_str = f"data_sen2_{date_range}_{bbox}"
+    cache_key = hashlib.md5(key_str.encode()).hexdigest() + ".nc"
+    cache_path = os.path.join(cache_dir, cache_key)
+    
+    if os.path.exists(cache_path):
+        print(f"✅ Loading cached S2 (coord) data from {cache_path}")
+        return xr.open_dataset(cache_path)
+    
+    import pystac_client
+    import planetary_computer
+    import odc.stac
+    
+    catalog = pystac_client.Client.open(
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
+        modifier=planetary_computer.sign_inplace,
+    )
+    search = catalog.search(
+        collections=["sentinel-2-l2a"],
+        bbox=bbox,
+        datetime=f"{date_range[0]}/{date_range[1]}",
+    )
+    items = list(search.items())
+    
+    data = odc.stac.load(
+        items,
+        bands=["red", "nir", "SCL"],
+        bbox=bbox,
+        crs="EPSG:32648",
+        resolution=RESOLUTION,
+        chunks={"x": 2048, "y": 2048, "time": 1},
+        groupby="solar_day"
+    )
+    if "SCL" in data.data_vars:
+        data = data.rename({"SCL": "scl"})
+        
+    data = data.compute()
+    print(f"💾 Caching S2 (coord) data to {cache_path}")
+    data.to_netcdf(cache_path)
+    
+    return data"""
+content = re.sub(r'def load_data_sen2\(dc, date_range, coordinates\):.*?return data', load_data_sen2_replacement, content, flags=re.DOTALL)
+
+
+with open('/home/x79/remote-sensing/new_import_ODC.py', 'w', encoding='utf-8') as f:
+    f.write(content)
+
+print("Patching successful.")

.temp/patch_load_sen1.py

@@ -0,0 +1,29 @@
+import json
+import glob
+
+def fix_load_sen1(file_path):
+    with open(file_path, 'r', encoding='utf-8') as f:
+        nb = json.load(f)
+        
+    changed = False
+    for cell in nb.get('cells', []):
+        if cell.get('cell_type') == 'code':
+            source = cell.get('source', [])
+            for i, line in enumerate(source):
+                if 'load_sen1(name_vh, name_vv)' in line:
+                    indent = line[:len(line) - len(line.lstrip())]
+                    replacement = (
+                        f"{indent}bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]\n"
+                        f"{indent}time_range = f'{{date_range[0]}}/{{date_range[1]}}'\n"
+                        f"{indent}{line.lstrip().replace('load_sen1(name_vh, name_vv)', 'load_sen1(bbox, time_range)')}"
+                    )
+                    source[i] = replacement
+                    changed = True
+                    
+    if changed:
+        with open(file_path, 'w', encoding='utf-8') as f:
+            json.dump(nb, f, indent=1)
+        print(f"Patched load_sen1 in {file_path}")
+
+for nb in glob.glob("*.ipynb"):
+    fix_load_sen1(nb)

.temp/patch_notebooks.py

@@ -0,0 +1,43 @@
+import json
+import glob
+
+def patch_notebook(file_path):
+    with open(file_path, 'r', encoding='utf-8') as f:
+        nb = json.load(f)
+        
+    changed = False
+    for cell in nb.get('cells', []):
+        if cell.get('cell_type') == 'code':
+            source = cell.get('source', [])
+            
+            # Check if this cell should be fully commented out
+            full_source = ''.join(source)
+            if 'dc.load(' in full_source or 'ds.vv' in full_source:
+                for i in range(len(source)):
+                    if not source[i].startswith('#'):
+                        source[i] = '# ' + source[i]
+                changed = True
+                continue
+            
+            # Otherwise, do line-by-line replacements
+            for i, line in enumerate(source):
+                if 'ST_training data_updated_1130points.shp' in line:
+                    source[i] = line.replace('ST_training data_updated_1130points.shp', 'ST_training_data_updated_1130points.shp')
+                    changed = True
+                if 'from new_import import *' in line:
+                    source[i] = line.replace('from new_import import *', 'from new_import_ODC import *')
+                    changed = True
+                if 'dc = datacube.Datacube()' in line:
+                    source[i] = line.replace('dc = datacube.Datacube()', 'dc = None')
+                    changed = True
+                if 'load_data(dc,' in line:
+                    source[i] = line.replace('load_data(dc,', 'load_data(None,')
+                    changed = True
+                    
+    if changed:
+        with open(file_path, 'w', encoding='utf-8') as f:
+            json.dump(nb, f, indent=1)
+        print(f"Patched {file_path}")
+
+for nb in glob.glob("*.ipynb"):
+    patch_notebook(nb)

.temp/patch_rf_to_xgb.py

@@ -0,0 +1,75 @@
+import json
+import glob
+import re
+
+def patch_python_script(filepath):
+    try:
+        with open(filepath, 'r', encoding='utf-8') as f:
+            content = f.read()
+            
+        original_content = content
+        
+        # Replace imports
+        content = re.sub(r'from sklearn\.ensemble import RandomForestClassifier', 
+                         'from xgboost import XGBClassifier', content)
+        
+        # Replace the model instantiations (for 01.train_ODC.py)
+        rf_pattern = re.compile(r'model\s*=\s*RandomForestClassifier\([^)]+\)', re.DOTALL)
+        xgb_replacement = """model = XGBClassifier(
+                    n_estimators=200,
+                    max_depth=30,
+                    tree_method="hist",
+                    device="cuda",
+                    random_state=42,
+                    n_jobs=-1,
+                    verbosity=1
+                )"""
+        
+        content = rf_pattern.sub(xgb_replacement, content)
+        
+        if content != original_content:
+            with open(filepath, 'w', encoding='utf-8') as f:
+                f.write(content)
+            print(f"Patched {filepath}")
+    except Exception as e:
+        print(f"Error patching {filepath}: {e}")
+
+def patch_notebook(filepath):
+    try:
+        with open(filepath, 'r', encoding='utf-8') as f:
+            nb = json.load(f)
+            
+        changed = False
+        import_pattern = re.compile(r'from\s+sklearn\.ensemble\s+import\s+RandomForestClassifier')
+        inst_pattern = re.compile(r'RandomForestClassifier\([^)]*\)')
+        
+        for cell in nb.get('cells', []):
+            if cell.get('cell_type') == 'code':
+                source = cell.get('source', [])
+                for i in range(len(source)):
+                    if import_pattern.search(source[i]):
+                        source[i] = import_pattern.sub('from xgboost import XGBClassifier', source[i])
+                        changed = True
+                        
+                    if inst_pattern.search(source[i]):
+                        source[i] = inst_pattern.sub("XGBClassifier(tree_method='hist', device='cuda', random_state=42, n_jobs=-1)", source[i])
+                        changed = True
+                        
+                    if "'classifier__criterion': ['gini', 'entropy']" in source[i]:
+                        source[i] = source[i].replace("'classifier__criterion': ['gini', 'entropy']",
+                                                      "'classifier__learning_rate': [0.01, 0.1, 0.2]")
+                        changed = True
+                        
+        if changed:
+            with open(filepath, 'w', encoding='utf-8') as f:
+                json.dump(nb, f, indent=1)
+            print(f"Patched {filepath}")
+    except Exception as e:
+        print(f"Error patching {filepath}: {e}")
+
+if __name__ == "__main__":
+    patch_python_script("01.train_ODC.py")
+    patch_python_script("new_train.py")
+    
+    for nb in glob.glob("*.ipynb"):
+        patch_notebook(nb)

.temp/processMask-test.ipynb

@@ -1132,7 +1132,7 @@
    "source": [
     "%%time\n",
     "%matplotlib inline\n",
-    "from new_import import *"
+    "from new_import_ODC import *"
    ]
   },
   {
@@ -1216,7 +1216,7 @@
     "%%time\n",
     "# Dask gateway\n",
     "cluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))\n",
-    "dc = datacube.Datacube()\n",
+    "dc = None\n",
     "\n",
     "# Configure s3 access\n",
     "configure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n",
@@ -1233,11 +1233,11 @@
    },
    "outputs": [],
    "source": [
-    "## cấu hình thời gian lấy ảnh và tọa độ\n",
-    "# date_range = ('2022-09-01', '2023-10-01')\n",
+    "## c\u1ea5u h\u00ecnh th\u1eddi gian l\u1ea5y \u1ea3nh v\u00e0 t\u1ecda \u0111\u1ed9\n",
+    "# date_range = ('2022-09-01', '2022-10-01')\n",
     "# longtitude_range = (105.86575, 105.94120)\n",
     "# latitude_range = (9.65070, 9.69850)\n",
-    "date_range = ('2022-09-01', '2023-10-01')\n",
+    "date_range = ('2022-09-01', '2022-10-01')\n",
     "longtitude_range = (105.5, 106.4)\n",
     "latitude_range = (9.2, 10.0) "
    ]
@@ -1396,7 +1396,7 @@
        "\n",
        ".xr-section-summary-in + label:before {\n",
        "  display: inline-block;\n",
-       "  content: '►';\n",
+       "  content: '\u25ba';\n",
        "  font-size: 11px;\n",
        "  width: 15px;\n",
        "  text-align: center;\n",
@@ -1407,7 +1407,7 @@
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label:before {\n",
-       "  content: '▼';\n",
+       "  content: '\u25bc';\n",
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label > span {\n",
@@ -2424,8 +2424,8 @@
     }
    ],
    "source": [
-    "## truy vấn ảnh vệ tinh sen2\n",
-    "data = load_data(dc, date_range, longtitude_range, latitude_range)\n",
+    "## truy v\u1ea5n \u1ea3nh v\u1ec7 tinh sen2\n",
+    "data = load_data(None, date_range, longtitude_range, latitude_range)\n",
     "notebook_utils.heading(notebook_utils.xarray_object_size(data))\n",
     "display(data)"
    ]
@@ -2440,10 +2440,10 @@
    "outputs": [],
    "source": [
     "# Specify the start and end times \n",
-    "min_date = '2022-09-01' # Thời gian bắt đầu lấy data cho quá trình train\n",
-    "max_date = '2023-10-01' # Thời gian kết thúc lấy data cho quá trình train\n",
+    "min_date = '2022-09-01' # Th\u1eddi gian b\u1eaft \u0111\u1ea7u l\u1ea5y data cho qu\u00e1 tr\u00ecnh train\n",
+    "max_date = '2022-10-01' # Th\u1eddi gian k\u1ebft th\u00fac l\u1ea5y data cho qu\u00e1 tr\u00ecnh train\n",
     "# Just do 1 month for testing\n",
-    "# max_date = '2022-10-01' # Thời gian kết thúc lấy data cho quá trình train\n",
+    "# max_date = '2022-10-01' # Th\u1eddi gian k\u1ebft th\u00fac l\u1ea5y data cho qu\u00e1 tr\u00ecnh train\n",
     "\n",
     "# Specify a spatail region to search using latitude/longitude cooridinates\n",
     "min_longitude, max_longitude = (105.5, 106.4)\n",
@@ -2657,7 +2657,7 @@
        "\n",
        ".xr-section-summary-in + label:before {\n",
        "  display: inline-block;\n",
-       "  content: '►';\n",
+       "  content: '\u25ba';\n",
        "  font-size: 11px;\n",
        "  width: 15px;\n",
        "  text-align: center;\n",
@@ -2668,7 +2668,7 @@
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label:before {\n",
-       "  content: '▼';\n",
+       "  content: '\u25bc';\n",
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label > span {\n",
@@ -3423,23 +3423,23 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "CPU times: user 1.57 s, sys: 177 µs, total: 1.57 s\n",
+      "CPU times: user 1.57 s, sys: 177 \u00b5s, total: 1.57 s\n",
       "Wall time: 1.66 s\n"
      ]
     }
    ],
    "source": [
-    "%%time\n",
+    "# %%time\n",
     "# The replacement \"dc.load()\" function for this product\n",
-    "data = load_s2l2a_with_offset(\n",
-    "    dc,\n",
-    "    query | load_params   # Combine the two dicts that contain our search and load parameters\n",
-    ")\n",
-    "\n",
+    "# data = load_s2l2a_with_offset(\n",
+    "#     dc,\n",
+    "#     query | load_params   # Combine the two dicts that contain our search and load parameters\n",
+    "# )\n",
+    "# \n",
     "# This line prints the total size of the dataset hat was loaded\n",
-    "notebook_utils.heading(notebook_utils.xarray_object_size(data))\n",
-    "\n",
-    "display(data)"
+    "# notebook_utils.heading(notebook_utils.xarray_object_size(data))\n",
+    "# \n",
+    "# display(data)"
    ]
   },
   {
@@ -3546,9 +3546,9 @@
    ],
    "source": [
     "# %%time\n",
-    "# # Tiến hành loại bỏ các vị trí bị mây ảnh hưởng\n",
+    "# # Ti\u1ebfn h\u00e0nh lo\u1ea1i b\u1ecf c\u00e1c v\u1ecb tr\u00ed b\u1ecb m\u00e2y \u1ea3nh h\u01b0\u1edfng\n",
     "# result = mask_clean(data)\n",
-    "# progress(result)"
+    "# # progress(result)"
    ]
   },
   {
@@ -3684,7 +3684,7 @@
        "\n",
        ".xr-section-summary-in + label:before {\n",
        "  display: inline-block;\n",
-       "  content: '►';\n",
+       "  content: '\u25ba';\n",
        "  font-size: 11px;\n",
        "  width: 15px;\n",
        "  text-align: center;\n",
@@ -3695,7 +3695,7 @@
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label:before {\n",
-       "  content: '▼';\n",
+       "  content: '\u25bc';\n",
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label > span {\n",
@@ -4174,7 +4174,7 @@
     }
    ],
    "source": [
-    "# Tiến hành tính toán NDVI\n",
+    "# Ti\u1ebfn h\u00e0nh t\u00ednh to\u00e1n NDVI\n",
     "ds1 = calculate_indices(result, index='NDVI', satellite_mission='s2')\n",
     "ndvi = ds1[\"NDVI\"]\n",
     "display(ndvi)"
@@ -4213,9 +4213,9 @@
    ],
    "source": [
     "%%time\n",
-    "## tính ndvi theo tháng\n",
+    "## t\u00ednh ndvi theo th\u00e1ng\n",
     "average_ndvi = ndvi.resample(time='1M').mean().persist()\n",
-    "progress(average_ndvi)"
+    "# progress(average_ndvi)"
    ]
   },
   {
@@ -4253,12 +4253,12 @@
     }
    ],
    "source": [
-    "# cấu hình vh vv file\n",
+    "# c\u1ea5u h\u00ecnh vh vv file\n",
     "# name_vh = \"ThuanHoa/ThuanHoa_VH.tif\"\n",
     "# name_vv = \"ThuanHoa/ThuanHoa_VV.tif\"\n",
     "\n",
-    "# load dữ liệu sen1\n",
-    "# dsvh, dsvv = load_sen1(name_vh, name_vv)\n",
+    "# load d\u1eef li\u1ec7u sen1\n",
+    "bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]\ntime_range = f'{date_range[0]}/{date_range[1]}'\n# dsvh, dsvv = load_sen1(bbox, time_range)\n",
     "\n",
     "name_vh = \"vh-0922_0923-full_ST.tif\"\n",
     "name_vv = \"vv-0922_0923-full_ST.tif\"\n",
@@ -4268,7 +4268,7 @@
     "if not os.path.exists(name_vv):\n",
     "    !aws s3 cp s3://easi-asia-dc-data/staging/ctu/sentinel-1/vv-0922_0923-full_ST.tif vv-0922_0923-full_ST.tif\n",
     "    \n",
-    "dsvh, dsvv = load_sen1(name_vh, name_vv)"
+    "bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]\ntime_range = f'{date_range[0]}/{date_range[1]}'\ndsvh, dsvv = load_sen1(bbox, time_range)"
    ]
   },
   {
@@ -4326,7 +4326,7 @@
     {
      "data": {
       "text/html": [
-       "<style>#sk-container-id-1 {color: black;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>LinearRegression()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LinearRegression</label><div class=\"sk-toggleable__content\"><pre>LinearRegression()</pre></div></div></div></div></div>"
+       "<style>#sk-container-id-1 {color: black;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"\u25b8\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"\u25be\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>LinearRegression()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LinearRegression</label><div class=\"sk-toggleable__content\"><pre>LinearRegression()</pre></div></div></div></div></div>"
       ],
       "text/plain": [
        "LinearRegression()"
@@ -4397,7 +4397,7 @@
     }
    ],
    "source": [
-    "plt.imshow(average_ndvi_filled.isel(time=6))"
+    "plt.imshow(average_ndvi_filled.isel(time=0))"
    ]
   },
   {
@@ -4430,7 +4430,7 @@
     }
    ],
    "source": [
-    "plt.imshow(average_ndvi.isel(time=6))"
+    "plt.imshow(average_ndvi.isel(time=0))"
    ]
   },
   {
@@ -4440,7 +4440,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "train_path = \"train/ST_training data_updated_1130points.shp\""
+    "train_path = \"train/ST_training_data_updated_1130points.shp\""
    ]
   },
   {
@@ -4476,7 +4476,7 @@
    },
    "outputs": [],
    "source": [
-    "# cấu hình nhãn dữ liệu\n",
+    "# c\u1ea5u h\u00ecnh nh\u00e3n d\u1eef li\u1ec7u\n",
     "label_mapping = {\n",
     "    \"Lua tom\": \"0\",\n",
     "    \"Lua\": \"1\",\n",
@@ -4488,7 +4488,7 @@
     "    \"Rung\": \"7\"\n",
     "}\n",
     "\n",
-    "# chia tập dữ liệu train, val, test\n",
+    "# chia t\u1eadp d\u1eef li\u1ec7u train, val, test\n",
     "X_train, X_val, X_test, y_train, y_val, y_test = split_train_data(train, label_mapping, datasets)"
    ]
   },
@@ -4510,7 +4510,7 @@
     }
    ],
    "source": [
-    "# Huấn luyện mô hình\n",
+    "# Hu\u1ea5n luy\u1ec7n m\u00f4 h\u00ecnh\n",
     "grid_search = train_with_rf(X_train, X_val, y_train, y_val)"
    ]
   },
@@ -4531,7 +4531,7 @@
     }
    ],
    "source": [
-    "# kiểm tra độ chính xác với tập test\n",
+    "# ki\u1ec3m tra \u0111\u1ed9 ch\u00ednh x\u00e1c v\u1edbi t\u1eadp test\n",
     "y_pred_test = grid_search.predict(X_test)\n",
     "test_accuracy = accuracy_score(y_test, y_pred_test)\n",
     "print(f\"Accuracy for test data {round(test_accuracy, 2)*100} %\")"
@@ -4554,7 +4554,7 @@
     }
    ],
    "source": [
-    "# Lưu mô hình huấn luyện\n",
+    "# L\u01b0u m\u00f4 h\u00ecnh hu\u1ea5n luy\u1ec7n\n",
     "save_model(\"model_new.joblib\", grid_search)"
    ]
   },
@@ -4567,7 +4567,7 @@
    },
    "outputs": [],
    "source": [
-    "# đóng client, cluster\n",
+    "# \u0111\u00f3ng client, cluster\n",
     "client.close()\n",
     "cluster.close()"
    ]
@@ -4602,4 +4602,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 5
-}
+}

.temp/processMask-test.py

@@ -0,0 +1,272 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+get_ipython().run_cell_magic('time', '', '%matplotlib inline\nfrom new_import import *\n')
+
+
+# In[2]:
+
+
+get_ipython().run_cell_magic('time', '', '# Dask gateway\ncluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))\ndc = datacube.Datacube()\n\n# Configure s3 access\nconfigure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n\nclient\n')
+
+
+# In[3]:
+
+
+## cấu hình thời gian lấy ảnh và tọa độ
+# date_range = ('2022-09-01', '2023-10-01')
+# longtitude_range = (105.86575, 105.94120)
+# latitude_range = (9.65070, 9.69850)
+date_range = ('2022-09-01', '2023-10-01')
+longtitude_range = (105.5, 106.4)
+latitude_range = (9.2, 10.0) 
+
+
+# In[4]:
+
+
+## truy vấn ảnh vệ tinh sen2
+data = load_data(dc, date_range, longtitude_range, latitude_range)
+notebook_utils.heading(notebook_utils.xarray_object_size(data))
+display(data)
+
+
+# In[5]:
+
+
+# Specify the start and end times 
+min_date = '2022-09-01' # Thời gian bắt đầu lấy data cho quá trình train
+max_date = '2023-10-01' # Thời gian kết thúc lấy data cho quá trình train
+# Just do 1 month for testing
+# max_date = '2022-10-01' # Thời gian kết thúc lấy data cho quá trình train
+
+# Specify a spatail region to search using latitude/longitude cooridinates
+min_longitude, max_longitude = (105.5, 106.4)
+min_latitude, max_latitude = (9.2, 10.0)
+
+# Specify the product. In this case we want to use Sentinel-2 Level-2A data
+product = 's2_l2a'
+
+# Construct the search query dictionary
+query = {
+    'product': product,                     # Product name
+    'x': (min_longitude, max_longitude),    # "x" axis bounds
+    'y': (min_latitude, max_latitude),      # "y" axis bounds
+    'time': (min_date, max_date),           # Any parsable date strings
+}
+
+
+# In[6]:
+
+
+# Most common CRS
+native_crs = notebook_utils.mostcommon_crs(dc, query)
+print(f'Most common native CRS: {native_crs}')
+
+
+# In[7]:
+
+
+# Specify the spectral band measurements we want to use for a classification algorithm
+measurements = ['red', 'nir', 'scl']
+
+load_params = {
+    'measurements': measurements,                   # Selected measurement or alias names
+    'output_crs': native_crs,                       # Target EPSG code
+    'resolution': (-10, 10),                        # Target resolution
+    'group_by': 'solar_day',                        # Scene grouping
+    'dask_chunks': {'x': 2048, 'y': 2048},          # Dask chunks
+}
+
+
+# In[8]:
+
+
+get_ipython().run_cell_magic('time', '', '# The replacement "dc.load()" function for this product\ndata = load_s2l2a_with_offset(\n    dc,\n    query | load_params   # Combine the two dicts that contain our search and load parameters\n)\n\n# This line prints the total size of the dataset hat was loaded\nnotebook_utils.heading(notebook_utils.xarray_object_size(data))\n\ndisplay(data)\n')
+
+
+# In[9]:
+
+
+# %%time
+# # Tiến hành loại bỏ các vị trí bị mây ảnh hưởng
+# result = mask_clean(data)
+# progress(result)
+
+
+# In[10]:
+
+
+# Tiến hành tính toán NDVI
+ds1 = calculate_indices(result, index='NDVI', satellite_mission='s2')
+ndvi = ds1["NDVI"]
+display(ndvi)
+
+
+# In[11]:
+
+
+get_ipython().run_cell_magic('time', '', "## tính ndvi theo tháng\naverage_ndvi = ndvi.resample(time='1M').mean().persist()\nprogress(average_ndvi)\n")
+
+
+# In[12]:
+
+
+# compute average_ndvi
+average_ndvi = average_ndvi.compute()
+
+
+# In[13]:
+
+
+# cấu hình vh vv file
+# name_vh = "ThuanHoa/ThuanHoa_VH.tif"
+# name_vv = "ThuanHoa/ThuanHoa_VV.tif"
+
+# load dữ liệu sen1
+bbox = [105.5, 9.2, 106.4, 10.0]
+time_range = '2022-09-01/2023-10-01'
+# dsvh, dsvv = load_sen1(bbox, time_range)
+
+name_vh = "vh-0922_0923-full_ST.tif"
+name_vv = "vv-0922_0923-full_ST.tif"
+
+if not os.path.exists(name_vh):
+    get_ipython().system('aws s3 cp s3://easi-asia-dc-data/staging/ctu/sentinel-1/vh-0922_0923-full_ST.tif vh-0922_0923-full_ST.tif')
+if not os.path.exists(name_vv):
+    get_ipython().system('aws s3 cp s3://easi-asia-dc-data/staging/ctu/sentinel-1/vv-0922_0923-full_ST.tif vv-0922_0923-full_ST.tif')
+    
+bbox = [105.5, 9.2, 106.4, 10.0]
+time_range = '2022-09-01/2023-10-01'
+dsvh, dsvv = load_sen1(bbox, time_range)
+
+
+# In[27]:
+
+
+from sklearn.preprocessing import PolynomialFeatures
+from sklearn.linear_model import LinearRegression
+from sklearn.ensemble import RandomForestRegressor
+
+
+# In[28]:
+
+
+average_ndvi = average_ndvi[:, :7680, :8687]
+mask = ~np.isnan(average_ndvi)
+print(average_ndvi.shape)
+print(dsvh.shape)
+print(dsvv.shape)
+print(mask.shape)
+X_train = np.stack([dsvh.values[mask], dsvv.values[mask]], axis=1)
+y_train = average_ndvi.values[mask]
+
+
+# In[29]:
+
+
+model = LinearRegression()
+model.fit(X_train, y_train)
+
+
+# In[30]:
+
+
+X_pred = np.stack([dsvh.values[~mask], dsvv.values[~mask]], axis=1)
+average_ndvi.values[~mask] = model.predict(X_pred)
+
+
+# In[31]:
+
+
+average_ndvi_filled = xr.DataArray(average_ndvi, dims=average_ndvi.dims)
+
+
+# In[32]:
+
+
+plt.imshow(average_ndvi_filled.isel(time=6))
+
+
+# In[65]:
+
+
+plt.imshow(average_ndvi.isel(time=6))
+
+
+# In[33]:
+
+
+train_path = "train/ST_training data_updated_1130points.shp"
+
+
+# In[34]:
+
+
+train = load_train_data(train_path)
+
+
+# In[37]:
+
+
+datasets = get_data_sen1_and_sen2(train, average_ndvi_filled, dsvh, dsvv)
+
+
+# In[39]:
+
+
+# cấu hình nhãn dữ liệu
+label_mapping = {
+    "Lua tom": "0",
+    "Lua": "1",
+    "CHN": "2",
+    "CLN": "3",
+    "TS": "4",
+    "Song": "5",
+    "Dat xay dung": "6",
+    "Rung": "7"
+}
+
+# chia tập dữ liệu train, val, test
+X_train, X_val, X_test, y_train, y_val, y_test = split_train_data(train, label_mapping, datasets)
+
+
+# In[40]:
+
+
+# Huấn luyện mô hình
+grid_search = train_with_rf(X_train, X_val, y_train, y_val)
+
+
+# In[41]:
+
+
+# kiểm tra độ chính xác với tập test
+y_pred_test = grid_search.predict(X_test)
+test_accuracy = accuracy_score(y_test, y_pred_test)
+print(f"Accuracy for test data {round(test_accuracy, 2)*100} %")
+
+
+# In[42]:
+
+
+# Lưu mô hình huấn luyện
+save_model("model_new.joblib", grid_search)
+
+
+# In[43]:
+
+
+# đóng client, cluster
+client.close()
+cluster.close()
+
+
+# In[ ]:
+
+
+
+

.temp/processMask.ipynb

@@ -1132,7 +1132,7 @@
    "source": [
     "%%time\n",
     "%matplotlib inline\n",
-    "from new_import import *"
+    "from new_import_ODC import *"
    ]
   },
   {
@@ -1216,7 +1216,7 @@
     "%%time\n",
     "# Dask gateway\n",
     "cluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))\n",
-    "dc = datacube.Datacube()\n",
+    "dc = None\n",
     "\n",
     "# Configure s3 access\n",
     "configure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n",
@@ -1233,8 +1233,8 @@
    },
    "outputs": [],
    "source": [
-    "## cấu hình thời gian lấy ảnh và tọa độ\n",
-    "date_range = ('2022-09-01', '2023-10-01')\n",
+    "## c\u1ea5u h\u00ecnh th\u1eddi gian l\u1ea5y \u1ea3nh v\u00e0 t\u1ecda \u0111\u1ed9\n",
+    "date_range = ('2022-09-01', '2022-10-01')\n",
     "longtitude_range = (105.86575, 105.94120)\n",
     "latitude_range = (9.65070, 9.69850)"
    ]
@@ -1393,7 +1393,7 @@
        "\n",
        ".xr-section-summary-in + label:before {\n",
        "  display: inline-block;\n",
-       "  content: '►';\n",
+       "  content: '\u25ba';\n",
        "  font-size: 11px;\n",
        "  width: 15px;\n",
        "  text-align: center;\n",
@@ -1404,7 +1404,7 @@
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label:before {\n",
-       "  content: '▼';\n",
+       "  content: '\u25bc';\n",
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label > span {\n",
@@ -2333,8 +2333,8 @@
     }
    ],
    "source": [
-    "## truy vấn ảnh vệ tinh sen2\n",
-    "data = load_data(dc, date_range, longtitude_range, latitude_range)\n",
+    "## truy v\u1ea5n \u1ea3nh v\u1ec7 tinh sen2\n",
+    "data = load_data(None, date_range, longtitude_range, latitude_range)\n",
     "notebook_utils.heading(notebook_utils.xarray_object_size(data))\n",
     "display(data)"
    ]
@@ -2443,9 +2443,9 @@
    ],
    "source": [
     "%%time\n",
-    "# Tiến hành loại bỏ các vị trí bị mây ảnh hưởng\n",
+    "# Ti\u1ebfn h\u00e0nh lo\u1ea1i b\u1ecf c\u00e1c v\u1ecb tr\u00ed b\u1ecb m\u00e2y \u1ea3nh h\u01b0\u1edfng\n",
     "result = mask_clean(data)\n",
-    "progress(result)"
+    "# progress(result)"
    ]
   },
   {
@@ -2581,7 +2581,7 @@
        "\n",
        ".xr-section-summary-in + label:before {\n",
        "  display: inline-block;\n",
-       "  content: '►';\n",
+       "  content: '\u25ba';\n",
        "  font-size: 11px;\n",
        "  width: 15px;\n",
        "  text-align: center;\n",
@@ -2592,7 +2592,7 @@
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label:before {\n",
-       "  content: '▼';\n",
+       "  content: '\u25bc';\n",
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label > span {\n",
@@ -3047,7 +3047,7 @@
     }
    ],
    "source": [
-    "# Tiến hành tính toán NDVI\n",
+    "# Ti\u1ebfn h\u00e0nh t\u00ednh to\u00e1n NDVI\n",
     "ds1 = calculate_indices(result, index='NDVI', satellite_mission='s2')\n",
     "ndvi = ds1[\"NDVI\"]\n",
     "display(ndvi)"
@@ -3086,9 +3086,9 @@
    ],
    "source": [
     "%%time\n",
-    "## tính ndvi theo tháng\n",
+    "## t\u00ednh ndvi theo th\u00e1ng\n",
     "average_ndvi = ndvi.resample(time='1M').mean().persist()\n",
-    "progress(average_ndvi)"
+    "# progress(average_ndvi)"
    ]
   },
   {
@@ -3113,12 +3113,12 @@
    },
    "outputs": [],
    "source": [
-    "# cấu hình vh vv file\n",
+    "# c\u1ea5u h\u00ecnh vh vv file\n",
     "name_vh = \"ThuanHoa/ThuanHoa_VH.tif\"\n",
     "name_vv = \"ThuanHoa/ThuanHoa_VV.tif\"\n",
     "\n",
-    "# load dữ liệu sen1\n",
-    "dsvh, dsvv = load_sen1(name_vh, name_vv)"
+    "# load d\u1eef li\u1ec7u sen1\n",
+    "bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]\ntime_range = f'{date_range[0]}/{date_range[1]}'\ndsvh, dsvv = load_sen1(bbox, time_range)"
    ]
   },
   {
@@ -3160,7 +3160,7 @@
     {
      "data": {
       "text/html": [
-       "<style>#sk-container-id-1 {color: black;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>LinearRegression()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LinearRegression</label><div class=\"sk-toggleable__content\"><pre>LinearRegression()</pre></div></div></div></div></div>"
+       "<style>#sk-container-id-1 {color: black;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"\u25b8\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"\u25be\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>LinearRegression()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LinearRegression</label><div class=\"sk-toggleable__content\"><pre>LinearRegression()</pre></div></div></div></div></div>"
       ],
       "text/plain": [
        "LinearRegression()"
@@ -3297,4 +3297,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 5
-}
+}

.temp/processMask.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+get_ipython().run_cell_magic('time', '', '%matplotlib inline\nfrom new_import import *\n')
+
+
+# In[2]:
+
+
+get_ipython().run_cell_magic('time', '', '# Dask gateway\ncluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))\ndc = datacube.Datacube()\n\n# Configure s3 access\nconfigure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n\nclient\n')
+
+
+# In[3]:
+
+
+## cấu hình thời gian lấy ảnh và tọa độ
+date_range = ('2022-09-01', '2023-10-01')
+longtitude_range = (105.86575, 105.94120)
+latitude_range = (9.65070, 9.69850)
+
+
+# In[4]:
+
+
+## truy vấn ảnh vệ tinh sen2
+data = load_data(dc, date_range, longtitude_range, latitude_range)
+notebook_utils.heading(notebook_utils.xarray_object_size(data))
+display(data)
+
+
+# In[5]:
+
+
+get_ipython().run_cell_magic('time', '', '# Tiến hành loại bỏ các vị trí bị mây ảnh hưởng\nresult = mask_clean(data)\nprogress(result)\n')
+
+
+# In[6]:
+
+
+# Tiến hành tính toán NDVI
+ds1 = calculate_indices(result, index='NDVI', satellite_mission='s2')
+ndvi = ds1["NDVI"]
+display(ndvi)
+
+
+# In[17]:
+
+
+get_ipython().run_cell_magic('time', '', "## tính ndvi theo tháng\naverage_ndvi = ndvi.resample(time='1M').mean().persist()\nprogress(average_ndvi)\n")
+
+
+# In[18]:
+
+
+# compute average_ndvi
+average_ndvi = average_ndvi.compute()
+
+
+# In[9]:
+
+
+# cấu hình vh vv file
+name_vh = "ThuanHoa/ThuanHoa_VH.tif"
+name_vv = "ThuanHoa/ThuanHoa_VV.tif"
+
+# load dữ liệu sen1
+bbox = [105.5, 9.2, 106.4, 10.0]
+time_range = '2022-09-01/2023-10-01'
+dsvh, dsvv = load_sen1(bbox, time_range)
+
+
+# In[10]:
+
+
+from sklearn.preprocessing import PolynomialFeatures
+from sklearn.linear_model import LinearRegression
+from sklearn.ensemble import RandomForestRegressor
+
+
+# In[11]:
+
+
+mask = ~np.isnan(average_ndvi)
+X_train = np.stack([dsvh.values[mask], dsvv.values[mask]], axis=1)
+y_train = average_ndvi.values[mask]
+
+
+# In[12]:
+
+
+model = LinearRegression()
+model.fit(X_train, y_train)
+
+
+# In[13]:
+
+
+X_pred = np.stack([dsvh.values[~mask], dsvv.values[~mask]], axis=1)
+average_ndvi.values[~mask] = model.predict(X_pred)
+
+
+# In[14]:
+
+
+average_ndvi_filled = xr.DataArray(average_ndvi, dims=average_ndvi.dims)
+
+
+# In[16]:
+
+
+plt.imshow(average_ndvi_filled.isel(time=1))
+
+
+# In[19]:
+
+
+plt.imshow(average_ndvi.isel(time=1))
+
+
+# In[ ]:
+
+
+
+

.temp/rebuild_notebook.py

@@ -0,0 +1,145 @@
+import json
+
+def get_content(filename):
+    with open(filename, "r", encoding="utf-8") as f:
+        return f.read()
+
+fe_content = get_content("feature_extractor.py")
+tm_content = get_content("train_module.py")
+dt_content = get_content("train_land_decision_tree_gpu.py")
+
+notebook = {
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Tải Dữ liệu Vệ tinh qua Colab (Self-contained)\n",
+    "Notebook này đã được nhúng sẵn toàn bộ mã nguồn xử lý. Bạn không cần upload cả thư mục `remote-sensing` nữa.\n",
+    "\n",
+    "## Bước 1: Upload Shapefile (BẮT BUỘC)\n",
+    "Mô hình cần biết các điểm tọa độ đất để lấy dữ liệu. Hãy nén thư mục `train/` trên máy bạn thành `train.zip` và chạy ô dưới đây để upload nó trực tiếp lên Colab."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": None,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from google.colab import files\n",
+    "import os\n",
+    "\n",
+    "print(\"Hãy chọn file train.zip từ máy tính của bạn:\")\n",
+    "uploaded = files.upload()\n",
+    "\n",
+    "if \"train.zip\" in uploaded:\n",
+    "    !unzip -q -o train.zip -d /content/train_tmp/\n",
+    "    # Move the extracted files directly to /content/train/\n",
+    "    !mkdir -p /content/train\n",
+    "    !mv /content/train_tmp/*/* /content/train/ 2>/dev/null || mv /content/train_tmp/* /content/train/\n",
+    "    print(\"Đã giải nén shapefile thành công vào thư mục /content/train/\")\n",
+    "else:\n",
+    "    print(\"LỖI: Bạn chưa upload file có tên là train.zip!\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Bước 2: Cài đặt thư viện & Tạo môi trường"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": None,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!pip install planetary-computer pystac-client odc-stac geopandas rasterio xarray joblib scikit-learn xgboost lightgbm"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": None,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%writefile feature_extractor.py\n" + fe_content
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": None,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%writefile train_module.py\n" + tm_content
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": None,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%writefile train_land_decision_tree_gpu.py\n" + dt_content
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Bước 3: Chạy tiến trình tải ảnh vệ tinh và tạo Cache"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": None,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!python train_land_decision_tree_gpu.py"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Bước 4: Tải file Cache về máy"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": None,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from google.colab import files\n",
+    "import glob\n",
+    "\n",
+    "cache_files = glob.glob(\"dataset_cache/*.joblib\")\n",
+    "if cache_files:\n",
+    "    latest_cache = max(cache_files, key=os.path.getctime)\n",
+    "    print(f\"Đang tải file {latest_cache} về máy...\")\n",
+    "    files.download(latest_cache)\n",
+    "else:\n",
+    "    print(\"Chưa tìm thấy file cache. Hãy chắc chắn bước 3 đã chạy thành công!\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
+
+with open("Download_Cache_Colab.ipynb", "w", encoding="utf-8") as f:
+    json.dump(notebook, f, indent=1, ensure_ascii=False)
+
+print("Notebook updated successfully!")

.temp/remove_datacube.py

@@ -0,0 +1,35 @@
+import json
+import glob
+
+def fix_notebook(file_path):
+    with open(file_path, 'r', encoding='utf-8') as f:
+        nb = json.load(f)
+    changed = False
+    for cell in nb.get('cells', []):
+        if cell.get('cell_type') == 'code':
+            source = cell.get('source', [])
+            if isinstance(source, list):
+                for i, line in enumerate(source):
+                    if "dc = datacube.Datacube()" in line:
+                        source[i] = "dc = None\n"
+                        changed = True
+                    if "ds = dc.load(" in line:
+                        source[i] = "ds = None\n"
+                        changed = True
+                    if "data = dc.load(" in line:
+                        source[i] = "data = None\n"
+                        changed = True
+                    # If ds is None, ds.vv will fail
+                    if "vv_data = ds.vv" in line:
+                        source[i] = "vv_data = None\n"
+                        changed = True
+                    if "notebook_utils.xarray_object_size(ds)" in line:
+                        source[i] = line.replace("notebook_utils.xarray_object_size(ds)", "'ds is None'")
+                        changed = True
+    if changed:
+        with open(file_path, 'w', encoding='utf-8') as f:
+            json.dump(nb, f, indent=1)
+        print(f"Removed datacube from {file_path}")
+
+for nb in glob.glob("*.ipynb"):
+    fix_notebook(nb)

.temp/run_mock_parallel_and_compare.py

@@ -0,0 +1,110 @@
+import os
+import glob
+import json
+import subprocess
+import time
+from tabulate import tabulate
+
+scripts = [
+    "train_land_randomforest.py",
+    "train_cloud_cnn.py",
+    "train_cloud_swin_unet.py",
+    "train_ndvi_statistical.py",
+    "train_ndvi_lstm_gru.py",
+    "train_ndvi_convlstm.py",
+    "train_ndvi_hybrid_physics.py",
+    "train_ndvi_ensemble.py"
+]
+
+print("🚀 Đang khởi chạy song song tất cả các mô hình...")
+processes = []
+for script in scripts:
+    if os.path.exists(script):
+        cmd = f"source /home/x79/miniconda3/etc/profile.d/conda.sh && conda activate env_01 && python {script}"
+        p = subprocess.Popen(["bash", "-c", cmd], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        processes.append((script, p))
+
+for script, p in processes:
+    p.wait()
+
+print("✅ Đã chạy xong tất cả các mô hình!\n")
+print("📊 BẢNG SO SÁNH KẾT QUẢ CÁC MÔ HÌNH\n")
+
+# 1. Phân loại đất
+print("### 1. Nhóm Phân loại Lớp phủ (Land Classification)")
+land_data = []
+
+# Đọc XGBoost từ thư mục gốc
+if os.path.exists("model_xgboost_info.json"):
+    with open("model_xgboost_info.json", 'r') as f:
+        data = json.load(f)
+        params = data.get('params', {})
+        param_str = f"estimators:{params.get('n_estimators')}, depth:{params.get('max_depth')}" if params else "N/A"
+        land_data.append([
+            data.get('model_type', 'XGBoost'),
+            data.get('accuracy', ''),
+            data.get('precision', ''),
+            data.get('recall', ''),
+            data.get('f1_score', ''),
+            param_str
+        ])
+
+# Đọc các model khác trong model_train
+for info_file in glob.glob("model_train/*_info.json"):
+    with open(info_file, 'r') as f:
+        data = json.load(f)
+        # Chỉ lấy các model có độ chính xác (để lọc model rác/cũ)
+        if 'accuracy' not in data and 'f1_score' not in data:
+            continue
+            
+        params = data.get('params', {})
+        param_str = f"estimators:{params.get('n_estimators')}, depth:{params.get('max_depth')}" if params else "N/A"
+        
+        # Fallback for Random Forest
+        if data.get('model_type') == 'RandomForest_RealData':
+            param_str = "estimators:100, depth:15"
+            
+        land_data.append([
+            data.get('model_type', ''),
+            data.get('accuracy', ''),
+            data.get('precision', ''),
+            data.get('recall', ''),
+            data.get('f1_score', ''),
+            param_str
+        ])
+
+if land_data:
+    print(tabulate(land_data, headers=["Model", "Accuracy", "Precision", "Recall", "F1-Score", "Parameters"], tablefmt="github"))
+print("\n")
+
+# 2. Xóa mây
+print("### 2. Nhóm Xóa mây (Cloud Removal)")
+cloud_data = []
+for info_file in glob.glob("cloud_removal_model/*_info.json"):
+    with open(info_file, 'r') as f:
+        data = json.load(f)
+        cloud_data.append([
+            data.get('model_type', ''),
+            data.get('epoch', ''),
+            data.get('train_loss', ''),
+            data.get('val_loss', '')
+        ])
+if cloud_data:
+    print(tabulate(cloud_data, headers=["Model", "Epochs", "Train Loss", "Val Loss"], tablefmt="github"))
+print("\n")
+
+# 3. Dự báo NDVI
+print("### 3. Nhóm Dự báo Thực vật (NDVI Forecasting)")
+ndvi_data = []
+for info_file in glob.glob("ndvi_forecast_model/*_info.json"):
+    with open(info_file, 'r') as f:
+        data = json.load(f)
+        ndvi_data.append([
+            data.get('model_type', ''),
+            data.get('rmse', ''),
+            data.get('mae', ''),
+            data.get('epoch', 'N/A')
+        ])
+if ndvi_data:
+    print(tabulate(ndvi_data, headers=["Model", "RMSE", "MAE", "Epochs"], tablefmt="github"))
+print("\n")

.temp/test_VH_VV.ipynb

@@ -1141,12 +1141,12 @@
    "source": [
     "\n",
     "%matplotlib inline\n",
-    "from new_import import *\n",
+    "from new_import_ODC import *\n",
     "\n",
     "\n",
     "# Dask gateway\n",
     "cluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))\n",
-    "dc = datacube.Datacube()\n",
+    "dc = None\n",
     "\n",
     "\n",
     "# Configure s3 access\n",
@@ -1300,7 +1300,7 @@
        "\n",
        ".xr-section-summary-in + label:before {\n",
        "  display: inline-block;\n",
-       "  content: '►';\n",
+       "  content: '\u25ba';\n",
        "  font-size: 11px;\n",
        "  width: 15px;\n",
        "  text-align: center;\n",
@@ -1311,7 +1311,7 @@
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label:before {\n",
-       "  content: '▼';\n",
+       "  content: '\u25bc';\n",
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label > span {\n",
@@ -1880,20 +1880,20 @@
     }
    ],
    "source": [
-    "ds = dc.load(\n",
-    "    product=\"sentinel1_grd_gamma0_20m\",\n",
-    "    x=(105.5, 106.4),\n",
-    "    y=(9.2, 10.0),\n",
-    "    time=(\"2022-09-01\", \"2023-10-01\"),\n",
-    "    measurements=[\"vv\", \"vh\"],\n",
-    "    output_crs=\"EPSG:32648\",\n",
-    "    resolution=(-10,10),\n",
-    "    dask_chunks={\"x\":2048, \"y\":2048},\n",
-    "    skip_broken_datasets=True,\n",
-    "    group_by=\"solar_day\"\n",
-    ")\n",
-    "notebook_utils.heading(notebook_utils.xarray_object_size(ds))\n",
-    "ds"
+    "# ds = dc.load(\n",
+    "#     product=\"sentinel1_grd_gamma0_20m\",\n",
+    "#     x=(105.5, 106.4),\n",
+    "#     y=(9.2, 10.0),\n",
+    "#     time=(\"2022-09-01\", \"2022-10-01\"),\n",
+    "#     measurements=[\"vv\", \"vh\"],\n",
+    "#     output_crs=\"EPSG:32648\",\n",
+    "#     resolution=(-10,10),\n",
+    "#     dask_chunks={\"x\":2048, \"y\":2048},\n",
+    "#     skip_broken_datasets=True,\n",
+    "#     group_by=\"solar_day\"\n",
+    "# )\n",
+    "# notebook_utils.heading(notebook_utils.xarray_object_size(ds))\n",
+    "# ds"
    ]
   },
   {
@@ -1905,11 +1905,11 @@
    },
    "outputs": [],
    "source": [
-    "vh = ds.vh.resample(time='1M').mean().persist()\n",
-    "vh = vh.compute()\n",
-    "vv = ds.vv.resample(time='1M').mean().persist()\n",
-    "vv = vv.compute()\n",
-    "\n"
+    "# vh = ds.vh.resample(time='1M').mean().persist()\n",
+    "# vh = vh.compute()\n",
+    "# vv = ds.vv.resample(time='1M').mean().persist()\n",
+    "# vv = vv.compute()\n",
+    "# \n"
    ]
   },
   {
@@ -2045,7 +2045,7 @@
        "\n",
        ".xr-section-summary-in + label:before {\n",
        "  display: inline-block;\n",
-       "  content: '►';\n",
+       "  content: '\u25ba';\n",
        "  font-size: 11px;\n",
        "  width: 15px;\n",
        "  text-align: center;\n",
@@ -2056,7 +2056,7 @@
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label:before {\n",
-       "  content: '▼';\n",
+       "  content: '\u25bc';\n",
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label > span {\n",
@@ -2365,4 +2365,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 5
-}
+}

.temp/test_VH_VV.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[2]:
+
+
+get_ipython().run_line_magic('matplotlib', 'inline')
+from new_import import *
+
+
+# Dask gateway
+cluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))
+dc = datacube.Datacube()
+
+
+# Configure s3 access
+configure_s3_access(aws_unsigned=False, requester_pays=True, client=client)
+
+
+
+
+# In[3]:
+
+
+ds = dc.load(
+    product="sentinel1_grd_gamma0_20m",
+    x=(105.5, 106.4),
+    y=(9.2, 10.0),
+    time=("2022-09-01", "2023-10-01"),
+    measurements=["vv", "vh"],
+    output_crs="EPSG:32648",
+    resolution=(-10,10),
+    dask_chunks={"x":2048, "y":2048},
+    skip_broken_datasets=True,
+    group_by="solar_day"
+)
+notebook_utils.heading(notebook_utils.xarray_object_size(ds))
+ds
+
+
+# In[18]:
+
+
+vh = ds.vh.resample(time='1M').mean().persist()
+vh = vh.compute()
+vv = ds.vv.resample(time='1M').mean().persist()
+vv = vv.compute()
+
+
+
+# In[28]:
+
+
+vv.min()
+
+
+# In[33]:
+
+
+import matplotlib.pyplot as plt
+
+# Plot the data
+plt.imshow(vh.isel(time=0), cmap='viridis', vmin=0, vmax=1)
+plt.colorbar()  # Add colorbar for reference
+plt.show()
+
+
+# In[ ]:
+
+
+
+

.temp/test_bug.py

@@ -0,0 +1,33 @@
+import torch
+import numpy as np
+
+device = "cpu"
+input_array = np.zeros((4, 16, 16), dtype=np.float32)
+input_tensor = torch.from_numpy(input_array).unsqueeze(0).to(device)
+
+print("Before pad:", input_tensor.shape)
+
+from train_cloud_removal import UNet
+model = UNet(in_channels=6, out_channels=4).to(device)
+
+if hasattr(model, 'inc') and hasattr(model.inc.double_conv[0], 'in_channels'):
+    expected_channels = model.inc.double_conv[0].in_channels
+elif hasattr(model, 'conv1') and hasattr(model.conv1, 'in_channels'):
+    expected_channels = model.conv1.in_channels
+else:
+    expected_channels = list(model.parameters())[0].shape[1]
+
+print("Expected channels:", expected_channels)
+
+if expected_channels > input_tensor.shape[1]:
+    pad_channels = expected_channels - input_tensor.shape[1]
+    padding = torch.zeros(1, pad_channels, *input_tensor.shape[2:]).to(device)
+    input_tensor = torch.cat([input_tensor, padding], dim=1)
+
+print("After pad:", input_tensor.shape)
+
+try:
+    model(input_tensor)
+    print("Success!")
+except Exception as e:
+    print("Error:", e)

.temp/test_caching.py

@@ -0,0 +1,25 @@
+import os
+import sys
+sys.path.insert(0, os.getcwd())
+import new_import_ODC
+import time
+import xarray as xr
+
+# Mock minimal params to test load_data
+date_range = ('2023-01-01', '2023-01-31')
+longtitude_range = (105.0, 105.1)
+latitude_range = (9.5, 9.6)
+
+print("--- First Call (Downloading & Caching) ---")
+start = time.time()
+data1 = new_import_ODC.load_data(None, date_range, longtitude_range, latitude_range)
+end = time.time()
+print(f"Time taken: {end - start:.2f}s")
+
+print("--- Second Call (Loading from Cache) ---")
+start = time.time()
+data2 = new_import_ODC.load_data(None, date_range, longtitude_range, latitude_range)
+end = time.time()
+print(f"Time taken: {end - start:.2f}s")
+
+print("✅ Test completed")

.temp/test_checkpoint.py

@@ -0,0 +1,7 @@
+import torch
+checkpoint = torch.load('cloud_removal_model/cloud_removal_unet_best.pth', map_location='cpu')
+print(checkpoint.keys())
+print("in_channels in checkpoint:", 'in_channels' in checkpoint)
+if 'in_channels' in checkpoint:
+    print(checkpoint['in_channels'])
+print("Shape of inc.double_conv.0.weight:", checkpoint['model_state_dict']['inc.double_conv.0.weight'].shape)

.temp/test_cloud.py

@@ -0,0 +1,4 @@
+import torch
+from cloud_removal import DeepInpaintingStrategy
+cloud_remover = DeepInpaintingStrategy()
+print("Model channels:", list(cloud_remover.model.parameters())[0].shape[1])

.temp/test_cloud2.py

@@ -0,0 +1,10 @@
+from cloud_removal import DeepInpaintingStrategy
+import torch
+import numpy as np
+
+cr = DeepInpaintingStrategy(model_path="cloud_removal_model/cloud_removal_unet_best.pth")
+if cr.model is not None:
+    expected = list(cr.model.parameters())[0].shape[1]
+    print("Expected channels:", expected)
+else:
+    print("Failed to load model")

.temp/test_empty_points.py

@@ -0,0 +1,53 @@
+import geopandas as gpd
+import planetary_computer
+import pystac_client
+import odc.stac
+import numpy as np
+from shapely.geometry import Point, shape
+from pyproj import Transformer
+
+bbox = [105.5, 9.2, 106.3, 10.0]
+time_range = "2023-01-01/2023-04-30"
+catalog = pystac_client.Client.open(
+    "https://planetarycomputer.microsoft.com/api/stac/v1",
+    modifier=planetary_computer.sign_inplace,
+)
+items = list(catalog.search(collections=["sentinel-2-l2a"], bbox=bbox, datetime=time_range, query={"eo:cloud_cover": {"lt": 30}}).items())
+items = sorted(items, key=lambda x: x.properties["eo:cloud_cover"])
+
+gdf = gpd.read_file("train/ST_training_data_updated_1130points_new.shp")
+gdf = gdf.to_crs("EPSG:32648")
+
+# Find a point that fails. Let's just test a few points.
+for idx, row in gdf.head(20).iterrows():
+    x_coord = row['geometry'].x
+    y_coord = row['geometry'].y
+    
+    transformer = Transformer.from_crs("epsg:32648", "epsg:4326", always_xy=True)
+    lon, lat = transformer.transform(x_coord, y_coord)
+    point = Point(lon, lat)
+    
+    filtered = []
+    for item in items:
+        if shape(item.geometry).contains(point):
+            filtered.append(item)
+            
+    filtered = [planetary_computer.sign(item) for item in filtered]
+    if not filtered:
+        print(f"Point {idx}: NO ITEMS CONTAINS POINT!")
+        continue
+        
+    ds = odc.stac.load(
+        filtered,
+        bands=["B02"],
+        x=(x_coord - 80, x_coord + 80),
+        y=(y_coord - 80, y_coord + 80),
+        crs="EPSG:32648",
+        resolution=10,
+        patch_url=planetary_computer.sign,
+        fail_on_error=False
+    ).compute()
+    
+    sums = ds["B02"].sum(dim=["x", "y"]).values
+    non_zero = (sums > 0).sum()
+    print(f"Point {idx}: {len(filtered)} items, {non_zero} non-zero time steps")

.temp/test_extract_one.py

@@ -0,0 +1,37 @@
+import geopandas as gpd
+import planetary_computer
+import pystac_client
+import odc.stac
+import sys
+
+bbox = [105.5, 9.2, 106.3, 10.0]
+time_range = "2023-01-01/2023-04-30"
+catalog = pystac_client.Client.open("https://planetarycomputer.microsoft.com/api/stac/v1", modifier=planetary_computer.sign_inplace)
+search = catalog.search(collections=["sentinel-2-l2a"], bbox=bbox, datetime=time_range, query={"eo:cloud_cover": {"lt": 30}})
+items = list(search.items())
+items = sorted(items, key=lambda x: x.properties.get("eo:cloud_cover", 100))[:4]
+items = [planetary_computer.sign(item) for item in items]
+
+gdf = gpd.read_file("train/ST_training_data_updated_1130points_new.shp")
+gdf = gdf.to_crs("EPSG:32648")
+
+row = gdf.iloc[0]
+x, y_coord = row.geometry.x, row.geometry.y
+point_bbox = [x - 80, y_coord - 80, x + 80, y_coord + 80]
+
+patch_s2 = odc.stac.load(
+    items,
+    bands=["B02", "B03", "B04", "B08", "SCL"],
+    x=(x - 80, x + 80),
+    y=(y_coord - 80, y_coord + 80),
+    crs="EPSG:32648",
+    resolution=10,
+    patch_url=planetary_computer.sign,
+    fail_on_error=False
+).compute()
+
+print("patch_s2 vars:", patch_s2.data_vars)
+if patch_s2.dims['x'] < 16 or patch_s2.dims['y'] < 16:
+    print("Too small:", patch_s2.dims)
+else:
+    print("Success dimension:", patch_s2.dims)

.temp/test_gdf.py

@@ -0,0 +1,3 @@
+import geopandas as gpd
+gdf = gpd.read_file("train/ST_training_data_updated_1130points_new.shp")
+print(gdf.head(1)['HT_code'])

.temp/test_gdf_cols.py

@@ -0,0 +1,3 @@
+import geopandas as gpd
+gdf = gpd.read_file("train/ST_training_data_updated_1130points_new.shp")
+print(gdf.columns)

.temp/test_gpu.py

@@ -0,0 +1,23 @@
+import numpy as np
+from xgboost import XGBClassifier
+from sklearn.datasets import make_classification
+from sklearn.metrics import accuracy_score
+
+print("🚀 Testing XGBoost with CUDA GPU...")
+try:
+    X, y = make_classification(n_samples=10000, n_features=20, n_classes=2, random_state=42)
+    model = XGBClassifier(
+        n_estimators=100,
+        max_depth=10,
+        tree_method="hist",
+        device="cuda",
+        random_state=42,
+        verbosity=1
+    )
+    print("Training model...")
+    model.fit(X, y)
+    y_pred = model.predict(X)
+    acc = accuracy_score(y, y_pred)
+    print(f"✅ Training successful! Accuracy: {acc*100:.2f}%")
+except Exception as e:
+    print(f"❌ Error during training: {e}")

.temp/test_keys.py

@@ -0,0 +1,3 @@
+import torch
+checkpoint = torch.load('cloud_removal_model/cloud_removal_unet_best.pth', map_location='cpu')
+print(list(checkpoint['model_state_dict'].keys())[:5])

.temp/test_model_struct.py

@@ -0,0 +1,13 @@
+import torch
+from pathlib import Path
+model = torch.load('cloud_removal_model/cloud_removal_unet_best.pth', map_location='cpu')
+print(type(model))
+print("hasattr inc:", hasattr(model, 'inc'))
+if hasattr(model, 'inc'):
+    print("hasattr double_conv:", hasattr(model.inc, 'double_conv'))
+    if hasattr(model.inc, 'double_conv'):
+        print("in_channels:", model.inc.double_conv[0].in_channels)
+else:
+    for name, param in model.named_parameters():
+        print(name, param.shape)
+        break

.temp/test_odc_stac.py

@@ -0,0 +1,19 @@
+import geopandas as gpd
+import planetary_computer
+import pystac_client
+import odc.stac
+
+bbox = [105.5, 9.2, 106.3, 10.0]
+time_range = "2023-01-01/2023-01-30"
+catalog = pystac_client.Client.open("https://planetarycomputer.microsoft.com/api/stac/v1", modifier=planetary_computer.sign_inplace)
+items = list(catalog.search(collections=["sentinel-2-l2a"], bbox=bbox, datetime=time_range).items())[:1]
+items = [planetary_computer.sign(item) for item in items]
+
+x = 561609
+y = 1024183
+try:
+    ds = odc.stac.load(items, bands=["B02"], crs="EPSG:32648", resolution=10, x=(x-80, x+80), y=(y-80, y+80))
+    print("Success with x/y:", ds.dims)
+except Exception as e:
+    print("Error with x/y:", e)
+

.temp/test_overlap.py

@@ -0,0 +1,18 @@
+import geopandas as gpd
+import planetary_computer
+import pystac_client
+import odc.stac
+import numpy as np
+
+bbox = [105.5, 9.2, 106.3, 10.0]
+time_range = "2023-01-01/2023-04-30"
+catalog = pystac_client.Client.open("https://planetarycomputer.microsoft.com/api/stac/v1", modifier=planetary_computer.sign_inplace)
+# Get ALL items
+items = list(catalog.search(collections=["sentinel-2-l2a"], bbox=bbox, datetime=time_range, query={"eo:cloud_cover": {"lt": 30}}).items())
+items = [planetary_computer.sign(item) for item in items]
+print(f"Total items: {len(items)}")
+
+x = 561609
+y = 1024183
+ds = odc.stac.load(items, bands=["B02"], x=(x-80, x+80), y=(y-80, y+80), crs="EPSG:32648", resolution=10, patch_url=planetary_computer.sign).compute()
+print("Time dimension size:", ds.dims['time'])

.temp/test_overlap2.py

@@ -0,0 +1,20 @@
+import geopandas as gpd
+import planetary_computer
+import pystac_client
+import odc.stac
+import numpy as np
+
+bbox = [105.5, 9.2, 106.3, 10.0]
+time_range = "2023-01-01/2023-04-30"
+catalog = pystac_client.Client.open("https://planetarycomputer.microsoft.com/api/stac/v1", modifier=planetary_computer.sign_inplace)
+items = list(catalog.search(collections=["sentinel-2-l2a"], bbox=bbox, datetime=time_range, query={"eo:cloud_cover": {"lt": 30}}).items())
+items = [planetary_computer.sign(item) for item in items]
+
+x = 561609
+y = 1024183
+ds = odc.stac.load(items, bands=["B02"], x=(x-80, x+80), y=(y-80, y+80), crs="EPSG:32648", resolution=10, patch_url=planetary_computer.sign, fail_on_error=False).compute()
+print("Original time size:", len(ds.time))
+ds2 = ds.dropna(dim="time", how="all")
+print("After dropna time size:", len(ds2.time))
+print("B02 mean:", np.nanmean(ds2["B02"].values))
+print("B02 non-nan count:", np.sum(~np.isnan(ds2["B02"].values)))

.temp/test_overlap_issue.py

@@ -0,0 +1,25 @@
+import geopandas as gpd
+import planetary_computer
+import pystac_client
+import odc.stac
+import numpy as np
+
+bbox = [105.5, 9.2, 106.3, 10.0]
+time_range = "2023-01-01/2023-04-30"
+catalog = pystac_client.Client.open("https://planetarycomputer.microsoft.com/api/stac/v1", modifier=planetary_computer.sign_inplace)
+items = list(catalog.search(collections=["sentinel-2-l2a"], bbox=bbox, datetime=time_range, query={"eo:cloud_cover": {"lt": 30}}).items())
+items = [planetary_computer.sign(item) for item in items]
+
+x = 561609
+y = 1024183
+ds = odc.stac.load(items, bands=["B02", "B03", "B04", "B08", "SCL"], x=(x-80, x+80), y=(y-80, y+80), crs="EPSG:32648", resolution=10, patch_url=planetary_computer.sign, fail_on_error=False).compute()
+
+print("Original shape:", ds["B02"].shape)
+ds2 = ds.dropna(dim="time", how="all")
+print("After dropna time size:", len(ds2.time))
+if len(ds2.time) > 0:
+    ds2 = ds2.isel(time=slice(0, 4))
+    median = ds2["B02"].median(dim="time", skipna=True).values
+    print("Median shape:", median.shape)
+    print("Zeros in median:", np.sum(median == 0) / median.size)
+    print("NaNs in median:", np.sum(np.isnan(median)) / median.size)

.temp/test_planetary.py

@@ -0,0 +1,15 @@
+import sys
+# Thêm đường dẫn hiện tại vào PYTHONPATH để import được new_import_ODC nếu cần
+sys.path.append('.')
+import warnings
+warnings.filterwarnings('ignore')
+from new_import_ODC import load_sen1
+
+print("Testing load_sen1 with a short time range to speed up Dask compute...")
+bbox = [105.5, 9.2, 106.4, 10.0]
+time_range = "2023-01-01/2023-01-31" # Short time range for fast testing
+vh, vv = load_sen1(bbox, time_range)
+print("VH shape:", vh.shape)
+print("VV shape:", vv.shape)
+print("VH CRS:", vh.rio.crs)
+print("Success!")

.temp/test_planetary2.py

@@ -0,0 +1,49 @@
+import warnings
+warnings.filterwarnings('ignore')
+
+def load_sen1(bbox, time_range):
+    import pystac_client
+    import planetary_computer
+    import odc.stac
+    
+    catalog = pystac_client.Client.open(
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
+        modifier=planetary_computer.sign_inplace,
+    )
+
+    search = catalog.search(
+        collections=["sentinel-1-rtc"],
+        bbox=bbox,
+        datetime=time_range,
+    )
+    items = list(search.items())
+    print("Found items:", len(items))
+
+    ds_s1 = odc.stac.load(
+        items,
+        bands=["vv", "vh"],
+        bbox=bbox,
+        crs="EPSG:32648",
+        resolution=10,
+        chunks={"x": 2048, "y": 2048, "time": 1}
+    )
+
+    ds_median = ds_s1.median(dim="time").compute()
+    vv = ds_median["vv"]
+    vh = ds_median["vh"]
+
+    vv = vv.expand_dims(dim="band")
+    vh = vh.expand_dims(dim="band")
+
+    vv = vv.rio.write_crs("EPSG:32648")
+    vh = vh.rio.write_crs("EPSG:32648")
+
+    return vh, vv
+
+print("Testing load_sen1...")
+bbox = [105.5, 9.2, 106.4, 10.0]
+time_range = "2022-09-01/2023-10-01"
+vh, vv = load_sen1(bbox, time_range)
+print("VH shape:", vh.shape)
+print("VV shape:", vv.shape)
+print("Success!")

.temp/test_raw.py

@@ -0,0 +1,17 @@
+import geopandas as gpd
+import planetary_computer
+import pystac_client
+import odc.stac
+import numpy as np
+
+bbox = [105.5, 9.2, 106.3, 10.0]
+time_range = "2023-01-01/2023-04-30"
+catalog = pystac_client.Client.open("https://planetarycomputer.microsoft.com/api/stac/v1", modifier=planetary_computer.sign_inplace)
+items = list(catalog.search(collections=["sentinel-2-l2a"], bbox=bbox, datetime=time_range, query={"eo:cloud_cover": {"lt": 30}}).items())[:4]
+items = [planetary_computer.sign(item) for item in items]
+
+x = 561609
+y = 1024183
+ds = odc.stac.load(items, bands=["B02", "B03", "B04", "B08"], x=(x-80, x+80), y=(y-80, y+80), crs="EPSG:32648", resolution=10, patch_url=planetary_computer.sign).compute()
+print("B04 nanmean:", np.nanmean(ds["B04"].values))
+print("B04 nanmax:", np.nanmax(ds["B04"].values))

.temp/test_shape.py

@@ -0,0 +1,41 @@
+import new_import_ODC
+importlib = __import__('importlib')
+importlib.reload(new_import_ODC)
+from new_import_ODC import *
+import numpy as np
+
+date_range = ("2022-09-01", "2022-10-01")
+longtitude_range = (105.86, 105.94)
+latitude_range = (9.65, 9.69)
+coordinates = (longtitude_range, latitude_range)
+
+print("Loading S2...")
+data = load_data(None, date_range, longtitude_range, latitude_range)
+result = mask_clean(data)
+ds1 = calculate_indices(result, index="NDVI", satellite_mission="s2")
+ndvi = ds1["NDVI"]
+time_split = [
+    slice("2022-09-01", "2023-01-01"),
+    slice("2023-01-01", "2023-05-01"),
+    slice("2023-05-01", "2023-07-01"),
+    slice("2023-07-01", "2022-10-01"),
+]
+fill_nan_ndvi = fill_nan(ndvi, time_split)
+average_ndvi = fill_nan_ndvi.resample(time="1M").mean().compute()
+
+print("Loading S1...")
+dsvh, dsvv = load_data_sen1(None, date_range, coordinates)
+average_vv = calculate_average(dsvv, time_pattern='1M')
+average_vh = calculate_average(dsvh, time_pattern='1M')
+
+train = load_train_data("train/ST_training_data_updated_1130points_new.shp")
+point = train.iloc[0]
+
+ndvi_val = average_ndvi.sel(x=point.geometry.x, y=point.geometry.y, method='nearest').values
+vh_val = average_vh.sel(x=point.geometry.x, y=point.geometry.y, method='nearest').values
+vv_val = average_vv.sel(x=point.geometry.x, y=point.geometry.y, method='nearest').values
+
+print("NDVI shape:", ndvi_val.shape, "ndim:", ndvi_val.ndim)
+print("VH shape:", vh_val.shape, "ndim:", vh_val.ndim)
+print("VV shape:", vv_val.shape, "ndim:", vv_val.ndim)
+

.temp/test_spatial_filter.py

@@ -0,0 +1,42 @@
+import geopandas as gpd
+import planetary_computer
+import pystac_client
+import odc.stac
+import numpy as np
+import time
+from shapely.geometry import Point, box, shape
+
+bbox = [105.5, 9.2, 106.3, 10.0]
+time_range = "2023-01-01/2023-04-30"
+catalog = pystac_client.Client.open("https://planetarycomputer.microsoft.com/api/stac/v1", modifier=planetary_computer.sign_inplace)
+items = list(catalog.search(collections=["sentinel-2-l2a"], bbox=bbox, datetime=time_range, query={"eo:cloud_cover": {"lt": 30}}).items())
+
+x = 561609
+y = 1024183
+
+start = time.time()
+# Filter items by spatial intersection
+from pyproj import Transformer
+# The items geometry are in EPSG:4326 (lon, lat)
+# Our x, y are in EPSG:32648
+transformer = Transformer.from_crs("epsg:32648", "epsg:4326", always_xy=True)
+lon, lat = transformer.transform(x, y)
+point = Point(lon, lat)
+
+filtered_items = []
+for item in items:
+    geom = shape(item.geometry)
+    if geom.contains(point):
+        filtered_items.append(item)
+
+filtered_items = sorted(filtered_items, key=lambda x: x.properties["eo:cloud_cover"])
+
+print("Original items:", len(items))
+print("Filtered items:", len(filtered_items))
+print("Time to filter:", time.time() - start)
+
+start = time.time()
+filtered_items = [planetary_computer.sign(item) for item in filtered_items]
+ds = odc.stac.load(filtered_items[:4], bands=["B02"], x=(x-80, x+80), y=(y-80, y+80), crs="EPSG:32648", resolution=10, patch_url=planetary_computer.sign, fail_on_error=False).compute()
+print("Time to load 4 items:", time.time() - start)
+print(ds["B02"].shape)

.temp/test_unet.py

@@ -0,0 +1,5 @@
+from train_cloud_removal import UNet
+model = UNet(in_channels=6, out_channels=4)
+print(hasattr(model, 'inc'))
+print(hasattr(model, 'conv1'))
+print(list(model.parameters())[0].shape)

.temp/test_y_mapped.py

@@ -0,0 +1,13 @@
+import numpy as np
+
+y = []
+# simulate appending 1130 labels
+for i in range(1130):
+    y.append(i % 5)
+
+y = np.array(y)
+unique_labels = sorted(list(np.unique(y)))
+label_map = {lbl: i for i, lbl in enumerate(unique_labels)}
+y_mapped = np.array([label_map[l] for l in y])
+
+print(len(y), len(y_mapped))

api_server.py

@@ -26,20 +26,20 @@ import socket
 import urllib.request
 
 # Import report generator
-from report_generator import generate_training_report, generate_prediction_report
+from scripts.inference.report_generator import generate_training_report, generate_prediction_report
 
 # Import Model Manager
-from model_manager import ModelManager, get_model_manager
+from core.model_manager import ModelManager, get_model_manager
 
 # Import Vietnam provinces data
-from vietnam_provinces import get_all_provinces, get_provinces_by_region, get_province_bbox, search_province
-from vietnam_provinces_merged import (
+from core.vietnam_provinces import get_all_provinces, get_provinces_by_region, get_province_bbox, search_province
+from core.vietnam_provinces_merged import (
     get_all_provinces_32, get_provinces_by_region_32, get_province_bbox_32, 
     search_province_32, get_merged_info, get_provinces_statistics
 )
 
 # Import cloud removal module
-from cloud_removal import process_cloud_removal, get_available_methods
+from core.cloud_removal import process_cloud_removal, get_available_methods
 
 # Import planetary computer libraries (conditional)
 try:
@@ -507,6 +507,57 @@ async def get_cloud_removal_methods():
     }
 
 
+@app.get("/api/ndvi-forecast/models")
+async def list_ndvi_forecast_models():
+    """Liệt kê các NDVI forecast models đã train"""
+    model_dir = Path("ndvi_forecast_model")
+    if not model_dir.exists():
+        return {"models": [], "count": 0}
+    
+    models = []
+    # Search for all models
+    for model_file in list(model_dir.rglob("*.pth")) + list(model_dir.rglob("*.joblib")):
+        try:
+            import json
+            
+            # Try to load metadata from .json sidecar file first
+            metadata_file = model_file.with_name(model_file.stem + "_info.json")
+            if metadata_file.exists():
+                try:
+                    with open(metadata_file, 'r') as f:
+                        metadata = json.load(f)
+                    
+                    models.append({
+                        "filename": model_file.name,
+                        "path": str(model_file),
+                        "model_type": metadata.get('model_type', 'Unknown'),
+                        "target": metadata.get('target', 'NDVI'),
+                        "rmse": metadata.get('rmse', 0),
+                        "mae": metadata.get('mae', 0),
+                        "epoch": metadata.get('epoch', 0),
+                        "created": model_file.stat().st_mtime,
+                        "size_mb": model_file.stat().st_size / (1024 * 1024),
+                    })
+                    continue
+                except Exception as e:
+                    print(f"Error reading JSON {metadata_file}: {e}")
+                    
+            # Fallback for models without metadata
+            models.append({
+                "filename": model_file.name,
+                "path": str(model_file),
+                "model_type": "Unknown",
+                "created": model_file.stat().st_mtime,
+                "size_mb": model_file.stat().st_size / (1024 * 1024)
+            })
+        except Exception as e:
+            print(f"Error loading model info for {model_file}: {e}")
+            
+    # Sort by creation time (newest first)
+    models.sort(key=lambda x: x['created'], reverse=True)
+    return {"models": models, "count": len(models)}
+
+
 @app.get("/api/cloud-removal/models")
 async def list_cloud_removal_models():
     """Liệt kê các cloud removal models đã train"""

backup_S3_download_Amazon/new_import_S3.py

@@ -1,354 +0,0 @@
-import matplotlib.pyplot as plt
-
-# Common imports and settings
-import os, sys
-os.environ['USE_PYGEOS'] = '0'
-from IPython.display import Markdown
-import pandas as pd
-pd.set_option("display.max_rows", None)
-import xarray as xr
-
-# Datacube
-import datacube
-from datacube.utils.rio import configure_s3_access
-from datacube.utils import masking
-from datacube.utils.cog import write_cog
-# https://github.com/GeoscienceAustralia/dea-notebooks/tree/develop/Tools
-from dea_tools.plotting import display_map, rgb
-from dea_tools.datahandling import mostcommon_crs
-
-# EASI defaults
-easinotebooksrepo = '/home/jovyan/easi-notebooks'
-if easinotebooksrepo not in sys.path: sys.path.append(easinotebooksrepo)
-from easi_tools import EasiDefaults, xarray_object_size, notebook_utils, unset_cachingproxy
-from easi_tools.load_s2l2a import load_s2l2a_with_offset
-from dask.distributed import progress
-
-# Data tools
-import numpy as np
-from datetime import datetime
-
-# Datacube
-from datacube.utils import masking  # https://github.com/opendatacube/datacube-core/blob/develop/datacube/utils/masking.py
-from odc.algo import enum_to_bool   # https://github.com/opendatacube/odc-algo/blob/main/odc/algo/_masking.py
-from odc.algo import xr_reproject   # https://github.com/opendatacube/odc-algo/blob/main/odc/algo/_warp.py
-from datacube.utils.geometry import GeoBox, box  # https://github.com/opendatacube/datacube-core/blob/develop/datacube/utils/geometry/_base.py
-
-# Holoviews, Datashader and Bokeh
-import hvplot.pandas
-import hvplot.xarray
-import holoviews as hv
-import panel as pn
-import colorcet as cc
-import cartopy.crs as ccrs
-from datashader import reductions
-from holoviews import opts
-from utils import load_data_geo
-import rasterio
-import rioxarray
-# import geoviews as gv
-# from holoviews.operation.datashader import rasterize
-hv.extension('bokeh', logo=False)
-
-from deafrica_tools.bandindices import calculate_indices
-from sklearn.ensemble import RandomForestClassifier
-from sklearn.model_selection import train_test_split
-from sklearn.metrics import accuracy_score, classification_report
-from sklearn.preprocessing import LabelEncoder
-
-from sklearn.pipeline import Pipeline
-from sklearn.ensemble import RandomForestClassifier
-from sklearn.impute import SimpleImputer
-from sklearn.preprocessing import StandardScaler
-from sklearn.model_selection import GridSearchCV
-from sklearn.model_selection import train_test_split
-from sklearn.metrics import accuracy_score
-from shapely.geometry import Point, Polygon
-import geopandas as gpd
-from pyproj import CRS
-from matplotlib.colors import ListedColormap
-from holoviews import opts
-from datashader import reductions
-from bokeh.models.tickers import FixedTicker
-from rioxarray.merge import merge_arrays
-
-import joblib
-
-
-def load_data(dc, date_range, longtitude_range, latitude_range):
-    product = 's2_l2a'
-    query = {
-        'product': product,                     # Product name
-        'x': longtitude_range,    # "x" axis bounds
-        'y': latitude_range,      # "y" axis bounds
-        'time': date_range,           # Any parsable date strings
-    }
-    native_crs = notebook_utils.mostcommon_crs(dc, query)
-    print(f'Most common native CRS: {native_crs}')
-    measurements = [ 'red', 'nir', 'scl']
-
-    load_params = {
-        'measurements': measurements,                   # Selected measurement or alias names
-        'output_crs': native_crs,                       # Target EPSG code
-        'resolution': (-10, 10),                        # Target resolution
-        'group_by': 'solar_day',                        # Scene grouping
-        'dask_chunks': {'x': 2048, 'y': 2048},          # Dask chunks
-    }
-    data = load_s2l2a_with_offset(
-        dc,
-        query | load_params   # Combine the two dicts that contain our search and load parameters
-    )
-    return data
-
-
-def mask_clean(data):
-    flag_name = 'scl'
-    flag_desc = masking.describe_variable_flags(data[flag_name])  # Pandas dataframe
-    display(flag_desc)
-    display(flag_desc.loc['qa'].values[1])
-    # Create a "data quality" Mask layer
-    flags_def = flag_desc.loc['qa'].values[1]
-    good_pixel_flags = [flags_def[str(i)] for i in [2, 4, 5, 6]]  # To pass strings to enum_to_bool()
-
-    # enum_to_bool calculates the pixel-wise "or" of each set of pixels given by good_pixel_flags
-    # 1 = good data
-    # 0 = "bad" data
-    good_pixel_mask = enum_to_bool(data[flag_name], good_pixel_flags)
-    data_layer_names = [x for x in data.data_vars if x != 'scl']
-    # Apply good pixel mask to blue, green, red and nir.
-    result = data[data_layer_names].where(good_pixel_mask).persist()
-    return result
-
-
-def fill_nan(ndvi, time_split):
-    rs = []
-    for times in time_split:
-        tmp = ndvi.sel(time=times)
-        fill_ds = tmp.sel(time=times).bfill(dim='time')
-        fill_ds = fill_ds.sel(time=times).ffill(dim='time')   
-        rs.append(fill_ds)
-    merged_ndvi = xr.concat([i for i in rs], dim="time")
-    fill_m = merged_ndvi.bfill(dim="time")
-    fill_m = fill_m.ffill(dim="time")
-    return fill_m
-
-
-def load_train_data(train_path):
-    train = load_data_geo(train_path)
-    return train
-
-
-def load_sen1(name_vh, name_vv):
-    dsvv = rioxarray.open_rasterio(name_vv)
-    dsvh = rioxarray.open_rasterio(name_vh)
-    return dsvh, dsvv
-
-
-def get_data_sen1_and_sen2(train, average_ndvi, dsvh, dsvv):
-    loaded_datasets = {}
-    for idx, point in train.iterrows():
-        key = f"point_{idx + 1}"
-        try:
-            ndvi_data = average_ndvi.sel(x=point.geometry.x, y=point.geometry.y, method='nearest').values
-            vh_data = dsvh.sel(x=point.geometry.x, y=point.geometry.y, method='nearest').values
-            vv_data = dsvv.sel(x=point.geometry.x, y=point.geometry.y, method='nearest').values
-            loaded_datasets[key] = {
-                "data": np.concatenate((ndvi_data, vh_data, vv_data)),
-                "label": point.HT_code
-                                   }
-        except Exception as e:
-            # loaded_datasets[key] = None
-            print(e)
-    return loaded_datasets
-
-
-def split_train_data(train, label_mapping, datasets):
-    label_encoder = LabelEncoder()
-
-    # Fit and transform the labels
-    labels = train.Hientrang.values
-    numeric_labels = label_encoder.fit_transform([label_mapping[label] for label in labels])
-    X = []
-    x_new = []
-    lb_new = []
-    for k, v in datasets.items():
-        X.append(v)
-    for i in range(len(X)):
-        if X[i] is not None:
-            x_new.append(X[i]["data"])
-            lb_new.append(numeric_labels[i])
-    X_train, X_temp, y_train, y_temp= train_test_split(x_new, lb_new, test_size=0.4, random_state=42)
-    X_val, X_test, y_val, y_test = train_test_split(X_temp, y_temp, test_size=0.5, random_state=42)
-    return X_train, X_val, X_test, y_train, y_val, y_test
-
-
-def train_with_rf(X_train, X_val, y_train, y_val):
-    # Takes 1-2 minutes to complete
-
-    # Tạo RandomForestClassifier mặc định để sử dụng làm mô hình ban đầu trong pipeline
-    base_model = RandomForestClassifier(random_state=42, n_jobs=-1)
-
-    # Tạo pipeline
-    pipeline = Pipeline([
-        # ('imputer', SimpleImputer(strategy='mean')),
-        ('scaler', StandardScaler()),
-        ('classifier', base_model),
-    ])
-    # Thiết lập các tham số bạn muốn tối ưu hóa
-    param_grid = {
-        'classifier__n_estimators': [100, 300, 500, 700, 1000],
-        'classifier__max_depth': [6, 8, 10, 15, 20],
-        'classifier__criterion': ['gini', 'entropy'],
-    }
-
-    # Sử dụng GridSearchCV để tìm bộ tham số tốt nhất
-    grid_search = GridSearchCV(pipeline, param_grid, cv=5, scoring='accuracy', n_jobs=-1)
-    grid_search.fit(X_train, y_train)
-
-    # In ra bộ tham số tốt nhất
-    best_params = grid_search.best_params_
-    print("Best Parameters:", best_params)
-
-    # Dự đoán trên tập kiểm tra
-    y_pred = grid_search.predict(X_val)
-
-    # Đánh giá kết quả
-    accuracy = accuracy_score(y_val, y_pred)
-    print(f"Accuracy: {round(accuracy, 2)*100} %")
-    return grid_search
-
-
-def save_model(name_file, grid_search):
-    dir_save_model = "model_train"
-    if not os.path.exists(dir_save_model):
-        os.mkdir(dir_save_model)
-    joblib.dump(grid_search, os.path.join(dir_save_model, name_file))
-    print("Done!")
-    
-    
-def predict(model, data_crs, ndvi, vh, vv):
-    data_predict = []
-    for i in range(ndvi.shape[1]):
-        ndvi_tmp = ndvi.isel(y=i).values
-        vh_data = vh.sel(y=ndvi.y.values[i], method='nearest').values
-        vv_data = vv.sel(y=ndvi.y.values[i], method='nearest').values
-        all_tmp = np.concatenate((ndvi_tmp, vh_data, vv_data), axis=0)
-        data_predict.extend(all_tmp.T)
-    y_pred = model.predict(data_predict)
-    final_label = y_pred.reshape(ndvi.y.shape[0], ndvi.x.shape[0])
-    
-    final_xarray_save = xr.DataArray(final_label, dims=("y", "x"))
-    final_xarray_save = final_xarray_save.rio.write_crs(data_crs)
-
-    x_values = ndvi.x.values
-    y_values = ndvi.y.values
-
-    data_array = xr.DataArray(final_xarray_save,
-                              coords={'x': x_values, 'y': y_values},
-                              dims=['y', 'x'])
-    data_array = data_array.rio.write_crs(ndvi.rio.crs)
-    return data_array
-
-
-def cut_according_shp(thuanhoa_path, average_ndvi, data_array):
-    gdf = gpd.read_file(thuanhoa_path)
-    gdf = gdf.to_crs(average_ndvi.rio.crs)
-    polygon_coords = list(gdf.geometry.values[0].exterior.coords)
-    polygon_coordinates = [(x, y) for x, y in polygon_coords]
-
-    geometries = [
-        {
-            'type': 'Polygon',
-            'coordinates': [polygon_coordinates]
-        }
-    ]
-    region_result = data_array.rio.clip(geometries, data_array.rio.crs, drop=False)
-    region_result = region_result.where(region_result >= 0, float('nan'))
-    return region_result
-
-
-def compare(KD_path, KetQuaPhanLoaiDat, CODE_MAP, HT_MAP):
-    gdf = gpd.read_file(KD_path, crs="EPSG:9209")
-    polygon = gdf.geometry.values
-    label = gdf.tenchu.values
-    ouput_image = rioxarray.open_rasterio(KetQuaPhanLoaiDat)
-    code_tq = HT_MAP["TQ"]["data"][0]
-    code_pnn = HT_MAP["PNN"]["data"][0]
-    result = {}
-    for key, values in HT_MAP.items():
-        print(f"process {key}")
-        array_list = []
-        for i in range(len(polygon)):
-            po = polygon[i]
-            lb = label[i]
-            code_lb = CODE_MAP.get(lb, code_tq)
-            try:
-                qr = ouput_image.rio.clip([po], "EPSG:9209")
-                if code_lb in values["data"]:
-                    if code_lb == code_pnn:
-                        qr = qr.where((qr != float(code_pnn)), np.nan)
-                        # qr = qr.where((qr != 3.0), np.nan)
-                    elif code_lb == code_tq:
-                        qr = qr.where((qr != float(code_pnn)), np.nan)
-                        qr = qr.where((qr != 3.0), np.nan)
-                    else: 
-                        qr = qr.where(qr != float(code_lb), np.nan)
-                else:
-                    qr.values[:, :, :] = np.nan
-                array_list.append(qr)
-            except Exception as e:
-               pass
-        result.update({key: array_list})
-    return result
-
-
-def save_result(result, HT_MAP):
-    # cmap = ListedColormap(colors)
-    save_path = "ThuanHoa/KetQua"
-    if not os.path.exists(save_path):
-        os.mkdir(save_path)
-
-    for k, v in result.items():
-        rs = merge_arrays(v, nodata = np.nan)
-        rs.rio.to_raster(f"{save_path}/{k}.tif")
-        print(f"save {save_path}/{k}.tif")
-        # img = rs.plot(cmap=cmap, add_colorbar=False)
-        # cbar = plt.colorbar(img)
-        # cbar.ax.set_yticklabels(labels)
-        # plt.title(f'{HT_MAP[k]["name"]}')
-        # plt.axis('off')
-        # plt.show()
-        
-
-def accuracy_test(test, data_array):
-    # cấu hình nhãn dữ liệu
-    label_mapping = {
-        "Lua tom": "0",
-        "Lua": "1",
-        "CHN": "2",
-        "CLN": "3",
-        "TS": "4",
-        "Song": "5",
-        "Dat xay dung": "6",
-        "Rung": "7"
-    }
-
-    chk = []
-    pred = []
-    dd = []
-    for idx, point in test.iterrows():
-        label = point.LULC
-        predict = data_array.sel(x=point.geometry.x, y=point.geometry.y, method='nearest').values
-        pred.append(label_mapping[label])
-        dd.append(str(predict))
-        chk.append(predict == int(label_mapping[label]))
-    test["code"] = pred
-    test["dd"] = dd
-    test["check"] = chk
-    path = "ThuanHoa/TestAccuracy"
-    if not os.path.exists(path):
-        os.mkdir(path)
-    test.to_file(f"{path}/result.shp")
-    
-    percentage_true = np.mean(chk) * 100
-    print(f"độ chính xác: {percentage_true:.2f}%")