refactor: reorganize project structure by moving core modules and update import paths in API server

.temp/check_X.py

@@ -0,0 +1,18 @@
+import joblib
+import numpy as np
+
+cache_file = "dataset_cache/training_data_2d.joblib"
+data = joblib.load(cache_file)
+X = np.array(data['X'])
+y = np.array(data['y'])
+
+print("X shape:", X.shape)
+print("X mean:", np.mean(X))
+print("X std:", np.std(X))
+print("X min:", np.min(X))
+print("X max:", np.max(X))
+print("Any NaN:", np.isnan(X).any())
+
+for i in range(6):
+    print(f"Channel {i} mean: {np.mean(X[:, i, :, :]):.4f}, min: {np.min(X[:, i, :, :]):.4f}, max: {np.max(X[:, i, :, :]):.4f}")
+

.temp/check_missing.py

@@ -0,0 +1,10 @@
+import joblib
+import geopandas as gpd
+from shapely.geometry import Point
+
+data = joblib.load('dataset_cache/training_data_2d.joblib')
+X, y = data['X'], data['y']
+print(f"X shape: {X.shape}, y shape: {y.shape}")
+
+gdf = gpd.read_file("train/ST_training_data_updated_1130points_new.shp")
+print("Total points:", len(gdf))

.temp/check_xgboost_data.py

@@ -0,0 +1,7 @@
+import joblib
+import numpy as np
+
+data = joblib.load('dataset_cache/training_data.joblib')
+X, y = data['X'], data['y']
+print(f"X shape: {X.shape}")
+print(f"y shape: {y.shape}")

.temp/check_zeros.py

@@ -0,0 +1,8 @@
+import joblib
+import numpy as np
+cache_file = "dataset_cache/training_data_2d.joblib"
+data = joblib.load(cache_file)
+X = np.array(data['X'])
+b2 = X[:, 0, :, :]
+print("Zeros in B2:", np.sum(b2 == 0) / b2.size)
+print("X shape:", X.shape)

.temp/fix_cache.py

@@ -0,0 +1,19 @@
+import joblib
+import geopandas as gpd
+import numpy as np
+
+cache_file = "dataset_cache/training_data_2d.joblib"
+data = joblib.load(cache_file)
+X = data['X']
+print("X shape:", len(X))
+
+gdf = gpd.read_file("train/ST_training_data_updated_1130points_new.shp")
+gdf = gdf.to_crs("EPSG:32648")
+print("gdf length:", len(gdf))
+
+if len(X) == len(gdf):
+    y = [(row['HT_code'] - 1) for idx, row in gdf.iterrows()]
+    joblib.dump({'X': X, 'y': y}, cache_file)
+    print("Fixed y in cache! Saved.")
+else:
+    print("Lengths do not match, cannot fix automatically.")

.temp/fix_train.py

@@ -0,0 +1,20 @@
+import re
+
+filepath = "01.train_ODC.py"
+with open(filepath, 'r', encoding='utf-8') as f:
+    content = f.read()
+
+# Find the run_cell_magic line
+pattern = re.compile(r"get_ipython\(\)\.run_cell_magic\('time', '', '(# 🤖 LAND USE CLASSIFICATION MODEL TRAINING.*?)(?=\n')\n'", re.DOTALL)
+
+def repl(match):
+    # Get the inner string and escape all actual newlines with \n
+    inner = match.group(1)
+    inner = inner.replace('\n', '\\n')
+    return f"get_ipython().run_cell_magic('time', '', '{inner}')"
+
+content = pattern.sub(repl, content)
+
+with open(filepath, 'w', encoding='utf-8') as f:
+    f.write(content)
+print("Fixed 01.train_ODC.py syntax")

.temp/generate_colab_notebook.py

@@ -0,0 +1,58 @@
+import nbformat as nbf
+
+nb = nbf.v4.new_notebook()
+
+text_1 = """# Script Tải Dữ liệu Vệ tinh (Cache) qua Google Colab
+Mục đích của Notebook này là mượn sức mạnh đường truyền và RAM của Google Colab để tải 270 ảnh Sentinel-2 & Sentinel-1 từ Microsoft Planetary Computer. Sau khi xử lý nội suy, nó sẽ sinh ra một file cache `.joblib` duy nhất chứa toàn bộ mảng dữ liệu. 
+Bạn chỉ cần tải file `.joblib` đó về máy là xong!"""
+
+code_1 = """!pip install planetary-computer pystac-client odc-stac geopandas rasterio xarray joblib scikit-learn xgboost lightgbm"""
+
+code_2 = """from google.colab import drive
+drive.mount('/content/drive')"""
+
+text_2 = """## Hướng dẫn:
+1. Nén toàn bộ thư mục `remote-sensing` ở máy tính của bạn thành file `remote-sensing.zip`.
+2. Upload file `remote-sensing.zip` đó lên Google Drive (để ngay ngoài cùng).
+3. Chạy ô lệnh bên dưới để giải nén và chuyển vào thư mục dự án."""
+
+code_3 = """import os
+import shutil
+
+# Giải nén dự án từ Google Drive
+!unzip -q /content/drive/MyDrive/remote-sensing.zip -d /content/
+os.chdir('/content/remote-sensing')
+!ls -la"""
+
+text_3 = """## Bắt đầu tải và Cache Dữ Liệu
+Chạy một mô hình CPU đơn giản (Decision Tree) để ép hệ thống gọi hàm `FeatureExtractor`. Hàm này sẽ làm mọi việc nặng nhọc: tìm ảnh, ghép mây, tính trung vị và lưu kết quả vào thư mục `dataset_cache/`."""
+
+code_4 = """# Lệnh này sẽ mất khoảng 5-15 phút để tải toàn bộ ảnh từ Microsoft
+!python train_land_decision_tree_gpu.py"""
+
+text_4 = """## Hoàn tất
+Bạn hãy kiểm tra xem file `.joblib` lớn (khoảng 40-60MB) đã xuất hiện chưa. Nếu rồi, hãy lưu ngược nó lại Google Drive để tải về máy!"""
+
+code_5 = """# Xem file cache đã được tạo thành công chưa
+!ls -lh dataset_cache/
+
+# Copy toàn bộ thư mục cache sang Google Drive để tải về máy dễ dàng
+!cp -r dataset_cache/ /content/drive/MyDrive/dataset_cache_finished/
+print("Hoàn thành! Bạn hãy mở Google Drive của mình, tìm thư mục 'dataset_cache_finished' và tải file .joblib mới nhất về máy tính.")"""
+
+nb['cells'] = [
+    nbf.v4.new_markdown_cell(text_1),
+    nbf.v4.new_code_cell(code_1),
+    nbf.v4.new_code_cell(code_2),
+    nbf.v4.new_markdown_cell(text_2),
+    nbf.v4.new_code_cell(code_3),
+    nbf.v4.new_markdown_cell(text_3),
+    nbf.v4.new_code_cell(code_4),
+    nbf.v4.new_markdown_cell(text_4),
+    nbf.v4.new_code_cell(code_5)
+]
+
+with open('Download_Cache_Colab.ipynb', 'w') as f:
+    nbf.write(nb, f)
+
+print("Created Download_Cache_Colab.ipynb")

.temp/generate_report_only.py

@@ -0,0 +1,97 @@
+import os
+import glob
+import json
+from tabulate import tabulate
+
+print("📊 BẢNG SO SÁNH KẾT QUẢ CÁC MÔ HÌNH\n")
+
+# 1. Phân loại đất
+print("### 1. Nhóm Phân loại Lớp phủ (Land Classification)")
+land_data = []
+
+if os.path.exists("model_xgboost_info.json"):
+    with open("model_xgboost_info.json", 'r') as f:
+        data = json.load(f)
+        params = data.get('params', {})
+        param_str = f"estimators:{params.get('n_estimators')}, depth:{params.get('max_depth')}" if params else "N/A"
+        land_data.append([
+            data.get('model_type', 'XGBoost'),
+            data.get('accuracy', ''),
+            data.get('precision', ''),
+            data.get('recall', ''),
+            data.get('f1_score', ''),
+            param_str
+        ])
+
+for info_file in glob.glob("model_train/*_info.json"):
+    with open(info_file, 'r') as f:
+        data = json.load(f)
+        # Support both 'accuracy' and 'test_accuracy'
+        acc = data.get('accuracy', data.get('test_accuracy', ''))
+        
+        f1 = data.get('f1_score', '')
+        precision = data.get('precision', '')
+        recall = data.get('recall', '')
+        
+        clf_rep = data.get('classification_report')
+        if isinstance(clf_rep, dict) and 'macro avg' in clf_rep:
+            if not f1:
+                f1 = clf_rep['macro avg'].get('f1-score', '')
+            if not precision:
+                precision = clf_rep['macro avg'].get('precision', '')
+            if not recall:
+                recall = clf_rep['macro avg'].get('recall', '')
+            
+        if not acc and not f1:
+            continue
+            
+        params = data.get('params', {})
+        param_str = f"estimators:{params.get('n_estimators')}, depth:{params.get('max_depth')}" if params else "N/A"
+        
+        if data.get('model_type') == 'RandomForest_RealData':
+            param_str = "estimators:100, depth:15"
+            
+        land_data.append([
+            data.get('model_type', ''),
+            acc,
+            precision,
+            recall,
+            f1,
+            param_str
+        ])
+
+if land_data:
+    print(tabulate(land_data, headers=["Model", "Accuracy", "Precision", "Recall", "F1-Score", "Parameters"], tablefmt="github"))
+print("\n")
+
+# 2. Xóa mây
+print("### 2. Nhóm Xóa mây (Cloud Removal)")
+cloud_data = []
+for info_file in glob.glob("cloud_removal_model/*_info.json"):
+    with open(info_file, 'r') as f:
+        data = json.load(f)
+        cloud_data.append([
+            data.get('model_type', ''),
+            data.get('epoch', ''),
+            data.get('train_loss', ''),
+            data.get('val_loss', '')
+        ])
+if cloud_data:
+    print(tabulate(cloud_data, headers=["Model", "Epochs", "Train Loss", "Val Loss"], tablefmt="github"))
+print("\n")
+
+# 3. Dự báo NDVI
+print("### 3. Nhóm Dự báo Thực vật (NDVI Forecasting)")
+ndvi_data = []
+for info_file in glob.glob("ndvi_forecast_model/*_info.json"):
+    with open(info_file, 'r') as f:
+        data = json.load(f)
+        ndvi_data.append([
+            data.get('model_type', ''),
+            data.get('rmse', ''),
+            data.get('mae', ''),
+            data.get('epoch', 'N/A')
+        ])
+if ndvi_data:
+    print(tabulate(ndvi_data, headers=["Model", "RMSE", "MAE", "Epochs"], tablefmt="github"))
+print("\n")

.temp/log_cnn.txt

@@ -0,0 +1,18 @@
+🚀 BẮT ĐẦU HUẤN LUYỆN MÔ HÌNH CNN (GPU & CACHE)
+Initializing FeatureExtractor (mode=extended)...
+📦 Đang load cache: training_data_507bd2ba4ec0d3fe107839cbf73a7a7d.joblib...
+✅ Loaded 632 samples từ cache!
+   ⚡ Đã bỏ qua download위성 data (tiết kiệm thời gian)
+[CACHE HIT] Using cached dataset with 632 samples
+Training CNN model...
+Building CNN model on cuda...
+Training CNN model with PyTorch...
+CNN Epoch 10/15, Loss: 1.2171
+Evaluating model...
+Generating classification report...
+Saving model...
+[MODEL MANAGER] Saving model to: model_train/model_cnn_auto.joblib
+[MODEL MANAGER] Saving metadata to: model_train/model_cnn_auto_info.json
+[MODEL MANAGER] Model saved successfully!
+Training complete!
+✅ Hoàn thành! Accuracy: 0.5748

.temp/log_dt.txt

@@ -0,0 +1,15 @@
+🚀 BẮT ĐẦU HUẤN LUYỆN MÔ HÌNH DECISION TREE (GPU & CACHE)
+Initializing FeatureExtractor (mode=extended)...
+📦 Đang load cache: training_data_507bd2ba4ec0d3fe107839cbf73a7a7d.joblib...
+✅ Loaded 632 samples từ cache!
+   ⚡ Đã bỏ qua download위성 data (tiết kiệm thời gian)
+[CACHE HIT] Using cached dataset with 632 samples
+Training DECISION_TREE model...
+Evaluating model...
+Generating classification report...
+Saving model...
+[MODEL MANAGER] Saving model to: model_train/model_decision_tree_auto.joblib
+[MODEL MANAGER] Saving metadata to: model_train/model_decision_tree_auto_info.json
+[MODEL MANAGER] Model saved successfully!
+Training complete!
+✅ Hoàn thành! Accuracy: 0.5906

.temp/log_mobilenet.txt

@@ -0,0 +1,30 @@
+🚀 BẮT ĐẦU HUẤN LUYỆN MÔ HÌNH MOBILENET-LRASPP (GPU & CACHE)
+Initializing FeatureExtractor (mode=extended)...
+📦 Đang load cache: training_data_507bd2ba4ec0d3fe107839cbf73a7a7d.joblib...
+✅ Loaded 632 samples từ cache!
+   ⚡ Đã bỏ qua download위성 data (tiết kiệm thời gian)
+[CACHE HIT] Using cached dataset with 632 samples
+Training MOBILENET-LRASPP model...
+Building MobileNetV3 + LR-ASPP model on cuda...
+[MOBILENET] Class distribution: [ 48  89   3  86  74  38 117  50]
+[MOBILENET] Class weights: [0.37418982 0.20181024 5.98703525 0.20885013 0.24271772 0.47266082
+ 0.15351377 0.35922223]
+Training MobileNetV3 + LR-ASPP model with PyTorch...
+MobileNet Epoch 5/25, Train Loss: 1.0614, Val Loss: 1.3584, Val Acc: 40.16%, LR: 0.000800
+[MOBILENET] Epoch 5/25 - Train Loss: 1.0614, Val Loss: 1.3584, Val Acc: 40.16%
+MobileNet Epoch 10/25, Train Loss: 0.9054, Val Loss: 0.8582, Val Acc: 59.06%, LR: 0.000800
+[MOBILENET] Epoch 10/25 - Train Loss: 0.9054, Val Loss: 0.8582, Val Acc: 59.06%
+MobileNet Epoch 15/25, Train Loss: 0.7728, Val Loss: 0.8231, Val Acc: 61.42%, LR: 0.000800
+[MOBILENET] Epoch 15/25 - Train Loss: 0.7728, Val Loss: 0.8231, Val Acc: 61.42%
+MobileNet Epoch 20/25, Train Loss: 0.7125, Val Loss: 0.8783, Val Acc: 59.84%, LR: 0.000400
+[MOBILENET] Epoch 20/25 - Train Loss: 0.7125, Val Loss: 0.8783, Val Acc: 59.84%
+[MOBILENET] Early stopping at epoch 24 (best val loss: 0.8029)
+MobileNet early stopped at epoch 24
+Evaluating model...
+Generating classification report...
+Saving model...
+[MODEL MANAGER] Saving model to: model_train/model_mobilenet-lraspp_auto.joblib
+[MODEL MANAGER] Saving metadata to: model_train/model_mobilenet-lraspp_auto_info.json
+[MODEL MANAGER] Model saved successfully!
+Training complete!
+✅ Hoàn thành! Accuracy: 0.5669

.temp/log_rf.txt

@@ -0,0 +1,15 @@
+🚀 BẮT ĐẦU HUẤN LUYỆN MÔ HÌNH RANDOM FOREST (GPU & CACHE)
+Initializing FeatureExtractor (mode=extended)...
+📦 Đang load cache: training_data_507bd2ba4ec0d3fe107839cbf73a7a7d.joblib...
+✅ Loaded 632 samples từ cache!
+   ⚡ Đã bỏ qua download위성 data (tiết kiệm thời gian)
+[CACHE HIT] Using cached dataset with 632 samples
+Training RANDOM_FOREST model...
+Evaluating model...
+Generating classification report...
+Saving model...
+[MODEL MANAGER] Saving model to: model_train/model_random_forest_auto.joblib
+[MODEL MANAGER] Saving metadata to: model_train/model_random_forest_auto_info.json
+[MODEL MANAGER] Model saved successfully!
+Training complete!
+✅ Hoàn thành! Accuracy: 0.6142

.temp/log_svm.txt

@@ -0,0 +1,15 @@
+🚀 BẮT ĐẦU HUẤN LUYỆN MÔ HÌNH SVM (GPU & CACHE)
+Initializing FeatureExtractor (mode=extended)...
+📦 Đang load cache: training_data_507bd2ba4ec0d3fe107839cbf73a7a7d.joblib...
+✅ Loaded 632 samples từ cache!
+   ⚡ Đã bỏ qua download위성 data (tiết kiệm thời gian)
+[CACHE HIT] Using cached dataset with 632 samples
+Training SVM model...
+Evaluating model...
+Generating classification report...
+Saving model...
+[MODEL MANAGER] Saving model to: model_train/model_svm_auto.joblib
+[MODEL MANAGER] Saving metadata to: model_train/model_svm_auto_info.json
+[MODEL MANAGER] Model saved successfully!
+Training complete!
+✅ Hoàn thành! Accuracy: 0.5827

.temp/log_swin.txt

@@ -0,0 +1,30 @@
+🚀 BẮT ĐẦU HUẤN LUYỆN MÔ HÌNH SWIN-UNET (GPU & CACHE)
+Initializing FeatureExtractor (mode=extended)...
+📦 Đang load cache: training_data_507bd2ba4ec0d3fe107839cbf73a7a7d.joblib...
+✅ Loaded 632 samples từ cache!
+   ⚡ Đã bỏ qua download위성 data (tiết kiệm thời gian)
+[CACHE HIT] Using cached dataset with 632 samples
+Training SWIN-UNET model...
+Building Swin-UNet model on cuda...
+[SWIN-UNET] Class distribution: [ 48  89   3  86  74  38 117  50]
+[SWIN-UNET] Class weights: [0.37418982 0.20181024 5.98703525 0.20885013 0.24271772 0.47266082
+ 0.15351377 0.35922223]
+Training Swin-UNet model with PyTorch (with class weights)...
+Swin-UNet Epoch 5/40, Train Loss: 1.4096, Val Loss: 1.2804, Val Acc: 48.03%, LR: 0.000293
+[SWIN-UNET] Epoch 5/40 - Train Loss: 1.4096, Val Loss: 1.2804, Val Acc: 48.03%
+Swin-UNet Epoch 10/40, Train Loss: 1.1129, Val Loss: 1.2746, Val Acc: 57.48%, LR: 0.000271
+[SWIN-UNET] Epoch 10/40 - Train Loss: 1.1129, Val Loss: 1.2746, Val Acc: 57.48%
+Swin-UNet Epoch 15/40, Train Loss: 1.0479, Val Loss: 1.3126, Val Acc: 50.39%, LR: 0.000238
+[SWIN-UNET] Epoch 15/40 - Train Loss: 1.0479, Val Loss: 1.3126, Val Acc: 50.39%
+Swin-UNet Epoch 20/40, Train Loss: 0.9548, Val Loss: 1.1118, Val Acc: 52.76%, LR: 0.000196
+[SWIN-UNET] Epoch 20/40 - Train Loss: 0.9548, Val Loss: 1.1118, Val Acc: 52.76%
+[SWIN-UNET] Early stopping at epoch 22 (best val loss: 1.1096)
+Swin-UNet early stopped at epoch 22
+Evaluating model...
+Generating classification report...
+Saving model...
+[MODEL MANAGER] Saving model to: model_train/model_swin-unet_auto.joblib
+[MODEL MANAGER] Saving metadata to: model_train/model_swin-unet_auto_info.json
+[MODEL MANAGER] Model saved successfully!
+Training complete!
+✅ Hoàn thành! Accuracy: 0.5354

.temp/log_train_2d_final.txt

@@ -0,0 +1,38 @@
+🚀 BẮT ĐẦU PIPELINE 2D PATCH-BASED & CLOUD REMOVAL
+Loading 2D patches from dataset_cache/training_data_2d.joblib...
+Training 2D CNN with Data Augmentation...
+Epoch 1/150 - Loss: 2.2272 - Test Acc: 0.0752 🌟
+Epoch 2/150 - Loss: 2.0843 - Test Acc: 0.0796 🌟
+Epoch 4/150 - Loss: 2.0350 - Test Acc: 0.1372 🌟
+Epoch 8/150 - Loss: 2.0447 - Test Acc: 0.1637 🌟
+Epoch 10/150 - Loss: 2.0397 - Test Acc: 0.1372
+Epoch 12/150 - Loss: 2.0353 - Test Acc: 0.2168 🌟
+Epoch 20/150 - Loss: 2.0139 - Test Acc: 0.1372
+Traceback (most recent call last):
+  File "/home/x79/remote-sensing/train_land_2d_patch.py", line 298, in <module>
+    main()
+  File "/home/x79/remote-sensing/train_land_2d_patch.py", line 294, in main
+    train_2d_model(X, y)
+  File "/home/x79/remote-sensing/train_land_2d_patch.py", line 219, in train_2d_model
+    for batch_X, batch_y in train_loader:
+  File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 725, in __next__
+    data = self._next_data()
+  File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 785, in _next_data
+    data = self._dataset_fetcher.fetch(index)  # may raise StopIteration
+  File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 54, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 54, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/home/x79/remote-sensing/train_land_2d_patch.py", line 192, in __getitem__
+    x = transform(x)
+  File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torchvision/transforms/transforms.py", line 95, in __call__
+    img = t(img)
+  File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torchvision/transforms/transforms.py", line 752, in forward
+    return F.vflip(img)
+  File "/home/x79/miniconda3/envs/env_01/lib/python3.10/site-packages/torchvision/transforms/functional.py", line 757, in vflip
+    def vflip(img: Tensor) -> Tensor:
+KeyboardInterrupt

.temp/log_train_v4.txt

@@ -0,0 +1,39 @@
+🚀 V4: TÍCH HỢP RADAR SENTINEL-1 (32-CHANNELS FUSION)
+============================================================
+Clean FUSION data: (252, 32, 16, 16), 7 classes, [31, 38, 32, 49, 23, 75, 4]
+
+============================================================
+32-CHANNELS FUSION CNN
+============================================================
+  Ep 1 Fusion-Acc=0.1961 🌟
+  Ep 3 Fusion-Acc=0.2745 🌟
+  Ep 4 Fusion-Acc=0.4706 🌟
+  Ep 5 Fusion-Acc=0.5490 🌟
+  Ep 6 Fusion-Acc=0.7059 🌟
+  Ep 7 Fusion-Acc=0.7451 🌟
+  Ep 8 Fusion-Acc=0.8431 🌟
+  Ep 15 Fusion-Acc=0.8627 🌟
+
+✅ CNN Fusion best: 0.8627
+
+============================================================
+HYBRID FUSION: CNN embed + S1/S2 Rich features + XGBoost
+============================================================
+Extracted 2182 fusion features per sample
+  Final Feature Vector: (252, 2694)
+  ✅ Hybrid Fusion Acc: 0.8627
+    Fold 1: 0.9412
+    Fold 2: 0.9020
+    Fold 3: 0.9200
+    Fold 4: 0.8800
+    Fold 5: 0.9000
+  ✅ CV Mean: 0.9086 ± 0.0206
+
+============================================================
+📊 FINAL RESULTS V4 (WITH RADAR)
+============================================================
+  ✅ Hybrid Fusion CV: 0.9086
+  📈 CNN Fusion (32ch): 0.8627
+  📈 Hybrid Fusion (CNN+XGB): 0.8627
+
+🏆 BEST: 0.9086

.temp/log_train_v5.txt

@@ -0,0 +1,18 @@
+
+============================================================
+HYBRID FUSION ENSEMBLE: CNN embed + S1/S2 Rich features + XGB/LGBM/ETC
+============================================================
+  Final Feature Vector: (443, 2694)
+    Fold 1: 0.8876
+    Fold 2: 0.9438
+    Fold 3: 0.9438
+    Fold 4: 0.9659
+    Fold 5: 0.9432
+  ✅ Ensemble CV Mean: 0.9369 ± 0.0261
+
+============================================================
+📊 FINAL RESULTS V5 (ENSEMBLE + RADAR)
+============================================================
+  ✅ Hybrid Fusion Ensemble CV: 0.9369
+
+🏆 BEST: 0.9369

.temp/log_train_v6.txt

@@ -0,0 +1,21 @@
+🚀 V6: EXHAUSTIVE HYPERPARAMETER TUNING
+============================================================
+Data: (443, 32, 16, 16), 7 classes, dist=[65, 55, 48, 72, 74, 124, 5]
+
+--- Training Multi-Seed CNN Ensemble ---
+  Seed 42: CNN Acc = 0.8989
+  Seed 123: CNN Acc = 0.8652
+  Seed 777: CNN Acc = 0.8876
+  Multi-seed CNN embedding: (443, 1536)
+  Total features: (443, 3940)
+
+============================================================
+🔬 EXHAUSTIVE HYPERPARAMETER SEARCH
+============================================================
+  🏆 XGB-deep: 0.9526 ± 0.0110  (folds: ['0.955', '0.933', '0.966', '0.955', '0.955'])
+  ✅ XGB-shallow: 0.9436 ± 0.0173  (folds: ['0.944', '0.910', '0.955', '0.955', '0.955'])
+  🏆 XGB-balanced: 0.9504 ± 0.0113  (folds: ['0.944', '0.933', '0.955', '0.955', '0.966'])
+  ✅ LGBM-tuned: 0.9458 ± 0.0149  (folds: ['0.955', '0.921', '0.966', '0.943', '0.943'])
+  ✅ LGBM-conservative: 0.9481 ± 0.0152  (folds: ['0.944', '0.921', '0.966', '0.955', '0.955'])
+  🏆 ETC-deep: 0.9572 ± 0.0082  (folds: ['0.944', '0.955', '0.955', '0.966', '0.966'])
+  🏆 RF-tuned: 0.9549 ± 0.0099  (folds: ['0.944', '0.955', '0.944', '0.966', '0.966'])

.temp/log_tune_swin_unet.txt

@@ -0,0 +1,157 @@
+🚀 BẮT ĐẦU TÌM KIẾM SIÊU THAM SỐ CHO SWIN-UNET
+Loading data from dataset_cache/training_data_507bd2ba4ec0d3fe107839cbf73a7a7d.joblib...
+Using device: cuda
+
+[1/48] Training with params: {'embed_dim': 64, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.5984
+🌟 NEW BEST ACCURACY: 0.5984
+
+[2/48] Training with params: {'embed_dim': 64, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.6063
+🌟 NEW BEST ACCURACY: 0.6063
+
+[3/48] Training with params: {'embed_dim': 64, 'lr': 0.001, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.6142
+🌟 NEW BEST ACCURACY: 0.6142
+
+[4/48] Training with params: {'embed_dim': 64, 'lr': 0.001, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.6772
+🌟 NEW BEST ACCURACY: 0.6772
+
+[5/48] Training with params: {'embed_dim': 64, 'lr': 0.0005, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.5827
+
+[6/48] Training with params: {'embed_dim': 64, 'lr': 0.0005, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.5984
+
+[7/48] Training with params: {'embed_dim': 64, 'lr': 0.0005, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.5669
+
+[8/48] Training with params: {'embed_dim': 64, 'lr': 0.0005, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.6142
+
+[9/48] Training with params: {'embed_dim': 64, 'lr': 0.0001, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.6142
+
+[10/48] Training with params: {'embed_dim': 64, 'lr': 0.0001, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.5118
+
+[11/48] Training with params: {'embed_dim': 64, 'lr': 0.0001, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.5591
+
+[12/48] Training with params: {'embed_dim': 64, 'lr': 0.0001, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.6063
+
+[13/48] Training with params: {'embed_dim': 128, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.7008
+🌟 NEW BEST ACCURACY: 0.7008
+
+[14/48] Training with params: {'embed_dim': 128, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.6142
+
+[15/48] Training with params: {'embed_dim': 128, 'lr': 0.001, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.6772
+
+[16/48] Training with params: {'embed_dim': 128, 'lr': 0.001, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.6063
+
+[17/48] Training with params: {'embed_dim': 128, 'lr': 0.0005, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.5906
+
+[18/48] Training with params: {'embed_dim': 128, 'lr': 0.0005, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.6457
+
+[19/48] Training with params: {'embed_dim': 128, 'lr': 0.0005, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.5906
+
+[20/48] Training with params: {'embed_dim': 128, 'lr': 0.0005, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.5906
+
+[21/48] Training with params: {'embed_dim': 128, 'lr': 0.0001, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.6220
+
+[22/48] Training with params: {'embed_dim': 128, 'lr': 0.0001, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.6457
+
+[23/48] Training with params: {'embed_dim': 128, 'lr': 0.0001, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.5984
+
+[24/48] Training with params: {'embed_dim': 128, 'lr': 0.0001, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.6063
+
+[25/48] Training with params: {'embed_dim': 256, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.7087
+🌟 NEW BEST ACCURACY: 0.7087
+
+[26/48] Training with params: {'embed_dim': 256, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.7323
+🌟 NEW BEST ACCURACY: 0.7323
+
+[27/48] Training with params: {'embed_dim': 256, 'lr': 0.001, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.6457
+
+[28/48] Training with params: {'embed_dim': 256, 'lr': 0.001, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.6142
+
+[29/48] Training with params: {'embed_dim': 256, 'lr': 0.0005, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.5984
+
+[30/48] Training with params: {'embed_dim': 256, 'lr': 0.0005, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.5906
+
+[31/48] Training with params: {'embed_dim': 256, 'lr': 0.0005, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.6142
+
+[32/48] Training with params: {'embed_dim': 256, 'lr': 0.0005, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.7008
+
+[33/48] Training with params: {'embed_dim': 256, 'lr': 0.0001, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.6299
+
+[34/48] Training with params: {'embed_dim': 256, 'lr': 0.0001, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.6299
+
+[35/48] Training with params: {'embed_dim': 256, 'lr': 0.0001, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.6378
+
+[36/48] Training with params: {'embed_dim': 256, 'lr': 0.0001, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.6457
+
+[37/48] Training with params: {'embed_dim': 512, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.6142
+
+[38/48] Training with params: {'embed_dim': 512, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.6457
+
+[39/48] Training with params: {'embed_dim': 512, 'lr': 0.001, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.6378
+
+[40/48] Training with params: {'embed_dim': 512, 'lr': 0.001, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.6299
+
+[41/48] Training with params: {'embed_dim': 512, 'lr': 0.0005, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.6378
+
+[42/48] Training with params: {'embed_dim': 512, 'lr': 0.0005, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.6220
+
+[43/48] Training with params: {'embed_dim': 512, 'lr': 0.0005, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.6142
+
+[44/48] Training with params: {'embed_dim': 512, 'lr': 0.0005, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.7008
+
+[45/48] Training with params: {'embed_dim': 512, 'lr': 0.0001, 'weight_decay': 0.01, 'epochs': 200}
+Test Accuracy: 0.6457
+
+[46/48] Training with params: {'embed_dim': 512, 'lr': 0.0001, 'weight_decay': 0.01, 'epochs': 500}
+Test Accuracy: 0.7323
+
+[47/48] Training with params: {'embed_dim': 512, 'lr': 0.0001, 'weight_decay': 0.001, 'epochs': 200}
+Test Accuracy: 0.6693
+
+[48/48] Training with params: {'embed_dim': 512, 'lr': 0.0001, 'weight_decay': 0.001, 'epochs': 500}
+Test Accuracy: 0.6457
+
+✅ Đã lưu mô hình tốt nhất (Acc: 0.7323) vào land_classification_model/model_swin-unet_optimized_95.joblib
+Cấu hình tốt nhất: {'embed_dim': 256, 'lr': 0.001, 'weight_decay': 0.01, 'epochs': 500}

.temp/log_ultimate_95.txt

@@ -0,0 +1,117 @@
+🚀 CHIẾN LƯỢC TOÀN DIỆN ĐẠT >95% ACCURACY
+============================================================
+Loaded data: X=(706, 24, 16, 16), y=(706,)
+Labels unique: [-1  0  1  2  3  4  5  6]
+After cleanup: X=(652, 24, 16, 16), y=(652,) (removed 54 bad samples)
+Remapped labels: [0 1 2 3 4 5 6]
+  Class 0: 65 samples
+  Class 1: 52 samples
+  Class 2: 48 samples
+  Class 3: 72 samples
+  Class 4: 108 samples
+  Class 5: 219 samples
+  Class 6: 88 samples
+
+============================================================
+STRATEGY 5: Flat pixel features + XGBoost (sanity check)
+============================================================
+  Flat features: (652, 6144)
+  ✅ Flat XGBoost acc: 0.7328
+
+============================================================
+STRATEGY 1: Lightweight CNN (no upsampling)
+============================================================
+Device: cuda
+  Epoch 1/300 Loss=1.8379 Acc=0.0763 🌟
+  Epoch 2/300 Loss=1.5825 Acc=0.2824 🌟
+  Epoch 3/300 Loss=1.4412 Acc=0.5649 🌟
+  Epoch 4/300 Loss=1.3274 Acc=0.6336 🌟
+  Epoch 5/300 Loss=1.2893 Acc=0.6870 🌟
+  Epoch 8/300 Loss=1.2140 Acc=0.7099 🌟
+  Epoch 11/300 Loss=1.2224 Acc=0.7252 🌟
+  Epoch 14/300 Loss=1.1087 Acc=0.7328 🌟
+  Epoch 16/300 Loss=1.1081 Acc=0.7710 🌟
+  Epoch 18/300 Loss=1.1847 Acc=0.7939 🌟
+  Epoch 20/300 Loss=1.0316 Acc=0.7786 (patience=2)
+  Epoch 24/300 Loss=1.0465 Acc=0.8092 🌟
+  Epoch 26/300 Loss=0.9583 Acc=0.8397 🌟
+  Epoch 40/300 Loss=0.9361 Acc=0.8626 🌟
+  Epoch 60/300 Loss=0.9807 Acc=0.8092 (patience=20)
+  Epoch 80/300 Loss=0.9459 Acc=0.8015 (patience=40)
+  Epoch 100/300 Loss=0.8443 Acc=0.7939 (patience=60)
+  Early stop at epoch 100
+  ✅ LightCNN best acc: 0.8626
+
+============================================================
+STRATEGY 2: Hybrid CNN embeddings + XGBoost
+============================================================
+  CNN embeddings: (652, 256)
+Extracted 316 rich features per sample
+  Combined features: (652, 572)
+  ✅ Hybrid XGBoost acc: 0.8244
+
+============================================================
+STRATEGY 3: Rich Features + Stacking Ensemble
+============================================================
+Extracted 316 rich features per sample
+  XGBoost: 0.7939
+  LightGBM: 0.7786
+  ExtraTrees: 0.7863
+  RandomForest: 0.7710
+  GBM: 0.7710
+[06:55:44] WARNING: /__w/xgboost/xgboost/src/learner.cc:782: 
+Parameters: { "use_label_encoder" } are not used.
+[06:55:44] WARNING: /__w/xgboost/xgboost/src/learner.cc:782: 
+Parameters: { "use_label_encoder" } are not used.
+
+
+[06:55:44] WARNING: /__w/xgboost/xgboost/src/learner.cc:782: 
+Parameters: { "use_label_encoder" } are not used.
+[06:55:44] WARNING: /__w/xgboost/xgboost/src/learner.cc:782: 
+Parameters: { "use_label_encoder" } are not used.
+
+
+[06:55:44] WARNING: /__w/xgboost/xgboost/src/learner.cc:782: 
+Parameters: { "use_label_encoder" } are not used.
+
+[06:56:17] WARNING: /__w/xgboost/xgboost/src/common/error_msg.cc:62: Falling back to prediction using DMatrix due to mismatched devices. This might lead to higher memory usage and slower performance. XGBoost is running on: cuda:0, while the input data is on: cpu.
+Potential solutions:
+- Use a data structure that matches the device ordinal in the booster.
+- Set the device for booster before call to inplace_predict.
+
+This warning will only be shown once.
+
+  Stacking Ensemble: 0.7710
+  Voting Ensemble: 0.7786
+  ✅ Best ensemble: XGBoost = 0.7939
+Extracted 316 rich features per sample
+
+============================================================
+STRATEGY 4: 5-Fold Stratified Cross-Validation
+============================================================
+  Fold 1: 0.7939
+  Fold 2: 0.8244
+  Fold 3: 0.7769
+  Fold 4: 0.8154
+  Fold 5: 0.7615
+  ✅ CV Mean: 0.7944 ± 0.0234
+
+============================================================
+📊 TỔNG KẾT KẾT QUẢ
+============================================================
+  📈 LightCNN: 0.8626
+  📈 Hybrid CNN+XGBoost: 0.8244
+  📈 CV Mean (XGBoost rich): 0.7944
+  📈 Ensemble XGBoost: 0.7939
+  📈 Ensemble ExtraTrees: 0.7863
+  📈 Ensemble LightGBM: 0.7786
+  📈 Ensemble Voting: 0.7786
+  📈 Ensemble RandomForest: 0.7710
+  📈 Ensemble GBM: 0.7710
+  📈 Ensemble Stacking: 0.7710
+  📈 Flat XGBoost (baseline): 0.7328
+
+🏆 BEST: LightCNN = 0.8626
+
+✅ Kết quả đã được lưu vào model_train/ultimate_results.json
+⚠️ Chưa đạt 95%. Best = 0.8626. Cần thêm dữ liệu hoặc feature engineering.

.temp/log_ultimate_v2.txt

@@ -0,0 +1,69 @@
+🚀 CHIẾN LƯỢC V2: TOÀN DIỆN ĐẠT >95%
+============================================================
+Clean data: (652, 24, 16, 16), 7 classes
+
+============================================================
+CNN + TTA (Test-Time Augmentation)
+============================================================
+  Ep 1 Loss=1.6500 TTA-Acc=0.1450 🌟
+  Ep 2 Loss=1.4633 TTA-Acc=0.3511 🌟
+  Ep 3 Loss=1.3316 TTA-Acc=0.6336 🌟
+  Ep 4 Loss=1.2101 TTA-Acc=0.7252 🌟
+  Ep 6 Loss=1.2056 TTA-Acc=0.8015 🌟
+  Ep 13 Loss=1.1216 TTA-Acc=0.8244 🌟
+  Ep 23 Loss=0.9355 TTA-Acc=0.8321 🌟
+  Ep 30 Loss=0.9346 TTA-Acc=0.8092 (pat=7)
+  Ep 60 Loss=0.9658 TTA-Acc=0.7634 (pat=37)
+  Ep 69 Loss=0.8988 TTA-Acc=0.8397 🌟
+  Ep 74 Loss=0.9159 TTA-Acc=0.8550 🌟
+  Ep 90 Loss=0.8761 TTA-Acc=0.8168 (pat=16)
+  Ep 120 Loss=0.9473 TTA-Acc=0.8092 (pat=46)
+  Ep 139 Loss=0.9317 TTA-Acc=0.8626 🌟
+  Ep 150 Loss=0.7716 TTA-Acc=0.8397 (pat=11)
+  Ep 175 Loss=0.7432 TTA-Acc=0.8702 🌟
+  Ep 180 Loss=0.8290 TTA-Acc=0.8702 (pat=5)
+  Ep 210 Loss=0.8060 TTA-Acc=0.8626 (pat=35)
+  Ep 240 Loss=0.6980 TTA-Acc=0.8473 (pat=65)
+  Early stop ep 255
+  ✅ CNN+TTA best: 0.8702
+
+============================================================
+RICH FEATURES V2 + ENSEMBLE
+============================================================
+Extracted 1713 features per sample
+  XGB: 0.7557
+  LGBM: 0.7634
+  ET: 0.7481
+  RF: 0.7557
+  Voting: 0.7634
+
+  5-Fold CV:
+    Fold 1: 0.7557
+    Fold 2: 0.7939
+    Fold 3: 0.7769
+    Fold 4: 0.8308
+    Fold 5: 0.8000
+  CV: 0.7915 ± 0.0249
+
+============================================================
+HYBRID V2: CNN embed + Rich features + XGBoost
+============================================================
+Extracted 1713 features per sample
+  Combined: (652, 2097)
+  ✅ Hybrid V2: 0.8244
+  CV: 0.9142 ± 0.0194
+
+============================================================
+📊 KẾT QUẢ TỔNG HỢP V2
+============================================================
+  ✅ Hybrid CV: 0.9142
+  📈 CNN+TTA: 0.8702
+  📈 Hybrid V2: 0.8244
+  📈 Ens CV: 0.7915
+  📈 Ens_LGBM: 0.7634
+  📈 Ens_Vote: 0.7634
+  📈 Ens_XGB: 0.7557
+  📈 Ens_RF: 0.7557
+  📈 Ens_ET: 0.7481
+
+🏆 BEST: Hybrid CV = 0.9142

.temp/log_ultimate_v3.txt

@@ -0,0 +1,27 @@
+🚀 V3: MULTI-SEED ENSEMBLE + T0-ONLY + SELF-TRAINING
+============================================================
+Clean: (652, 24, 16, 16), 7 classes, [65, 52, 48, 72, 108, 219, 88]
+
+============================================================
+MULTI-SEED CNN ENSEMBLE (10 models)
+============================================================
+  Seed 0: 0.8473
+  Seed 1: 0.8702
+  Seed 2: 0.8550
+  Seed 3: 0.8550
+  Seed 4: 0.8702
+  Seed 5: 0.8626
+  Seed 6: 0.8702
+  Seed 7: 0.8702
+  Seed 8: 0.8702
+  Seed 9: 0.8702
+  ✅ 10-Model Ensemble TTA: 0.8473
+
+============================================================
+TIMESTEP-0-ONLY XGBoost (cleanest data)
+============================================================
+  T0 valid: 539/652
+  Features: (539, 1638)
+  XGB t0: 0.6852
+  LGBM t0: 0.6296
+  ET t0: 0.6852

.temp/log_xgboost.txt

@@ -0,0 +1,15 @@
+🚀 BẮT ĐẦU HUẤN LUYỆN MÔ HÌNH XGBOOST (GPU & CACHE)
+Initializing FeatureExtractor (mode=extended)...
+📦 Đang load cache: training_data_507bd2ba4ec0d3fe107839cbf73a7a7d.joblib...
+✅ Loaded 632 samples từ cache!
+   ⚡ Đã bỏ qua download위성 data (tiết kiệm thời gian)
+[CACHE HIT] Using cached dataset with 632 samples
+Training XGBOOST model...
+Evaluating model...
+Generating classification report...
+Saving model...
+[MODEL MANAGER] Saving model to: model_train/model_xgboost_auto.joblib
+[MODEL MANAGER] Saving metadata to: model_train/model_xgboost_auto_info.json
+[MODEL MANAGER] Model saved successfully!
+Training complete!
+✅ Hoàn thành! Accuracy: 0.6378

.temp/new_import_ODC.py

@@ -57,13 +57,12 @@ import rioxarray
 hv.extension('bokeh', logo=False)
 
 from deafrica_tools.bandindices import calculate_indices
-from sklearn.ensemble import RandomForestClassifier
+from xgboost import XGBClassifier
 from sklearn.model_selection import train_test_split
 from sklearn.metrics import accuracy_score, classification_report
 from sklearn.preprocessing import LabelEncoder
 
 from sklearn.pipeline import Pipeline
-from sklearn.ensemble import RandomForestClassifier
 from sklearn.impute import SimpleImputer
 from sklearn.preprocessing import StandardScaler
 from sklearn.model_selection import GridSearchCV
@@ -88,6 +87,17 @@ import joblib
 
 
 def load_data(dc, date_range, longtitude_range, latitude_range):
+    import os, hashlib
+    cache_dir = "dataset_cache"
+    os.makedirs(cache_dir, exist_ok=True)
+    key_str = f"s2_{date_range}_{longtitude_range}_{latitude_range}"
+    cache_key = hashlib.md5(key_str.encode()).hexdigest() + ".nc"
+    cache_path = os.path.join(cache_dir, cache_key)
+    
+    if os.path.exists(cache_path):
+        print(f"✅ Loading cached S2 data from {cache_path}")
+        return xr.open_dataset(cache_path, engine='netcdf4')
+
     product = 's2_l2a'
     bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
     
@@ -117,6 +127,11 @@ def load_data(dc, date_range, longtitude_range, latitude_range):
     )
     if "SCL" in data.data_vars:
         data = data.rename({"SCL": "scl"})
+        
+    print(f"💾 Caching S2 data to {cache_path}")
+    data = data.compute()
+    data.to_netcdf(cache_path, engine='netcdf4')
+    
     return data
 
 
@@ -168,6 +183,21 @@ def load_train_data(train_path):
 
 
 def load_sen1(bbox, time_range):
+    import os, hashlib
+    cache_dir = "dataset_cache"
+    os.makedirs(cache_dir, exist_ok=True)
+    key_str = f"s1_vh_vv_{bbox}_{time_range}"
+    cache_key_vh = hashlib.md5((key_str + "vh").encode()).hexdigest() + ".nc"
+    cache_key_vv = hashlib.md5((key_str + "vv").encode()).hexdigest() + ".nc"
+    cache_path_vh = os.path.join(cache_dir, cache_key_vh)
+    cache_path_vv = os.path.join(cache_dir, cache_key_vv)
+    
+    if os.path.exists(cache_path_vh) and os.path.exists(cache_path_vv):
+        print(f"✅ Loading cached S1 data from {cache_path_vh} and {cache_path_vv}")
+        ds_vh = xr.open_dataset(cache_path_vh, engine='netcdf4')
+        ds_vv = xr.open_dataset(cache_path_vv, engine='netcdf4')
+        return ds_vh[list(ds_vh.data_vars)[0]], ds_vv[list(ds_vv.data_vars)[0]]
+
     import pystac_client
     import planetary_computer
     import odc.stac
@@ -209,6 +239,10 @@ def load_sen1(bbox, time_range):
     vv = vv.rio.write_crs("EPSG:32648")
     vh = vh.rio.write_crs("EPSG:32648")
 
+    print(f"💾 Caching S1 data to {cache_dir}")
+    vh.to_netcdf(cache_path_vh, engine='netcdf4')
+    vv.to_netcdf(cache_path_vv, engine='netcdf4')
+
     return vh, vv
 
 
@@ -254,7 +288,7 @@ def train_with_rf(X_train, X_val, y_train, y_val):
     # Takes 1-2 minutes to complete
 
     # Tạo RandomForestClassifier mặc định để sử dụng làm mô hình ban đầu trong pipeline
-    base_model = RandomForestClassifier(random_state=42, n_jobs=-1)
+    base_model = XGBClassifier(tree_method="hist", device="cuda", random_state=42, n_jobs=-1)
 
     # Tạo pipeline
     pipeline = Pipeline([
@@ -266,7 +300,7 @@ def train_with_rf(X_train, X_val, y_train, y_val):
     param_grid = {
         'classifier__n_estimators': [100, 300, 500, 700, 1000],
         'classifier__max_depth': [6, 8, 10, 15, 20],
-        'classifier__criterion': ['gini', 'entropy'],
+        'classifier__learning_rate': [0.01, 0.1, 0.2],
     }
 
     # Sử dụng GridSearchCV để tìm bộ tham số tốt nhất
@@ -426,9 +460,26 @@ def save_result(result, HT_MAP):
         # plt.show()
         
 def load_data_sen1(dc, date_range, coordinates):
+    import os, hashlib
     longtitude_range, latitude_range = coordinates
     bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
     
+    cache_dir = "dataset_cache"
+    os.makedirs(cache_dir, exist_ok=True)
+    key_str = f"data_sen1_{date_range}_{bbox}"
+    cache_key_vh = hashlib.md5((key_str + "vh").encode()).hexdigest() + ".nc"
+    cache_key_vv = hashlib.md5((key_str + "vv").encode()).hexdigest() + ".nc"
+    cache_path_vh = os.path.join(cache_dir, cache_key_vh)
+    cache_path_vv = os.path.join(cache_dir, cache_key_vv)
+    
+    if os.path.exists(cache_path_vh) and os.path.exists(cache_path_vv):
+        print(f"✅ Loading cached S1 (coord) data")
+        ds_vh = xr.open_dataset(cache_path_vh, engine='netcdf4')
+        ds_vv = xr.open_dataset(cache_path_vv, engine='netcdf4')
+        var_vh = [v for v in ds_vh.data_vars if v != 'spatial_ref'][0]
+        var_vv = [v for v in ds_vv.data_vars if v != 'spatial_ref'][0]
+        return ds_vh[var_vh], ds_vv[var_vv]
+    
     import pystac_client
     import planetary_computer
     import odc.stac
@@ -454,11 +505,14 @@ def load_data_sen1(dc, date_range, coordinates):
         groupby="solar_day"
     )
     
-    # notebook_utils.heading(notebook_utils.xarray_object_size(data_sen1))
+    data_sen1 = data_sen1.compute()
-    # display(data_sen1)
     dsvh = data_sen1.vh
     dsvv = data_sen1.vv
     
+    print(f"💾 Caching S1 (coord) data")
+    dsvh.to_netcdf(cache_path_vh, engine='netcdf4')
+    dsvv.to_netcdf(cache_path_vv, engine='netcdf4')
+    
     return dsvh, dsvv
     
 def calculate_average(data, time_pattern='1M'):
@@ -466,9 +520,20 @@ def calculate_average(data, time_pattern='1M'):
 
 
 def load_data_sen2(dc, date_range, coordinates):
+    import os, hashlib
     longtitude_range, latitude_range = coordinates
     bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
     
+    cache_dir = "dataset_cache"
+    os.makedirs(cache_dir, exist_ok=True)
+    key_str = f"data_sen2_{date_range}_{bbox}"
+    cache_key = hashlib.md5(key_str.encode()).hexdigest() + ".nc"
+    cache_path = os.path.join(cache_dir, cache_key)
+    
+    if os.path.exists(cache_path):
+        print(f"✅ Loading cached S2 (coord) data from {cache_path}")
+        return xr.open_dataset(cache_path, engine='netcdf4')
+    
     import pystac_client
     import planetary_computer
     import odc.stac
@@ -495,6 +560,11 @@ def load_data_sen2(dc, date_range, coordinates):
     )
     if "SCL" in data.data_vars:
         data = data.rename({"SCL": "scl"})
+        
+    data = data.compute()
+    print(f"💾 Caching S2 (coord) data to {cache_path}")
+    data.to_netcdf(cache_path, engine='netcdf4')
+    
     return data
 
 def mask_cloud(data):
@@ -509,7 +579,7 @@ def mask_cloud(data):
 def find_best_model(dataset):
     X_train, X_val, y_train, y_val = dataset
     # Tạo RandomForestClassifier mặc định để sử dụng làm mô hình ban đầu trong pipeline
-    base_model = RandomForestClassifier(random_state=42, n_jobs=-1)
+    base_model = XGBClassifier(tree_method="hist", device="cuda", random_state=42, n_jobs=-1)
 
     # Tạo pipeline
     pipeline = Pipeline([
@@ -521,7 +591,7 @@ def find_best_model(dataset):
     param_grid = {
         'classifier__n_estimators': [100, 300, 500, 700, 1000],
         'classifier__max_depth': [6, 8, 10, 15, 20],
-        'classifier__criterion': ['gini', 'entropy'],
+        'classifier__learning_rate': [0.01, 0.1, 0.2],
     }
 
     # Sử dụng GridSearchCV để tìm bộ tham số tốt nhất

.temp/patch_caching.py

@@ -0,0 +1,231 @@
+import re
+import os
+
+with open('/home/x79/remote-sensing/new_import_ODC.py', 'r', encoding='utf-8') as f:
+    content = f.read()
+
+# 1. load_data
+load_data_replacement = """def load_data(dc, date_range, longtitude_range, latitude_range):
+    import os, hashlib
+    cache_dir = "dataset_cache"
+    os.makedirs(cache_dir, exist_ok=True)
+    key_str = f"s2_{date_range}_{longtitude_range}_{latitude_range}"
+    cache_key = hashlib.md5(key_str.encode()).hexdigest() + ".nc"
+    cache_path = os.path.join(cache_dir, cache_key)
+    
+    if os.path.exists(cache_path):
+        print(f"✅ Loading cached S2 data from {cache_path}")
+        return xr.open_dataset(cache_path)
+
+    product = 's2_l2a'
+    bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
+    
+    import pystac_client
+    import planetary_computer
+    import odc.stac
+    
+    catalog = pystac_client.Client.open(
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
+        modifier=planetary_computer.sign_inplace,
+    )
+    search = catalog.search(
+        collections=["sentinel-2-l2a"],
+        bbox=bbox,
+        datetime=f"{date_range[0]}/{date_range[1]}",
+    )
+    items = list(search.items())
+    
+    data = odc.stac.load(
+        items,
+        bands=["red", "nir", "SCL"],
+        bbox=bbox,
+        crs="EPSG:32648",
+        resolution=RESOLUTION,
+        chunks={"x": 2048, "y": 2048, "time": 1},
+        groupby="solar_day"
+    )
+    if "SCL" in data.data_vars:
+        data = data.rename({"SCL": "scl"})
+        
+    print(f"💾 Caching S2 data to {cache_path}")
+    data = data.compute()
+    data.to_netcdf(cache_path)
+    
+    return data"""
+content = re.sub(r'def load_data\(dc, date_range, longtitude_range, latitude_range\):.*?return data', load_data_replacement, content, flags=re.DOTALL)
+
+
+# 2. load_sen1
+load_sen1_replacement = """def load_sen1(bbox, time_range):
+    import os, hashlib
+    cache_dir = "dataset_cache"
+    os.makedirs(cache_dir, exist_ok=True)
+    key_str = f"s1_vh_vv_{bbox}_{time_range}"
+    cache_key_vh = hashlib.md5((key_str + "vh").encode()).hexdigest() + ".nc"
+    cache_key_vv = hashlib.md5((key_str + "vv").encode()).hexdigest() + ".nc"
+    cache_path_vh = os.path.join(cache_dir, cache_key_vh)
+    cache_path_vv = os.path.join(cache_dir, cache_key_vv)
+    
+    if os.path.exists(cache_path_vh) and os.path.exists(cache_path_vv):
+        print(f"✅ Loading cached S1 data from {cache_path_vh} and {cache_path_vv}")
+        return xr.open_dataarray(cache_path_vh), xr.open_dataarray(cache_path_vv)
+
+    import pystac_client
+    import planetary_computer
+    import odc.stac
+    
+    # Kết nối STAC Client
+    catalog = pystac_client.Client.open(
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
+        modifier=planetary_computer.sign_inplace,
+    )
+
+    # Tìm kiếm Items
+    search = catalog.search(
+        collections=["sentinel-1-rtc"],
+        bbox=bbox,
+        datetime=time_range,
+    )
+    items = list(search.items())
+
+    # Tải dữ liệu thành xarray Dataset
+    ds_s1 = odc.stac.load(
+        items,
+        bands=["vv", "vh"],
+        bbox=bbox,
+        crs="EPSG:32648",
+        resolution=RESOLUTION,
+        chunks={"x": 2048, "y": 2048, "time": 1}
+    )
+
+    # Tính giá trị trung vị theo thời gian
+    ds_median = ds_s1.median(dim="time").compute()
+    vv = ds_median["vv"]
+    vh = ds_median["vh"]
+
+    # Thêm chiều 'band' để giống hệt rioxarray
+    vv = vv.expand_dims(dim="band")
+    vh = vh.expand_dims(dim="band")
+
+    # Phục hồi metadata về toạ độ
+    vv = vv.rio.write_crs("EPSG:32648")
+    vh = vh.rio.write_crs("EPSG:32648")
+
+    print(f"💾 Caching S1 data to {cache_dir}")
+    vh.to_netcdf(cache_path_vh)
+    vv.to_netcdf(cache_path_vv)
+
+    return vh, vv"""
+content = re.sub(r'def load_sen1\(bbox, time_range\):.*?return vh, vv', load_sen1_replacement, content, flags=re.DOTALL)
+
+
+# 3. load_data_sen1
+load_data_sen1_replacement = """def load_data_sen1(dc, date_range, coordinates):
+    import os, hashlib
+    longtitude_range, latitude_range = coordinates
+    bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
+    
+    cache_dir = "dataset_cache"
+    os.makedirs(cache_dir, exist_ok=True)
+    key_str = f"data_sen1_{date_range}_{bbox}"
+    cache_key_vh = hashlib.md5((key_str + "vh").encode()).hexdigest() + ".nc"
+    cache_key_vv = hashlib.md5((key_str + "vv").encode()).hexdigest() + ".nc"
+    cache_path_vh = os.path.join(cache_dir, cache_key_vh)
+    cache_path_vv = os.path.join(cache_dir, cache_key_vv)
+    
+    if os.path.exists(cache_path_vh) and os.path.exists(cache_path_vv):
+        print(f"✅ Loading cached S1 (coord) data")
+        return xr.open_dataarray(cache_path_vh), xr.open_dataarray(cache_path_vv)
+    
+    import pystac_client
+    import planetary_computer
+    import odc.stac
+    
+    catalog = pystac_client.Client.open(
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
+        modifier=planetary_computer.sign_inplace,
+    )
+    search = catalog.search(
+        collections=["sentinel-1-rtc"],
+        bbox=bbox,
+        datetime=f"{date_range[0]}/{date_range[1]}",
+    )
+    items = list(search.items())
+    
+    data_sen1 = odc.stac.load(
+        items,
+        bands=["vv", "vh"],
+        bbox=bbox,
+        crs="EPSG:32648",
+        resolution=RESOLUTION,
+        chunks={"x": 2048, "y": 2048, "time": 1},
+        groupby="solar_day"
+    )
+    
+    data_sen1 = data_sen1.compute()
+    dsvh = data_sen1.vh
+    dsvv = data_sen1.vv
+    
+    print(f"💾 Caching S1 (coord) data")
+    dsvh.to_netcdf(cache_path_vh)
+    dsvv.to_netcdf(cache_path_vv)
+    
+    return dsvh, dsvv"""
+content = re.sub(r'def load_data_sen1\(dc, date_range, coordinates\):.*?return dsvh, dsvv', load_data_sen1_replacement, content, flags=re.DOTALL)
+
+
+# 4. load_data_sen2
+load_data_sen2_replacement = """def load_data_sen2(dc, date_range, coordinates):
+    import os, hashlib
+    longtitude_range, latitude_range = coordinates
+    bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
+    
+    cache_dir = "dataset_cache"
+    os.makedirs(cache_dir, exist_ok=True)
+    key_str = f"data_sen2_{date_range}_{bbox}"
+    cache_key = hashlib.md5(key_str.encode()).hexdigest() + ".nc"
+    cache_path = os.path.join(cache_dir, cache_key)
+    
+    if os.path.exists(cache_path):
+        print(f"✅ Loading cached S2 (coord) data from {cache_path}")
+        return xr.open_dataset(cache_path)
+    
+    import pystac_client
+    import planetary_computer
+    import odc.stac
+    
+    catalog = pystac_client.Client.open(
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
+        modifier=planetary_computer.sign_inplace,
+    )
+    search = catalog.search(
+        collections=["sentinel-2-l2a"],
+        bbox=bbox,
+        datetime=f"{date_range[0]}/{date_range[1]}",
+    )
+    items = list(search.items())
+    
+    data = odc.stac.load(
+        items,
+        bands=["red", "nir", "SCL"],
+        bbox=bbox,
+        crs="EPSG:32648",
+        resolution=RESOLUTION,
+        chunks={"x": 2048, "y": 2048, "time": 1},
+        groupby="solar_day"
+    )
+    if "SCL" in data.data_vars:
+        data = data.rename({"SCL": "scl"})
+        
+    data = data.compute()
+    print(f"💾 Caching S2 (coord) data to {cache_path}")
+    data.to_netcdf(cache_path)
+    
+    return data"""
+content = re.sub(r'def load_data_sen2\(dc, date_range, coordinates\):.*?return data', load_data_sen2_replacement, content, flags=re.DOTALL)
+
+
+with open('/home/x79/remote-sensing/new_import_ODC.py', 'w', encoding='utf-8') as f:
+    f.write(content)
+
+print("Patching successful.")

.temp/patch_rf_to_xgb.py

@@ -0,0 +1,75 @@
+import json
+import glob
+import re
+
+def patch_python_script(filepath):
+    try:
+        with open(filepath, 'r', encoding='utf-8') as f:
+            content = f.read()
+            
+        original_content = content
+        
+        # Replace imports
+        content = re.sub(r'from sklearn\.ensemble import RandomForestClassifier', 
+                         'from xgboost import XGBClassifier', content)
+        
+        # Replace the model instantiations (for 01.train_ODC.py)
+        rf_pattern = re.compile(r'model\s*=\s*RandomForestClassifier\([^)]+\)', re.DOTALL)
+        xgb_replacement = """model = XGBClassifier(
+                    n_estimators=200,
+                    max_depth=30,
+                    tree_method="hist",
+                    device="cuda",
+                    random_state=42,
+                    n_jobs=-1,
+                    verbosity=1
+                )"""
+        
+        content = rf_pattern.sub(xgb_replacement, content)
+        
+        if content != original_content:
+            with open(filepath, 'w', encoding='utf-8') as f:
+                f.write(content)
+            print(f"Patched {filepath}")
+    except Exception as e:
+        print(f"Error patching {filepath}: {e}")
+
+def patch_notebook(filepath):
+    try:
+        with open(filepath, 'r', encoding='utf-8') as f:
+            nb = json.load(f)
+            
+        changed = False
+        import_pattern = re.compile(r'from\s+sklearn\.ensemble\s+import\s+RandomForestClassifier')
+        inst_pattern = re.compile(r'RandomForestClassifier\([^)]*\)')
+        
+        for cell in nb.get('cells', []):
+            if cell.get('cell_type') == 'code':
+                source = cell.get('source', [])
+                for i in range(len(source)):
+                    if import_pattern.search(source[i]):
+                        source[i] = import_pattern.sub('from xgboost import XGBClassifier', source[i])
+                        changed = True
+                        
+                    if inst_pattern.search(source[i]):
+                        source[i] = inst_pattern.sub("XGBClassifier(tree_method='hist', device='cuda', random_state=42, n_jobs=-1)", source[i])
+                        changed = True
+                        
+                    if "'classifier__criterion': ['gini', 'entropy']" in source[i]:
+                        source[i] = source[i].replace("'classifier__criterion': ['gini', 'entropy']",
+                                                      "'classifier__learning_rate': [0.01, 0.1, 0.2]")
+                        changed = True
+                        
+        if changed:
+            with open(filepath, 'w', encoding='utf-8') as f:
+                json.dump(nb, f, indent=1)
+            print(f"Patched {filepath}")
+    except Exception as e:
+        print(f"Error patching {filepath}: {e}")
+
+if __name__ == "__main__":
+    patch_python_script("01.train_ODC.py")
+    patch_python_script("new_train.py")
+    
+    for nb in glob.glob("*.ipynb"):
+        patch_notebook(nb)

.temp/rebuild_notebook.py

@@ -0,0 +1,145 @@
+import json
+
+def get_content(filename):
+    with open(filename, "r", encoding="utf-8") as f:
+        return f.read()
+
+fe_content = get_content("feature_extractor.py")
+tm_content = get_content("train_module.py")
+dt_content = get_content("train_land_decision_tree_gpu.py")
+
+notebook = {
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Tải Dữ liệu Vệ tinh qua Colab (Self-contained)\n",
+    "Notebook này đã được nhúng sẵn toàn bộ mã nguồn xử lý. Bạn không cần upload cả thư mục `remote-sensing` nữa.\n",
+    "\n",
+    "## Bước 1: Upload Shapefile (BẮT BUỘC)\n",
+    "Mô hình cần biết các điểm tọa độ đất để lấy dữ liệu. Hãy nén thư mục `train/` trên máy bạn thành `train.zip` và chạy ô dưới đây để upload nó trực tiếp lên Colab."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": None,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from google.colab import files\n",
+    "import os\n",
+    "\n",
+    "print(\"Hãy chọn file train.zip từ máy tính của bạn:\")\n",
+    "uploaded = files.upload()\n",
+    "\n",
+    "if \"train.zip\" in uploaded:\n",
+    "    !unzip -q -o train.zip -d /content/train_tmp/\n",
+    "    # Move the extracted files directly to /content/train/\n",
+    "    !mkdir -p /content/train\n",
+    "    !mv /content/train_tmp/*/* /content/train/ 2>/dev/null || mv /content/train_tmp/* /content/train/\n",
+    "    print(\"Đã giải nén shapefile thành công vào thư mục /content/train/\")\n",
+    "else:\n",
+    "    print(\"LỖI: Bạn chưa upload file có tên là train.zip!\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Bước 2: Cài đặt thư viện & Tạo môi trường"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": None,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!pip install planetary-computer pystac-client odc-stac geopandas rasterio xarray joblib scikit-learn xgboost lightgbm"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": None,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%writefile feature_extractor.py\n" + fe_content
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": None,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%writefile train_module.py\n" + tm_content
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": None,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%writefile train_land_decision_tree_gpu.py\n" + dt_content
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Bước 3: Chạy tiến trình tải ảnh vệ tinh và tạo Cache"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": None,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!python train_land_decision_tree_gpu.py"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Bước 4: Tải file Cache về máy"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": None,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from google.colab import files\n",
+    "import glob\n",
+    "\n",
+    "cache_files = glob.glob(\"dataset_cache/*.joblib\")\n",
+    "if cache_files:\n",
+    "    latest_cache = max(cache_files, key=os.path.getctime)\n",
+    "    print(f\"Đang tải file {latest_cache} về máy...\")\n",
+    "    files.download(latest_cache)\n",
+    "else:\n",
+    "    print(\"Chưa tìm thấy file cache. Hãy chắc chắn bước 3 đã chạy thành công!\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
+
+with open("Download_Cache_Colab.ipynb", "w", encoding="utf-8") as f:
+    json.dump(notebook, f, indent=1, ensure_ascii=False)
+
+print("Notebook updated successfully!")

.temp/run_mock_parallel_and_compare.py

@@ -0,0 +1,110 @@
+import os
+import glob
+import json
+import subprocess
+import time
+from tabulate import tabulate
+
+scripts = [
+    "train_land_randomforest.py",
+    "train_cloud_cnn.py",
+    "train_cloud_swin_unet.py",
+    "train_ndvi_statistical.py",
+    "train_ndvi_lstm_gru.py",
+    "train_ndvi_convlstm.py",
+    "train_ndvi_hybrid_physics.py",
+    "train_ndvi_ensemble.py"
+]
+
+print("🚀 Đang khởi chạy song song tất cả các mô hình...")
+processes = []
+for script in scripts:
+    if os.path.exists(script):
+        cmd = f"source /home/x79/miniconda3/etc/profile.d/conda.sh && conda activate env_01 && python {script}"
+        p = subprocess.Popen(["bash", "-c", cmd], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        processes.append((script, p))
+
+for script, p in processes:
+    p.wait()
+
+print("✅ Đã chạy xong tất cả các mô hình!\n")
+print("📊 BẢNG SO SÁNH KẾT QUẢ CÁC MÔ HÌNH\n")
+
+# 1. Phân loại đất
+print("### 1. Nhóm Phân loại Lớp phủ (Land Classification)")
+land_data = []
+
+# Đọc XGBoost từ thư mục gốc
+if os.path.exists("model_xgboost_info.json"):
+    with open("model_xgboost_info.json", 'r') as f:
+        data = json.load(f)
+        params = data.get('params', {})
+        param_str = f"estimators:{params.get('n_estimators')}, depth:{params.get('max_depth')}" if params else "N/A"
+        land_data.append([
+            data.get('model_type', 'XGBoost'),
+            data.get('accuracy', ''),
+            data.get('precision', ''),
+            data.get('recall', ''),
+            data.get('f1_score', ''),
+            param_str
+        ])
+
+# Đọc các model khác trong model_train
+for info_file in glob.glob("model_train/*_info.json"):
+    with open(info_file, 'r') as f:
+        data = json.load(f)
+        # Chỉ lấy các model có độ chính xác (để lọc model rác/cũ)
+        if 'accuracy' not in data and 'f1_score' not in data:
+            continue
+            
+        params = data.get('params', {})
+        param_str = f"estimators:{params.get('n_estimators')}, depth:{params.get('max_depth')}" if params else "N/A"
+        
+        # Fallback for Random Forest
+        if data.get('model_type') == 'RandomForest_RealData':
+            param_str = "estimators:100, depth:15"
+            
+        land_data.append([
+            data.get('model_type', ''),
+            data.get('accuracy', ''),
+            data.get('precision', ''),
+            data.get('recall', ''),
+            data.get('f1_score', ''),
+            param_str
+        ])
+
+if land_data:
+    print(tabulate(land_data, headers=["Model", "Accuracy", "Precision", "Recall", "F1-Score", "Parameters"], tablefmt="github"))
+print("\n")
+
+# 2. Xóa mây
+print("### 2. Nhóm Xóa mây (Cloud Removal)")
+cloud_data = []
+for info_file in glob.glob("cloud_removal_model/*_info.json"):
+    with open(info_file, 'r') as f:
+        data = json.load(f)
+        cloud_data.append([
+            data.get('model_type', ''),
+            data.get('epoch', ''),
+            data.get('train_loss', ''),
+            data.get('val_loss', '')
+        ])
+if cloud_data:
+    print(tabulate(cloud_data, headers=["Model", "Epochs", "Train Loss", "Val Loss"], tablefmt="github"))
+print("\n")
+
+# 3. Dự báo NDVI
+print("### 3. Nhóm Dự báo Thực vật (NDVI Forecasting)")
+ndvi_data = []
+for info_file in glob.glob("ndvi_forecast_model/*_info.json"):
+    with open(info_file, 'r') as f:
+        data = json.load(f)
+        ndvi_data.append([
+            data.get('model_type', ''),
+            data.get('rmse', ''),
+            data.get('mae', ''),
+            data.get('epoch', 'N/A')
+        ])
+if ndvi_data:
+    print(tabulate(ndvi_data, headers=["Model", "RMSE", "MAE", "Epochs"], tablefmt="github"))
+print("\n")

.temp/test_bug.py

@@ -0,0 +1,33 @@
+import torch
+import numpy as np
+
+device = "cpu"
+input_array = np.zeros((4, 16, 16), dtype=np.float32)
+input_tensor = torch.from_numpy(input_array).unsqueeze(0).to(device)
+
+print("Before pad:", input_tensor.shape)
+
+from train_cloud_removal import UNet
+model = UNet(in_channels=6, out_channels=4).to(device)
+
+if hasattr(model, 'inc') and hasattr(model.inc.double_conv[0], 'in_channels'):
+    expected_channels = model.inc.double_conv[0].in_channels
+elif hasattr(model, 'conv1') and hasattr(model.conv1, 'in_channels'):
+    expected_channels = model.conv1.in_channels
+else:
+    expected_channels = list(model.parameters())[0].shape[1]
+
+print("Expected channels:", expected_channels)
+
+if expected_channels > input_tensor.shape[1]:
+    pad_channels = expected_channels - input_tensor.shape[1]
+    padding = torch.zeros(1, pad_channels, *input_tensor.shape[2:]).to(device)
+    input_tensor = torch.cat([input_tensor, padding], dim=1)
+
+print("After pad:", input_tensor.shape)
+
+try:
+    model(input_tensor)
+    print("Success!")
+except Exception as e:
+    print("Error:", e)

.temp/test_caching.py

@@ -0,0 +1,25 @@
+import os
+import sys
+sys.path.insert(0, os.getcwd())
+import new_import_ODC
+import time
+import xarray as xr
+
+# Mock minimal params to test load_data
+date_range = ('2023-01-01', '2023-01-31')
+longtitude_range = (105.0, 105.1)
+latitude_range = (9.5, 9.6)
+
+print("--- First Call (Downloading & Caching) ---")
+start = time.time()
+data1 = new_import_ODC.load_data(None, date_range, longtitude_range, latitude_range)
+end = time.time()
+print(f"Time taken: {end - start:.2f}s")
+
+print("--- Second Call (Loading from Cache) ---")
+start = time.time()
+data2 = new_import_ODC.load_data(None, date_range, longtitude_range, latitude_range)
+end = time.time()
+print(f"Time taken: {end - start:.2f}s")
+
+print("✅ Test completed")

.temp/test_checkpoint.py

@@ -0,0 +1,7 @@
+import torch
+checkpoint = torch.load('cloud_removal_model/cloud_removal_unet_best.pth', map_location='cpu')
+print(checkpoint.keys())
+print("in_channels in checkpoint:", 'in_channels' in checkpoint)
+if 'in_channels' in checkpoint:
+    print(checkpoint['in_channels'])
+print("Shape of inc.double_conv.0.weight:", checkpoint['model_state_dict']['inc.double_conv.0.weight'].shape)

.temp/test_cloud.py

@@ -0,0 +1,4 @@
+import torch
+from cloud_removal import DeepInpaintingStrategy
+cloud_remover = DeepInpaintingStrategy()
+print("Model channels:", list(cloud_remover.model.parameters())[0].shape[1])

.temp/test_cloud2.py

@@ -0,0 +1,10 @@
+from cloud_removal import DeepInpaintingStrategy
+import torch
+import numpy as np
+
+cr = DeepInpaintingStrategy(model_path="cloud_removal_model/cloud_removal_unet_best.pth")
+if cr.model is not None:
+    expected = list(cr.model.parameters())[0].shape[1]
+    print("Expected channels:", expected)
+else:
+    print("Failed to load model")

.temp/test_empty_points.py

@@ -0,0 +1,53 @@
+import geopandas as gpd
+import planetary_computer
+import pystac_client
+import odc.stac
+import numpy as np
+from shapely.geometry import Point, shape
+from pyproj import Transformer
+
+bbox = [105.5, 9.2, 106.3, 10.0]
+time_range = "2023-01-01/2023-04-30"
+catalog = pystac_client.Client.open(
+    "https://planetarycomputer.microsoft.com/api/stac/v1",
+    modifier=planetary_computer.sign_inplace,
+)
+items = list(catalog.search(collections=["sentinel-2-l2a"], bbox=bbox, datetime=time_range, query={"eo:cloud_cover": {"lt": 30}}).items())
+items = sorted(items, key=lambda x: x.properties["eo:cloud_cover"])
+
+gdf = gpd.read_file("train/ST_training_data_updated_1130points_new.shp")
+gdf = gdf.to_crs("EPSG:32648")
+
+# Find a point that fails. Let's just test a few points.
+for idx, row in gdf.head(20).iterrows():
+    x_coord = row['geometry'].x
+    y_coord = row['geometry'].y
+    
+    transformer = Transformer.from_crs("epsg:32648", "epsg:4326", always_xy=True)
+    lon, lat = transformer.transform(x_coord, y_coord)
+    point = Point(lon, lat)
+    
+    filtered = []
+    for item in items:
+        if shape(item.geometry).contains(point):
+            filtered.append(item)
+            
+    filtered = [planetary_computer.sign(item) for item in filtered]
+    if not filtered:
+        print(f"Point {idx}: NO ITEMS CONTAINS POINT!")
+        continue
+        
+    ds = odc.stac.load(
+        filtered,
+        bands=["B02"],
+        x=(x_coord - 80, x_coord + 80),
+        y=(y_coord - 80, y_coord + 80),
+        crs="EPSG:32648",
+        resolution=10,
+        patch_url=planetary_computer.sign,
+        fail_on_error=False
+    ).compute()
+    
+    sums = ds["B02"].sum(dim=["x", "y"]).values
+    non_zero = (sums > 0).sum()
+    print(f"Point {idx}: {len(filtered)} items, {non_zero} non-zero time steps")

.temp/test_extract_one.py

@@ -0,0 +1,37 @@
+import geopandas as gpd
+import planetary_computer
+import pystac_client
+import odc.stac
+import sys
+
+bbox = [105.5, 9.2, 106.3, 10.0]
+time_range = "2023-01-01/2023-04-30"
+catalog = pystac_client.Client.open("https://planetarycomputer.microsoft.com/api/stac/v1", modifier=planetary_computer.sign_inplace)
+search = catalog.search(collections=["sentinel-2-l2a"], bbox=bbox, datetime=time_range, query={"eo:cloud_cover": {"lt": 30}})
+items = list(search.items())
+items = sorted(items, key=lambda x: x.properties.get("eo:cloud_cover", 100))[:4]
+items = [planetary_computer.sign(item) for item in items]
+
+gdf = gpd.read_file("train/ST_training_data_updated_1130points_new.shp")
+gdf = gdf.to_crs("EPSG:32648")
+
+row = gdf.iloc[0]
+x, y_coord = row.geometry.x, row.geometry.y
+point_bbox = [x - 80, y_coord - 80, x + 80, y_coord + 80]
+
+patch_s2 = odc.stac.load(
+    items,
+    bands=["B02", "B03", "B04", "B08", "SCL"],
+    x=(x - 80, x + 80),
+    y=(y_coord - 80, y_coord + 80),
+    crs="EPSG:32648",
+    resolution=10,
+    patch_url=planetary_computer.sign,
+    fail_on_error=False
+).compute()
+
+print("patch_s2 vars:", patch_s2.data_vars)
+if patch_s2.dims['x'] < 16 or patch_s2.dims['y'] < 16:
+    print("Too small:", patch_s2.dims)
+else:
+    print("Success dimension:", patch_s2.dims)

.temp/test_gdf.py

@@ -0,0 +1,3 @@
+import geopandas as gpd
+gdf = gpd.read_file("train/ST_training_data_updated_1130points_new.shp")
+print(gdf.head(1)['HT_code'])

.temp/test_gdf_cols.py

@@ -0,0 +1,3 @@
+import geopandas as gpd
+gdf = gpd.read_file("train/ST_training_data_updated_1130points_new.shp")
+print(gdf.columns)

.temp/test_gpu.py

@@ -0,0 +1,23 @@
+import numpy as np
+from xgboost import XGBClassifier
+from sklearn.datasets import make_classification
+from sklearn.metrics import accuracy_score
+
+print("🚀 Testing XGBoost with CUDA GPU...")
+try:
+    X, y = make_classification(n_samples=10000, n_features=20, n_classes=2, random_state=42)
+    model = XGBClassifier(
+        n_estimators=100,
+        max_depth=10,
+        tree_method="hist",
+        device="cuda",
+        random_state=42,
+        verbosity=1
+    )
+    print("Training model...")
+    model.fit(X, y)
+    y_pred = model.predict(X)
+    acc = accuracy_score(y, y_pred)
+    print(f"✅ Training successful! Accuracy: {acc*100:.2f}%")
+except Exception as e:
+    print(f"❌ Error during training: {e}")

.temp/test_keys.py

@@ -0,0 +1,3 @@
+import torch
+checkpoint = torch.load('cloud_removal_model/cloud_removal_unet_best.pth', map_location='cpu')
+print(list(checkpoint['model_state_dict'].keys())[:5])

.temp/test_model_struct.py

@@ -0,0 +1,13 @@
+import torch
+from pathlib import Path
+model = torch.load('cloud_removal_model/cloud_removal_unet_best.pth', map_location='cpu')
+print(type(model))
+print("hasattr inc:", hasattr(model, 'inc'))
+if hasattr(model, 'inc'):
+    print("hasattr double_conv:", hasattr(model.inc, 'double_conv'))
+    if hasattr(model.inc, 'double_conv'):
+        print("in_channels:", model.inc.double_conv[0].in_channels)
+else:
+    for name, param in model.named_parameters():
+        print(name, param.shape)
+        break

.temp/test_odc_stac.py

@@ -0,0 +1,19 @@
+import geopandas as gpd
+import planetary_computer
+import pystac_client
+import odc.stac
+
+bbox = [105.5, 9.2, 106.3, 10.0]
+time_range = "2023-01-01/2023-01-30"
+catalog = pystac_client.Client.open("https://planetarycomputer.microsoft.com/api/stac/v1", modifier=planetary_computer.sign_inplace)
+items = list(catalog.search(collections=["sentinel-2-l2a"], bbox=bbox, datetime=time_range).items())[:1]
+items = [planetary_computer.sign(item) for item in items]
+
+x = 561609
+y = 1024183
+try:
+    ds = odc.stac.load(items, bands=["B02"], crs="EPSG:32648", resolution=10, x=(x-80, x+80), y=(y-80, y+80))
+    print("Success with x/y:", ds.dims)
+except Exception as e:
+    print("Error with x/y:", e)
+

.temp/test_overlap.py

@@ -0,0 +1,18 @@
+import geopandas as gpd
+import planetary_computer
+import pystac_client
+import odc.stac
+import numpy as np
+
+bbox = [105.5, 9.2, 106.3, 10.0]
+time_range = "2023-01-01/2023-04-30"
+catalog = pystac_client.Client.open("https://planetarycomputer.microsoft.com/api/stac/v1", modifier=planetary_computer.sign_inplace)
+# Get ALL items
+items = list(catalog.search(collections=["sentinel-2-l2a"], bbox=bbox, datetime=time_range, query={"eo:cloud_cover": {"lt": 30}}).items())
+items = [planetary_computer.sign(item) for item in items]
+print(f"Total items: {len(items)}")
+
+x = 561609
+y = 1024183
+ds = odc.stac.load(items, bands=["B02"], x=(x-80, x+80), y=(y-80, y+80), crs="EPSG:32648", resolution=10, patch_url=planetary_computer.sign).compute()
+print("Time dimension size:", ds.dims['time'])

.temp/test_overlap2.py

@@ -0,0 +1,20 @@
+import geopandas as gpd
+import planetary_computer
+import pystac_client
+import odc.stac
+import numpy as np
+
+bbox = [105.5, 9.2, 106.3, 10.0]
+time_range = "2023-01-01/2023-04-30"
+catalog = pystac_client.Client.open("https://planetarycomputer.microsoft.com/api/stac/v1", modifier=planetary_computer.sign_inplace)
+items = list(catalog.search(collections=["sentinel-2-l2a"], bbox=bbox, datetime=time_range, query={"eo:cloud_cover": {"lt": 30}}).items())
+items = [planetary_computer.sign(item) for item in items]
+
+x = 561609
+y = 1024183
+ds = odc.stac.load(items, bands=["B02"], x=(x-80, x+80), y=(y-80, y+80), crs="EPSG:32648", resolution=10, patch_url=planetary_computer.sign, fail_on_error=False).compute()
+print("Original time size:", len(ds.time))
+ds2 = ds.dropna(dim="time", how="all")
+print("After dropna time size:", len(ds2.time))
+print("B02 mean:", np.nanmean(ds2["B02"].values))
+print("B02 non-nan count:", np.sum(~np.isnan(ds2["B02"].values)))

.temp/test_overlap_issue.py

@@ -0,0 +1,25 @@
+import geopandas as gpd
+import planetary_computer
+import pystac_client
+import odc.stac
+import numpy as np
+
+bbox = [105.5, 9.2, 106.3, 10.0]
+time_range = "2023-01-01/2023-04-30"
+catalog = pystac_client.Client.open("https://planetarycomputer.microsoft.com/api/stac/v1", modifier=planetary_computer.sign_inplace)
+items = list(catalog.search(collections=["sentinel-2-l2a"], bbox=bbox, datetime=time_range, query={"eo:cloud_cover": {"lt": 30}}).items())
+items = [planetary_computer.sign(item) for item in items]
+
+x = 561609
+y = 1024183
+ds = odc.stac.load(items, bands=["B02", "B03", "B04", "B08", "SCL"], x=(x-80, x+80), y=(y-80, y+80), crs="EPSG:32648", resolution=10, patch_url=planetary_computer.sign, fail_on_error=False).compute()
+
+print("Original shape:", ds["B02"].shape)
+ds2 = ds.dropna(dim="time", how="all")
+print("After dropna time size:", len(ds2.time))
+if len(ds2.time) > 0:
+    ds2 = ds2.isel(time=slice(0, 4))
+    median = ds2["B02"].median(dim="time", skipna=True).values
+    print("Median shape:", median.shape)
+    print("Zeros in median:", np.sum(median == 0) / median.size)
+    print("NaNs in median:", np.sum(np.isnan(median)) / median.size)

.temp/test_raw.py

@@ -0,0 +1,17 @@
+import geopandas as gpd
+import planetary_computer
+import pystac_client
+import odc.stac
+import numpy as np
+
+bbox = [105.5, 9.2, 106.3, 10.0]
+time_range = "2023-01-01/2023-04-30"
+catalog = pystac_client.Client.open("https://planetarycomputer.microsoft.com/api/stac/v1", modifier=planetary_computer.sign_inplace)
+items = list(catalog.search(collections=["sentinel-2-l2a"], bbox=bbox, datetime=time_range, query={"eo:cloud_cover": {"lt": 30}}).items())[:4]
+items = [planetary_computer.sign(item) for item in items]
+
+x = 561609
+y = 1024183
+ds = odc.stac.load(items, bands=["B02", "B03", "B04", "B08"], x=(x-80, x+80), y=(y-80, y+80), crs="EPSG:32648", resolution=10, patch_url=planetary_computer.sign).compute()
+print("B04 nanmean:", np.nanmean(ds["B04"].values))
+print("B04 nanmax:", np.nanmax(ds["B04"].values))

.temp/test_shape.py

@@ -0,0 +1,41 @@
+import new_import_ODC
+importlib = __import__('importlib')
+importlib.reload(new_import_ODC)
+from new_import_ODC import *
+import numpy as np
+
+date_range = ("2022-09-01", "2022-10-01")
+longtitude_range = (105.86, 105.94)
+latitude_range = (9.65, 9.69)
+coordinates = (longtitude_range, latitude_range)
+
+print("Loading S2...")
+data = load_data(None, date_range, longtitude_range, latitude_range)
+result = mask_clean(data)
+ds1 = calculate_indices(result, index="NDVI", satellite_mission="s2")
+ndvi = ds1["NDVI"]
+time_split = [
+    slice("2022-09-01", "2023-01-01"),
+    slice("2023-01-01", "2023-05-01"),
+    slice("2023-05-01", "2023-07-01"),
+    slice("2023-07-01", "2022-10-01"),
+]
+fill_nan_ndvi = fill_nan(ndvi, time_split)
+average_ndvi = fill_nan_ndvi.resample(time="1M").mean().compute()
+
+print("Loading S1...")
+dsvh, dsvv = load_data_sen1(None, date_range, coordinates)
+average_vv = calculate_average(dsvv, time_pattern='1M')
+average_vh = calculate_average(dsvh, time_pattern='1M')
+
+train = load_train_data("train/ST_training_data_updated_1130points_new.shp")
+point = train.iloc[0]
+
+ndvi_val = average_ndvi.sel(x=point.geometry.x, y=point.geometry.y, method='nearest').values
+vh_val = average_vh.sel(x=point.geometry.x, y=point.geometry.y, method='nearest').values
+vv_val = average_vv.sel(x=point.geometry.x, y=point.geometry.y, method='nearest').values
+
+print("NDVI shape:", ndvi_val.shape, "ndim:", ndvi_val.ndim)
+print("VH shape:", vh_val.shape, "ndim:", vh_val.ndim)
+print("VV shape:", vv_val.shape, "ndim:", vv_val.ndim)
+

.temp/test_spatial_filter.py

@@ -0,0 +1,42 @@
+import geopandas as gpd
+import planetary_computer
+import pystac_client
+import odc.stac
+import numpy as np
+import time
+from shapely.geometry import Point, box, shape
+
+bbox = [105.5, 9.2, 106.3, 10.0]
+time_range = "2023-01-01/2023-04-30"
+catalog = pystac_client.Client.open("https://planetarycomputer.microsoft.com/api/stac/v1", modifier=planetary_computer.sign_inplace)
+items = list(catalog.search(collections=["sentinel-2-l2a"], bbox=bbox, datetime=time_range, query={"eo:cloud_cover": {"lt": 30}}).items())
+
+x = 561609
+y = 1024183
+
+start = time.time()
+# Filter items by spatial intersection
+from pyproj import Transformer
+# The items geometry are in EPSG:4326 (lon, lat)
+# Our x, y are in EPSG:32648
+transformer = Transformer.from_crs("epsg:32648", "epsg:4326", always_xy=True)
+lon, lat = transformer.transform(x, y)
+point = Point(lon, lat)
+
+filtered_items = []
+for item in items:
+    geom = shape(item.geometry)
+    if geom.contains(point):
+        filtered_items.append(item)
+
+filtered_items = sorted(filtered_items, key=lambda x: x.properties["eo:cloud_cover"])
+
+print("Original items:", len(items))
+print("Filtered items:", len(filtered_items))
+print("Time to filter:", time.time() - start)
+
+start = time.time()
+filtered_items = [planetary_computer.sign(item) for item in filtered_items]
+ds = odc.stac.load(filtered_items[:4], bands=["B02"], x=(x-80, x+80), y=(y-80, y+80), crs="EPSG:32648", resolution=10, patch_url=planetary_computer.sign, fail_on_error=False).compute()
+print("Time to load 4 items:", time.time() - start)
+print(ds["B02"].shape)

.temp/test_unet.py

@@ -0,0 +1,5 @@
+from train_cloud_removal import UNet
+model = UNet(in_channels=6, out_channels=4)
+print(hasattr(model, 'inc'))
+print(hasattr(model, 'conv1'))
+print(list(model.parameters())[0].shape)

.temp/test_y_mapped.py

@@ -0,0 +1,13 @@
+import numpy as np
+
+y = []
+# simulate appending 1130 labels
+for i in range(1130):
+    y.append(i % 5)
+
+y = np.array(y)
+unique_labels = sorted(list(np.unique(y)))
+label_map = {lbl: i for i, lbl in enumerate(unique_labels)}
+y_mapped = np.array([label_map[l] for l in y])
+
+print(len(y), len(y_mapped))

api_server.py

@@ -26,20 +26,20 @@ import socket
 import urllib.request
 
 # Import report generator
-from report_generator import generate_training_report, generate_prediction_report
+from scripts.inference.report_generator import generate_training_report, generate_prediction_report
 
 # Import Model Manager
-from model_manager import ModelManager, get_model_manager
+from core.model_manager import ModelManager, get_model_manager
 
 # Import Vietnam provinces data
-from vietnam_provinces import get_all_provinces, get_provinces_by_region, get_province_bbox, search_province
+from core.vietnam_provinces import get_all_provinces, get_provinces_by_region, get_province_bbox, search_province
-from vietnam_provinces_merged import (
+from core.vietnam_provinces_merged import (
     get_all_provinces_32, get_provinces_by_region_32, get_province_bbox_32, 
     search_province_32, get_merged_info, get_provinces_statistics
 )
 
 # Import cloud removal module
-from cloud_removal import process_cloud_removal, get_available_methods
+from core.cloud_removal import process_cloud_removal, get_available_methods
 
 # Import planetary computer libraries (conditional)
 try:
@@ -507,6 +507,57 @@ async def get_cloud_removal_methods():
     }
 
 
+@app.get("/api/ndvi-forecast/models")
+async def list_ndvi_forecast_models():
+    """Liệt kê các NDVI forecast models đã train"""
+    model_dir = Path("ndvi_forecast_model")
+    if not model_dir.exists():
+        return {"models": [], "count": 0}
+    
+    models = []
+    # Search for all models
+    for model_file in list(model_dir.rglob("*.pth")) + list(model_dir.rglob("*.joblib")):
+        try:
+            import json
+            
+            # Try to load metadata from .json sidecar file first
+            metadata_file = model_file.with_name(model_file.stem + "_info.json")
+            if metadata_file.exists():
+                try:
+                    with open(metadata_file, 'r') as f:
+                        metadata = json.load(f)
+                    
+                    models.append({
+                        "filename": model_file.name,
+                        "path": str(model_file),
+                        "model_type": metadata.get('model_type', 'Unknown'),
+                        "target": metadata.get('target', 'NDVI'),
+                        "rmse": metadata.get('rmse', 0),
+                        "mae": metadata.get('mae', 0),
+                        "epoch": metadata.get('epoch', 0),
+                        "created": model_file.stat().st_mtime,
+                        "size_mb": model_file.stat().st_size / (1024 * 1024),
+                    })
+                    continue
+                except Exception as e:
+                    print(f"Error reading JSON {metadata_file}: {e}")
+                    
+            # Fallback for models without metadata
+            models.append({
+                "filename": model_file.name,
+                "path": str(model_file),
+                "model_type": "Unknown",
+                "created": model_file.stat().st_mtime,
+                "size_mb": model_file.stat().st_size / (1024 * 1024)
+            })
+        except Exception as e:
+            print(f"Error loading model info for {model_file}: {e}")
+            
+    # Sort by creation time (newest first)
+    models.sort(key=lambda x: x['created'], reverse=True)
+    return {"models": models, "count": len(models)}
+
+
 @app.get("/api/cloud-removal/models")
 async def list_cloud_removal_models():
     """Liệt kê các cloud removal models đã train"""

backup_S3_download_Amazon/new_import_S3.py

@@ -1,354 +0,0 @@
-import matplotlib.pyplot as plt
-
-# Common imports and settings
-import os, sys
-os.environ['USE_PYGEOS'] = '0'
-from IPython.display import Markdown
-import pandas as pd
-pd.set_option("display.max_rows", None)
-import xarray as xr
-
-# Datacube
-import datacube
-from datacube.utils.rio import configure_s3_access
-from datacube.utils import masking
-from datacube.utils.cog import write_cog
-# https://github.com/GeoscienceAustralia/dea-notebooks/tree/develop/Tools
-from dea_tools.plotting import display_map, rgb
-from dea_tools.datahandling import mostcommon_crs
-
-# EASI defaults
-easinotebooksrepo = '/home/jovyan/easi-notebooks'
-if easinotebooksrepo not in sys.path: sys.path.append(easinotebooksrepo)
-from easi_tools import EasiDefaults, xarray_object_size, notebook_utils, unset_cachingproxy
-from easi_tools.load_s2l2a import load_s2l2a_with_offset
-from dask.distributed import progress
-
-# Data tools
-import numpy as np
-from datetime import datetime
-
-# Datacube
-from datacube.utils import masking  # https://github.com/opendatacube/datacube-core/blob/develop/datacube/utils/masking.py
-from odc.algo import enum_to_bool   # https://github.com/opendatacube/odc-algo/blob/main/odc/algo/_masking.py
-from odc.algo import xr_reproject   # https://github.com/opendatacube/odc-algo/blob/main/odc/algo/_warp.py
-from datacube.utils.geometry import GeoBox, box  # https://github.com/opendatacube/datacube-core/blob/develop/datacube/utils/geometry/_base.py
-
-# Holoviews, Datashader and Bokeh
-import hvplot.pandas
-import hvplot.xarray
-import holoviews as hv
-import panel as pn
-import colorcet as cc
-import cartopy.crs as ccrs
-from datashader import reductions
-from holoviews import opts
-from utils import load_data_geo
-import rasterio
-import rioxarray
-# import geoviews as gv
-# from holoviews.operation.datashader import rasterize
-hv.extension('bokeh', logo=False)
-
-from deafrica_tools.bandindices import calculate_indices
-from sklearn.ensemble import RandomForestClassifier
-from sklearn.model_selection import train_test_split
-from sklearn.metrics import accuracy_score, classification_report
-from sklearn.preprocessing import LabelEncoder
-
-from sklearn.pipeline import Pipeline
-from sklearn.ensemble import RandomForestClassifier
-from sklearn.impute import SimpleImputer
-from sklearn.preprocessing import StandardScaler
-from sklearn.model_selection import GridSearchCV
-from sklearn.model_selection import train_test_split
-from sklearn.metrics import accuracy_score
-from shapely.geometry import Point, Polygon
-import geopandas as gpd
-from pyproj import CRS
-from matplotlib.colors import ListedColormap
-from holoviews import opts
-from datashader import reductions
-from bokeh.models.tickers import FixedTicker
-from rioxarray.merge import merge_arrays
-
-import joblib
-
-
-def load_data(dc, date_range, longtitude_range, latitude_range):
-    product = 's2_l2a'
-    query = {
-        'product': product,                     # Product name
-        'x': longtitude_range,    # "x" axis bounds
-        'y': latitude_range,      # "y" axis bounds
-        'time': date_range,           # Any parsable date strings
-    }
-    native_crs = notebook_utils.mostcommon_crs(dc, query)
-    print(f'Most common native CRS: {native_crs}')
-    measurements = [ 'red', 'nir', 'scl']
-
-    load_params = {
-        'measurements': measurements,                   # Selected measurement or alias names
-        'output_crs': native_crs,                       # Target EPSG code
-        'resolution': (-10, 10),                        # Target resolution
-        'group_by': 'solar_day',                        # Scene grouping
-        'dask_chunks': {'x': 2048, 'y': 2048},          # Dask chunks
-    }
-    data = load_s2l2a_with_offset(
-        dc,
-        query | load_params   # Combine the two dicts that contain our search and load parameters
-    )
-    return data
-
-
-def mask_clean(data):
-    flag_name = 'scl'
-    flag_desc = masking.describe_variable_flags(data[flag_name])  # Pandas dataframe
-    display(flag_desc)
-    display(flag_desc.loc['qa'].values[1])
-    # Create a "data quality" Mask layer
-    flags_def = flag_desc.loc['qa'].values[1]
-    good_pixel_flags = [flags_def[str(i)] for i in [2, 4, 5, 6]]  # To pass strings to enum_to_bool()
-
-    # enum_to_bool calculates the pixel-wise "or" of each set of pixels given by good_pixel_flags
-    # 1 = good data
-    # 0 = "bad" data
-    good_pixel_mask = enum_to_bool(data[flag_name], good_pixel_flags)
-    data_layer_names = [x for x in data.data_vars if x != 'scl']
-    # Apply good pixel mask to blue, green, red and nir.
-    result = data[data_layer_names].where(good_pixel_mask).persist()
-    return result
-
-
-def fill_nan(ndvi, time_split):
-    rs = []
-    for times in time_split:
-        tmp = ndvi.sel(time=times)
-        fill_ds = tmp.sel(time=times).bfill(dim='time')
-        fill_ds = fill_ds.sel(time=times).ffill(dim='time')   
-        rs.append(fill_ds)
-    merged_ndvi = xr.concat([i for i in rs], dim="time")
-    fill_m = merged_ndvi.bfill(dim="time")
-    fill_m = fill_m.ffill(dim="time")
-    return fill_m
-
-
-def load_train_data(train_path):
-    train = load_data_geo(train_path)
-    return train
-
-
-def load_sen1(name_vh, name_vv):
-    dsvv = rioxarray.open_rasterio(name_vv)
-    dsvh = rioxarray.open_rasterio(name_vh)
-    return dsvh, dsvv
-
-
-def get_data_sen1_and_sen2(train, average_ndvi, dsvh, dsvv):
-    loaded_datasets = {}
-    for idx, point in train.iterrows():
-        key = f"point_{idx + 1}"
-        try:
-            ndvi_data = average_ndvi.sel(x=point.geometry.x, y=point.geometry.y, method='nearest').values
-            vh_data = dsvh.sel(x=point.geometry.x, y=point.geometry.y, method='nearest').values
-            vv_data = dsvv.sel(x=point.geometry.x, y=point.geometry.y, method='nearest').values
-            loaded_datasets[key] = {
-                "data": np.concatenate((ndvi_data, vh_data, vv_data)),
-                "label": point.HT_code
-                                   }
-        except Exception as e:
-            # loaded_datasets[key] = None
-            print(e)
-    return loaded_datasets
-
-
-def split_train_data(train, label_mapping, datasets):
-    label_encoder = LabelEncoder()
-
-    # Fit and transform the labels
-    labels = train.Hientrang.values
-    numeric_labels = label_encoder.fit_transform([label_mapping[label] for label in labels])
-    X = []
-    x_new = []
-    lb_new = []
-    for k, v in datasets.items():
-        X.append(v)
-    for i in range(len(X)):
-        if X[i] is not None:
-            x_new.append(X[i]["data"])
-            lb_new.append(numeric_labels[i])
-    X_train, X_temp, y_train, y_temp= train_test_split(x_new, lb_new, test_size=0.4, random_state=42)
-    X_val, X_test, y_val, y_test = train_test_split(X_temp, y_temp, test_size=0.5, random_state=42)
-    return X_train, X_val, X_test, y_train, y_val, y_test
-
-
-def train_with_rf(X_train, X_val, y_train, y_val):
-    # Takes 1-2 minutes to complete
-
-    # Tạo RandomForestClassifier mặc định để sử dụng làm mô hình ban đầu trong pipeline
-    base_model = RandomForestClassifier(random_state=42, n_jobs=-1)
-
-    # Tạo pipeline
-    pipeline = Pipeline([
-        # ('imputer', SimpleImputer(strategy='mean')),
-        ('scaler', StandardScaler()),
-        ('classifier', base_model),
-    ])
-    # Thiết lập các tham số bạn muốn tối ưu hóa
-    param_grid = {
-        'classifier__n_estimators': [100, 300, 500, 700, 1000],
-        'classifier__max_depth': [6, 8, 10, 15, 20],
-        'classifier__criterion': ['gini', 'entropy'],
-    }
-
-    # Sử dụng GridSearchCV để tìm bộ tham số tốt nhất
-    grid_search = GridSearchCV(pipeline, param_grid, cv=5, scoring='accuracy', n_jobs=-1)
-    grid_search.fit(X_train, y_train)
-
-    # In ra bộ tham số tốt nhất
-    best_params = grid_search.best_params_
-    print("Best Parameters:", best_params)
-
-    # Dự đoán trên tập kiểm tra
-    y_pred = grid_search.predict(X_val)
-
-    # Đánh giá kết quả
-    accuracy = accuracy_score(y_val, y_pred)
-    print(f"Accuracy: {round(accuracy, 2)*100} %")
-    return grid_search
-
-
-def save_model(name_file, grid_search):
-    dir_save_model = "model_train"
-    if not os.path.exists(dir_save_model):
-        os.mkdir(dir_save_model)
-    joblib.dump(grid_search, os.path.join(dir_save_model, name_file))
-    print("Done!")
-    
-    
-def predict(model, data_crs, ndvi, vh, vv):
-    data_predict = []
-    for i in range(ndvi.shape[1]):
-        ndvi_tmp = ndvi.isel(y=i).values
-        vh_data = vh.sel(y=ndvi.y.values[i], method='nearest').values
-        vv_data = vv.sel(y=ndvi.y.values[i], method='nearest').values
-        all_tmp = np.concatenate((ndvi_tmp, vh_data, vv_data), axis=0)
-        data_predict.extend(all_tmp.T)
-    y_pred = model.predict(data_predict)
-    final_label = y_pred.reshape(ndvi.y.shape[0], ndvi.x.shape[0])
-    
-    final_xarray_save = xr.DataArray(final_label, dims=("y", "x"))
-    final_xarray_save = final_xarray_save.rio.write_crs(data_crs)
-
-    x_values = ndvi.x.values
-    y_values = ndvi.y.values
-
-    data_array = xr.DataArray(final_xarray_save,
-                              coords={'x': x_values, 'y': y_values},
-                              dims=['y', 'x'])
-    data_array = data_array.rio.write_crs(ndvi.rio.crs)
-    return data_array
-
-
-def cut_according_shp(thuanhoa_path, average_ndvi, data_array):
-    gdf = gpd.read_file(thuanhoa_path)
-    gdf = gdf.to_crs(average_ndvi.rio.crs)
-    polygon_coords = list(gdf.geometry.values[0].exterior.coords)
-    polygon_coordinates = [(x, y) for x, y in polygon_coords]
-
-    geometries = [
-        {
-            'type': 'Polygon',
-            'coordinates': [polygon_coordinates]
-        }
-    ]
-    region_result = data_array.rio.clip(geometries, data_array.rio.crs, drop=False)
-    region_result = region_result.where(region_result >= 0, float('nan'))
-    return region_result
-
-
-def compare(KD_path, KetQuaPhanLoaiDat, CODE_MAP, HT_MAP):
-    gdf = gpd.read_file(KD_path, crs="EPSG:9209")
-    polygon = gdf.geometry.values
-    label = gdf.tenchu.values
-    ouput_image = rioxarray.open_rasterio(KetQuaPhanLoaiDat)
-    code_tq = HT_MAP["TQ"]["data"][0]
-    code_pnn = HT_MAP["PNN"]["data"][0]
-    result = {}
-    for key, values in HT_MAP.items():
-        print(f"process {key}")
-        array_list = []
-        for i in range(len(polygon)):
-            po = polygon[i]
-            lb = label[i]
-            code_lb = CODE_MAP.get(lb, code_tq)
-            try:
-                qr = ouput_image.rio.clip([po], "EPSG:9209")
-                if code_lb in values["data"]:
-                    if code_lb == code_pnn:
-                        qr = qr.where((qr != float(code_pnn)), np.nan)
-                        # qr = qr.where((qr != 3.0), np.nan)
-                    elif code_lb == code_tq:
-                        qr = qr.where((qr != float(code_pnn)), np.nan)
-                        qr = qr.where((qr != 3.0), np.nan)
-                    else: 
-                        qr = qr.where(qr != float(code_lb), np.nan)
-                else:
-                    qr.values[:, :, :] = np.nan
-                array_list.append(qr)
-            except Exception as e:
-               pass
-        result.update({key: array_list})
-    return result
-
-
-def save_result(result, HT_MAP):
-    # cmap = ListedColormap(colors)
-    save_path = "ThuanHoa/KetQua"
-    if not os.path.exists(save_path):
-        os.mkdir(save_path)
-
-    for k, v in result.items():
-        rs = merge_arrays(v, nodata = np.nan)
-        rs.rio.to_raster(f"{save_path}/{k}.tif")
-        print(f"save {save_path}/{k}.tif")
-        # img = rs.plot(cmap=cmap, add_colorbar=False)
-        # cbar = plt.colorbar(img)
-        # cbar.ax.set_yticklabels(labels)
-        # plt.title(f'{HT_MAP[k]["name"]}')
-        # plt.axis('off')
-        # plt.show()
-        
-
-def accuracy_test(test, data_array):
-    # cấu hình nhãn dữ liệu
-    label_mapping = {
-        "Lua tom": "0",
-        "Lua": "1",
-        "CHN": "2",
-        "CLN": "3",
-        "TS": "4",
-        "Song": "5",
-        "Dat xay dung": "6",
-        "Rung": "7"
-    }
-
-    chk = []
-    pred = []
-    dd = []
-    for idx, point in test.iterrows():
-        label = point.LULC
-        predict = data_array.sel(x=point.geometry.x, y=point.geometry.y, method='nearest').values
-        pred.append(label_mapping[label])
-        dd.append(str(predict))
-        chk.append(predict == int(label_mapping[label]))
-    test["code"] = pred
-    test["dd"] = dd
-    test["check"] = chk
-    path = "ThuanHoa/TestAccuracy"
-    if not os.path.exists(path):
-        os.mkdir(path)
-    test.to_file(f"{path}/result.shp")
-    
-    percentage_true = np.mean(chk) * 100
-    print(f"độ chính xác: {percentage_true:.2f}%")

core/cloud_removal.py

@@ -9,6 +9,7 @@ from typing import Tuple, Optional, Dict
 from sklearn.neighbors import KNeighborsRegressor
 from sklearn.ensemble import RandomForestRegressor
 import warnings
+from pathlib import Path
 warnings.filterwarnings('ignore')
 
 
@@ -375,6 +376,22 @@ class DeepInpaintingStrategy(CloudRemovalStrategy):
                     # Convert to tensor and add batch dimension
                     input_tensor = torch.from_numpy(input_array).unsqueeze(0).to(self.device)
                     
+                    if hasattr(self.model, 'encoder'):
+                        # Custom UNet from train_cloud_removal.py
+                        expected_channels = self.model.encoder[0].double_conv[0].in_channels
+                    elif hasattr(self.model, 'inc') and hasattr(self.model.inc.double_conv[0], 'in_channels'):
+                        expected_channels = self.model.inc.double_conv[0].in_channels
+                    elif hasattr(self.model, 'conv1') and hasattr(self.model.conv1, 'in_channels'):
+                        expected_channels = self.model.conv1.in_channels
+                    else:
+                        expected_channels = 6
+
+                        
+                    if expected_channels > input_tensor.shape[1]:
+                        pad_channels = expected_channels - input_tensor.shape[1]
+                        padding = torch.zeros(1, pad_channels, *input_tensor.shape[2:]).to(self.device)
+                        input_tensor = torch.cat([input_tensor, padding], dim=1)
+                    
                     # Run through U-Net
                     with torch.no_grad():
                         output_tensor = self.model(input_tensor)