Merge remote-tracking branch 'origin/dev_01' into dev_01

update attribute table

.idea/inspectionProfiles/profiles_settings.xml

@@ -1,6 +0,0 @@
-<component name="InspectionProjectProfileManager">
-  <settings>
-    <option name="USE_PROJECT_PROFILE" value="false" />
-    <version value="1.0" />
-  </settings>
-</component>

.idea/modules.xml

@@ -1,8 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="ProjectModuleManager">
-    <modules>
-      <module fileurl="file://$PROJECT_DIR$/.idea/remote-sensing.iml" filepath="$PROJECT_DIR$/.idea/remote-sensing.iml" />
-    </modules>
-  </component>
-</project>

.idea/remote-sensing.iml

@@ -1,8 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<module type="PYTHON_MODULE" version="4">
-  <component name="NewModuleRootManager">
-    <content url="file://$MODULE_DIR$" />
-    <orderEntry type="inheritedJdk" />
-    <orderEntry type="sourceFolder" forTests="false" />
-  </component>
-</module>

.idea/vcs.xml

@@ -1,6 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="VcsDirectoryMappings">
-    <mapping directory="" vcs="Git" />
-  </component>
-</project>

.idea/workspace.xml

@@ -1,48 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="ChangeListManager">
-    <list default="true" id="8512e5bb-2b73-4d09-a37b-d0b357b3fbe9" name="Changes" comment="" />
-    <option name="SHOW_DIALOG" value="false" />
-    <option name="HIGHLIGHT_CONFLICTS" value="true" />
-    <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
-    <option name="LAST_RESOLUTION" value="IGNORE" />
-  </component>
-  <component name="Git.Settings">
-    <option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
-  </component>
-  <component name="ProjectColorInfo"><![CDATA[{
-  "associatedIndex": 1
-}]]></component>
-  <component name="ProjectId" id="39nRCQRaBb6bqtrPoIoBjpMe8Bs" />
-  <component name="ProjectViewState">
-    <option name="hideEmptyMiddlePackages" value="true" />
-    <option name="showLibraryContents" value="true" />
-  </component>
-  <component name="PropertiesComponent"><![CDATA[{
-  "keyToString": {
-    "ModuleVcsDetector.initialDetectionPerformed": "true",
-    "RunOnceActivity.ShowReadmeOnStart": "true",
-    "RunOnceActivity.TerminalTabsStorage.copyFrom.TerminalArrangementManager.252": "true",
-    "RunOnceActivity.git.unshallow": "true",
-    "git-widget-placeholder": "dev__01",
-    "last_opened_file_path": "//wsl.localhost/Ubuntu-22.04/home/x79/remote-sensing"
-  }
-}]]></component>
-  <component name="SharedIndexes">
-    <attachedChunks>
-      <set>
-        <option value="bundled-python-sdk-4762d8aabb82-6d6dccd035ac-com.jetbrains.pycharm.pro.sharedIndexes.bundled-PY-253.30387.173" />
-      </set>
-    </attachedChunks>
-  </component>
-  <component name="TaskManager">
-    <task active="true" id="Default" summary="Default task">
-      <changelist id="8512e5bb-2b73-4d09-a37b-d0b357b3fbe9" name="Changes" comment="" />
-      <created>1771329720024</created>
-      <option name="number" value="Default" />
-      <option name="presentableId" value="Default" />
-      <updated>1771329720024</updated>
-    </task>
-    <servers />
-  </component>
-</project>

01.train_ODC_XGBoost.py

@@ -0,0 +1,171 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+get_ipython().run_cell_magic('time', '', '%matplotlib inline\n\nimport importlib\nimport new_import_ODC  \n\nimportlib.reload(new_import_ODC)\n\nfrom new_import_ODC import *\n')
+
+
+# In[2]:
+
+
+get_ipython().run_cell_magic('time', '', '# Cấu hình Daskgateway\ncluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1, 10))\n# Khai báo 1 Datacube là dc\ndc = None\n\n# Cấu hình truy cập dịch vụ S3\nconfigure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n\nclient\n')
+
+
+# In[3]:
+
+
+## cấu hình thời gian lấy ảnh và tọa độ
+date_range = ("2022-09-01", "2022-10-01")
+longtitude_range = (105.86, 105.94)
+latitude_range = (9.65, 9.69)
+
+coordinates = (longtitude_range, latitude_range)
+
+
+# In[4]:
+
+
+## truy vấn ảnh vệ tinh sen2
+data = load_data(None, date_range, longtitude_range, latitude_range)
+notebook_utils.heading(notebook_utils.xarray_object_size(data))
+display(data)
+
+
+# In[5]:
+
+
+get_ipython().run_cell_magic('time', '', '# Tiến hành loại bỏ các vị trí bị mây ảnh hưởng\nresult = mask_clean(data)\n# progress(result)\n')
+
+
+# In[6]:
+
+
+# Tiến hành tính toán NDVI
+ds1 = calculate_indices(result, index="NDVI", satellite_mission="s2")
+ndvi = ds1["NDVI"]
+display(ndvi)
+
+
+# In[7]:
+
+
+## Hiển thị ảnh NDVI chưa điền các giá trị mây (chưa fill nan)
+plt.imshow(ndvi.isel(time=0))
+
+
+# In[8]:
+
+
+# Thiết lập giá trị trung bình mùa vụ để xử lý các điểm ảnh bị mây dựa vào sự thay đổi theo mùa
+time_split = [
+    slice("2022-09-01", "2023-01-01"),
+    slice("2023-01-01", "2023-05-01"),
+    slice("2023-05-01", "2023-07-01"),
+    slice("2023-07-01", "2022-10-01"),
+]
+
+# Điền mây ở các vị trí mang giá trị nan (fill nan)
+fill_nan_ndvi = fill_nan(ndvi, time_split)
+
+# In kết quả ảnh NDVI đã điền mây (đã fill nan)
+plt.imshow(fill_nan_ndvi.isel(time=0))
+
+
+# In[9]:
+
+
+get_ipython().run_cell_magic('time', '', '## tính ndvi theo tháng\naverage_ndvi = fill_nan_ndvi.resample(time="1M").mean().persist()\n# progress(average_ndvi)\n\n# compute average_ndvi\naverage_ndvi = average_ndvi.compute()\n')
+
+
+# In[10]:
+
+
+#Load dữ liệu ảnh Sentinel 1
+dsvh, dsvv = load_data_sen1(None, date_range, coordinates)
+average_vv = calculate_average(dsvv, time_pattern='1M')
+average_vh = calculate_average(dsvh, time_pattern='1M')
+
+
+# In[11]:
+
+
+## cấu hình bộ dữ liệu điểm huấn luyện mô hình (train file)
+train_path = "train/ST_training_data_updated_1130points_new.shp"  # đường dẫn shp file train
+
+## load dữ liệu điểm huấn luyện mô hình (train file)
+train = load_train_data(train_path)
+train.head()
+
+# cấu hình nhãn dữ liệu 
+label_mapping = {
+    "Lua tom": "0",
+    "Lua": "1",
+    "CHN": "2",
+    "CLN": "3",
+    "TS": "4",
+    "Song": "5",
+    "Dat xay dung": "6",
+    "Rung": "7",
+}
+
+# xây dựng tập dữ liệu (dataset) chứa dữ liệu VH, VV, NDVI
+datasets = get_data_sen1_and_sen2(train, average_ndvi, average_vh, average_vv)
+
+# chia tập dữ liệu thành các phần theo tỉ lệ 80(80-20)-20 tương ứng với tập train, validate, test
+X_train, X_val, X_test, y_train, y_val, y_test = split_train_data(
+    train, label_mapping, datasets
+)
+
+
+# In[ ]:
+
+
+get_ipython().run_cell_magic('time', '', '# Import XGBoost\nimport xgboost as xgb\nfrom sklearn.metrics import accuracy_score\nimport numpy as np\n\n# Convert to numpy arrays\nX_train_np = np.asarray(X_train, dtype=np.float32)\nX_val_np = np.asarray(X_val, dtype=np.float32)\ny_train_np = np.asarray(y_train, dtype=np.int32)\ny_val_np = np.asarray(y_val, dtype=np.int32)\n\nprint("🚀 Training XGBoost model...")\nprint(f"   Train samples: {len(X_train_np)}")\nprint(f"   Val samples: {len(X_val_np)}")\nprint(f"   Features: {X_train_np.shape[1]}")\nprint(f"   Classes: 8\\n")\n\n# XGBoost parameters\nparams = {\n    \'objective\': \'multi:softmax\',  # Multi-class classification\n    \'num_class\': 8,                 # 8 land use classes\n    \'max_depth\': 6,                 # Maximum tree depth\n    \'learning_rate\': 0.1,           # Learning rate\n    \'n_estimators\': 200,            # Number of trees\n    \'subsample\': 0.8,               # Subsample ratio\n    \'colsample_bytree\': 0.8,        # Feature sampling ratio\n    \'random_state\': 42,\n    \'n_jobs\': -1,                   # Use all CPU cores\n    \'eval_metric\': \'mlogloss\'       # Multi-class log loss\n}\n\n# Train XGBoost model\nmodel = xgb.XGBClassifier(**params)\n\nmodel.fit(\n    X_train_np, y_train_np,\n    eval_set=[(X_train_np, y_train_np), (X_val_np, y_val_np)],\n    verbose=True\n)\n\n# Validation accuracy\ny_val_pred = model.predict(X_val_np)\nval_accuracy = accuracy_score(y_val_np, y_val_pred)\nprint(f"\\n✅ Training completed!")\nprint(f"   Validation Accuracy: {val_accuracy:.4f} ({val_accuracy*100:.2f}%)")\n')
+
+
+# In[ ]:
+
+
+get_ipython().run_cell_magic('time', '', '# Evaluate on test set\nX_test_np = np.asarray(X_test, dtype=np.float32)\ny_test_np = np.asarray(y_test, dtype=np.int32)\n\nprint("📊 Evaluating XGBoost model on test set...\\n")\n\n# Predictions\ny_pred_test = model.predict(X_test_np)\n\n# Metrics\nfrom sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, confusion_matrix\n\ntest_accuracy = accuracy_score(y_test_np, y_pred_test)\nprecision = precision_score(y_test_np, y_pred_test, average=\'weighted\', zero_division=0)\nrecall = recall_score(y_test_np, y_pred_test, average=\'weighted\', zero_division=0)\nf1 = f1_score(y_test_np, y_pred_test, average=\'weighted\', zero_division=0)\n\nprint(f"📈 Test Results:")\nprint(f"   Accuracy:  {test_accuracy:.4f} ({test_accuracy*100:.2f}%)")\nprint(f"   Precision: {precision:.4f}")\nprint(f"   Recall:    {recall:.4f}")\nprint(f"   F1-Score:  {f1:.4f}\\n")\n\n# Confusion Matrix\nfrom sklearn.metrics import ConfusionMatrixDisplay\nimport matplotlib.pyplot as plt\n\n# Create figure first\nfig, ax = plt.subplots(figsize=(10, 8))\n\nclass_names = list(label_mapping.keys())\ncm = confusion_matrix(y_test_np, y_pred_test)\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=class_names)\ndisp.plot(cmap=\'Blues\', ax=ax)\nplt.xticks(rotation=45, ha=\'right\')\nplt.title(\'XGBoost Confusion Matrix\')\nplt.tight_layout()\nplt.show()\n')
+
+
+# In[ ]:
+
+
+# Lưu mô hình huấn luyện
+import json
+import joblib
+
+# Save XGBoost model
+model_path = "model_xgboost.joblib"
+joblib.dump(model, model_path)
+print(f"✅ Model saved to {model_path}")
+
+# Save model info
+info = {
+    "model_type": "XGBoost",
+    "num_classes": 8,
+    "classes": list(label_mapping.keys()),
+    "num_features": X_train_np.shape[1],
+    "params": params,
+    "accuracy": float(test_accuracy),
+    "precision": float(precision),
+    "recall": float(recall),
+    "f1_score": float(f1),
+}
+
+with open("model_xgboost_info.json", "w") as f:
+    json.dump(info, f, indent=2)
+
+print(f"✅ Model info saved to model_xgboost_info.json")
+
+
+# In[15]:
+
+
+# đóng client, cluster
+# client.close()
+# cluster.close()
+

01.train_ODC_local_with_Mic_supplyer.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[6]:
+
+
+get_ipython().run_cell_magic('time', '', '%matplotlib inline\n\n# Import Microsoft Planetary Computer libraries\nimport planetary_computer\nfrom pystac_client import Client\nfrom odc.stac import load as stac_load\n\n# Standard imports\nimport xarray as xr\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score, classification_report, confusion_matrix, ConfusionMatrixDisplay\nimport geopandas as gpd\n\n# XGBoost for GPU training\nimport xgboost as xgb\n\nfrom xgboost import XGBClassifier\n\nprint(f"   XGBoost version: {xgb.__version__}")\n\nprint("✅ All modules loaded successfully")\n')
+
+
+# In[7]:
+
+
+get_ipython().run_cell_magic('time', '', '# Kết nối tới Microsoft Planetary Computer STAC\nfrom pystac_client import Client\n\n# KHÔNG dùng modifier ở catalog level để tránh items bị convert thành dict\ncatalog = Client.open(\n    "https://planetarycomputer.microsoft.com/api/stac/v1"\n)\nprint("✅ Connected to Microsoft Planetary Computer")\n\nprint("\\n" + "="*70)\n')
+
+
+# In[8]:
+
+
+get_ipython().run_cell_magic('time', '', '# 🌍 Định nghĩa khu vực và thời gian\nprint("="*70)\nprint("CONFIGURATION")\nprint("="*70)\n\n# Khu vực quan tâm (Vietnam - Mekong Delta) - GIẢM DIỆN TÍCH ~40%\nbbox = [105.6, 9.3, 106.2, 9.8]  # [min_lon, min_lat, max_lon, max_lat]\n\n# GIẢM THỜI GIAN xuống 3 tháng để giảm kích thước dữ liệu cho PC\ntime_range = "2023-03-01/2023-05-31"  # 3 tháng (mùa khô)\n\nprint(f"\\n📍 Area of Interest:")\nprint(f"   Longitude: {bbox[0]} to {bbox[2]}")\nprint(f"   Latitude: {bbox[1]} to {bbox[3]}")\nprint(f"\\n📅 Time Range: {time_range}")\nprint(f"   ⚠️  Optimized for personal computer (3 months, reduced area)")\nprint(f"\\n🗺️  CRS: EPSG:32648")\nprint(f"   Resolution: 20m (reduced from 10m for smaller data size)")\n\nprint("="*70)\n')
+
+
+# In[9]:
+
+
+get_ipython().run_cell_magic('time', '', '# 📡 LOAD SENTINEL-2 FROM MICROSOFT PLANETARY COMPUTER\nprint("="*70)\nprint("LOADING SENTINEL-2 L2A")\nprint("="*70)\n\nprint("\\n🔍 Searching for Sentinel-2 scenes...")\nquery_s2 = catalog.search(\n    collections=["sentinel-2-l2a"],\n    bbox=bbox,\n    datetime=time_range,\n    query={"eo:cloud_cover": {"lt": 30}}  # Cloud cover < 30% (giảm từ 50%)\n)\n\nitems_s2 = list(query_s2.item_collection())\nprint(f"✅ Found {len(items_s2)} Sentinel-2 scenes")\n\n# GIỚI HẠN SỐ LƯỢNG SCENES cho PC cá nhân\nmax_scenes = 12  # Giảm xuống 12 scenes để tối ưu cho PC\nif len(items_s2) > max_scenes:\n    print(f"⚠️  Limiting to {max_scenes} scenes for personal computer")\n    # Chọn scenes đều đặn trong khoảng thời gian\n    step = len(items_s2) // max_scenes\n    items_s2 = items_s2[::step][:max_scenes]\n    print(f"   Selected {len(items_s2)} scenes evenly distributed")\n\nif len(items_s2) > 0:\n    # Show first few scenes\n    print(f"\\n📋 Sample scenes:")\n    for i, item in enumerate(items_s2[:5]):\n        date = item.datetime.strftime("%Y-%m-%d")\n        cloud = item.properties.get("eo:cloud_cover", "N/A")\n        print(f"   [{i+1}] {date} - Cloud: {cloud}%")\n    \n    # Re-sign items to ensure fresh URLs (keep as pystac objects)\n    print(f"\\n🔑 Signing STAC items...")\n    items_s2 = [planetary_computer.sign(item) for item in items_s2]\n    \n    # Load Sentinel-2 data (without Dask chunks)\n    print(f"\\n⏳ Loading Sentinel-2 data...")\n    ds_s2 = stac_load(\n        items_s2,\n        bands=["B04", "B08", "SCL"],  # Red (B04), NIR (B08), Scene Classification (SCL)\n        crs="EPSG:32648",\n        resolution=20,  # 20m resolution (4x smaller data than 10m)\n        bbox=bbox,\n        patch_url=planetary_computer.sign,  # Re-sign URLs during loading\n        fail_on_error=False,  # Skip problematic tiles instead of crashing\n    )\n    \n    # Rename bands to simpler names\n    ds_s2 = ds_s2.rename({"B04": "red", "B08": "nir", "SCL": "scl"})\n    \n    print(f"\\n✅ Sentinel-2 loaded!")\n    print(f"   Shape: {dict(ds_s2.dims)}")\n    print(f"   Variables: {list(ds_s2.data_vars)}")\n    display(ds_s2)\nelse:\n    print(f"❌ No Sentinel-2 scenes found")\n\n    ds_s2 = Noneprint("="*70)\n')
+
+
+# In[10]:
+
+
+get_ipython().run_cell_magic('time', '', '# 📡 LOAD SENTINEL-1 FROM MICROSOFT PLANETARY COMPUTER\nprint("="*70)\nprint("LOADING SENTINEL-1 RTC")\nprint("="*70)\n\nprint("\\n🔍 Searching for Sentinel-1 scenes...")\nquery_s1 = catalog.search(\n    collections=["sentinel-1-rtc"],\n    bbox=bbox,\n    datetime=time_range,\n)\n\nitems_s1 = list(query_s1.item_collection())\nprint(f"✅ Found {len(items_s1)} Sentinel-1 scenes")\n\n# GIỚI HẠN SỐ LƯỢNG SCENES cho PC cá nhân\nmax_scenes = 12  # Giảm xuống 12 scenes để tối ưu cho PC\nif len(items_s1) > max_scenes:\n    print(f"⚠️  Limiting to {max_scenes} scenes for personal computer")\n    # Chọn scenes đều đặn trong khoảng thời gian\n    step = len(items_s1) // max_scenes\n    items_s1 = items_s1[::step][:max_scenes]\n    print(f"   Selected {len(items_s1)} scenes evenly distributed")\n\nif len(items_s1) > 0:\n    # Show first few scenes\n    print(f"\\n📋 Sample scenes:")\n    for i, item in enumerate(items_s1[:5]):\n        date = item.datetime.strftime("%Y-%m-%d")\n        orbit = item.properties.get("sat:orbit_state", "N/A")\n        print(f"   [{i+1}] {date} - Orbit: {orbit}")\n    \n    # Re-sign items to ensure fresh URLs (keep as pystac objects)\n    print(f"\\n🔑 Signing STAC items...")\n    items_s1 = [planetary_computer.sign(item) for item in items_s1]\n    \n    # Load Sentinel-1 data (without Dask chunks)\n    print(f"\\n⏳ Loading Sentinel-1 data...")\n    ds_s1 = stac_load(\n        items_s1,\n        bands=["vv", "vh"],  # VV and VH polarizations\n        crs="EPSG:32648",\n        resolution=20,  # 20m resolution (4x smaller data than 10m)\n        bbox=bbox,\n        patch_url=planetary_computer.sign,  # Re-sign URLs during loading\n        fail_on_error=False,  # Skip problematic tiles instead of crashing\n    )\n    \n    # Convert to dB (Microsoft S1 is in linear power)\n    print(f"\\n🔄 Converting to dB...")\n    ds_s1[\'vv_db\'] = 10 * np.log10(ds_s1[\'vv\'].where(ds_s1[\'vv\'] > 0))\n    ds_s1[\'vh_db\'] = 10 * np.log10(ds_s1[\'vh\'].where(ds_s1[\'vh\'] > 0))\n    \n    print(f"\\n✅ Sentinel-1 loaded!")\n    print(f"   Shape: {dict(ds_s1.dims)}")\n    print(f"   Variables: {list(ds_s1.data_vars)}")\n    display(ds_s1)\nelse:\n    print(f"❌ No Sentinel-1 scenes found")\n\n    ds_s1 = Noneprint("="*70)\n')
+
+
+# In[11]:
+
+
+get_ipython().run_cell_magic('time', '', '# 🌿 CALCULATE NDVI AND PROCESS DATA\nprint("="*70)\nprint("DATA PROCESSING")\nprint("="*70)\n\nif ds_s2 is not None:\n    print("\\n[1] Calculating NDVI...")\n    # NDVI = (NIR - Red) / (NIR + Red)\n    ndvi = (ds_s2[\'nir\'] - ds_s2[\'red\']) / (ds_s2[\'nir\'] + ds_s2[\'red\'] + 1e-8)\n    \n    print(f"✅ NDVI calculated")\n    print(f"   Shape: {ndvi.shape}")\n    print(f"   Time steps: {len(ndvi.time)}")\n    \n    # Cloud masking using SCL band\n    print(f"\\n[2] Applying cloud mask...")\n    # SCL values: 1=defective, 3=cloud shadow, 8=cloud medium, 9=cloud high, 10=cirrus\n    cloud_mask = ds_s2[\'scl\'].isin([1, 3, 8, 9, 10])\n    ndvi_masked = ndvi.where(~cloud_mask)\n    \n    print(f"✅ Cloud mask applied")\n    \n    # Temporal aggregation (mean over time)\n    print(f"\\n[3] Computing mean NDVI across time...")\n    ndvi_mean = ndvi_masked.mean(dim=\'time\')\n    \n    # Data already in memory, no need to compute() again\n    print(f"✅ Mean NDVI computed")\n    print(f"   Shape: {ndvi_mean.shape}")\n    \nelse:\n    print("❌ No Sentinel-2 data to process")\n    ndvi_mean = None\n\nprint("="*70)\n')
+
+
+# In[ ]:
+
+
+get_ipython().run_cell_magic('time', '', '# 🎯 EXTRACT TRAINING DATA FEATURES\nprint("="*70)\nprint("FEATURE EXTRACTION")\nprint("="*70)\n\n# Check if required data is available\nif \'ndvi_mean\' not in globals() or \'ds_s1\' not in globals():\n    print("❌ Error: Please run Cell 6 (DATA PROCESSING) first!")\n    print("   Required variables: ndvi_mean, ds_s1")\n    raise RuntimeError("Missing required data. Run cells in order: Cell 4 → Cell 5 → Cell 6 → Cell 7")\n\n# Load training shapefile\nimport geopandas as gpd\n\ntrain_path = \'train/ST_training data_updated_1130points_new.shp\'\nprint(f"\\n[1] Loading training data from: {train_path}")\ntrain_gdf = gpd.read_file(train_path)\n\n# Ensure CRS matches\nif train_gdf.crs != \'EPSG:32648\':\n    print(f"   Reprojecting from {train_gdf.crs} to EPSG:32648...")\n    train_gdf = train_gdf.to_crs(\'EPSG:32648\')\n\nprint(f"✅ Loaded {len(train_gdf)} training points")\nprint(f"   Available columns: {list(train_gdf.columns)}")\n\n# Auto-detect label column (look for common names)\nlabel_column = None\nfor col in [\'HT_code\', \'Ma_LU\', \'LU2022\', \'class\', \'Class\', \'CLASS\', \'label\', \'Label\', \'LABEL\', \'LU_CODE\', \'LU_code\']:\n    if col in train_gdf.columns:\n        label_column = col\n        break\n\nif label_column is None:\n    print(f"❌ Cannot find label column. Available columns: {list(train_gdf.columns)}")\n    print(f"   Please check your shapefile and update the code.")\nelse:\n    print(f"   Using label column: \'{label_column}\'")\n    print(f"   Classes: {sorted(train_gdf[label_column].unique())}")\n    \n    # Extract features at each training point\n    print(f"\\n[2] Extracting features at training points...")\n    \n    features = []\n    labels = []\n    skipped = 0\n    \n    for idx, row in train_gdf.iterrows():\n        point = row.geometrychro\n        x_coord = point.x\n        y_coord = point.y\n        label = row[label_column]\n        \n        # Extract NDVI at this location\n        if ndvi_mean is not None and ds_s1 is not None:\n            try:\n                ndvi_val = ndvi_mean.sel(x=x_coord, y=y_coord, method=\'nearest\').values\n                \n                # Extract Sentinel-1 VH/VV at this location (mean across time)\n                # Data already in memory, no need to compute()\n                vh_val = ds_s1[\'vh_db\'].sel(x=x_coord, y=y_coord, method=\'nearest\').mean(dim=\'time\').values\n                vv_val = ds_s1[\'vv_db\'].sel(x=x_coord, y=y_coord, method=\'nearest\').mean(dim=\'time\').values\n                \n                # Create feature vector: [NDVI, VH_dB, VV_dB]\n                feature_vec = [ndvi_val, vh_val, vv_val]\n                \n                # Only add if all features are valid (not NaN)\n                if not np.isnan(feature_vec).any():\n                    features.append(feature_vec)\n                    labels.append(label)\n                else:\n                    skipped += 1\n            except Exception as e:\n                # Skip points outside the data extent\n                skipped += 1\n                continue\n    \n    features = np.array(features)\n    labels = np.array(labels)\n    \n    print(f"✅ Extracted features for {len(features)} valid points")\n    print(f"   Skipped {skipped} points (outside extent or NaN values)")\n    print(f"   Feature shape: {features.shape}")\n    print(f"   Feature names: [\'NDVI_mean\', \'VH_dB_mean\', \'VV_dB_mean\']")\n    print(f"\\n   Class distribution:")\n    unique, counts = np.unique(labels, return_counts=True)\n    for cls, cnt in zip(unique, counts):\n        print(f"      Class {cls}: {cnt} samples ({cnt/len(labels)*100:.1f}%)")\n\nprint("="*70)\n')
+
+
+# In[21]:
+
+
+get_ipython().run_cell_magic('time', '', '# 🤖 TRAIN XGBOOST MODEL ON GPU (RTX 4060)\nprint("="*70)\nprint("MODEL TRAINING - GPU ACCELERATED")\nprint("="*70)\n\nfrom xgboost import XGBClassifier\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import classification_report, confusion_matrix, ConfusionMatrixDisplay\nimport matplotlib.pyplot as plt\n\n# Encode labels to ensure they are 0, 1, 2, ... n-1\nprint("\\n[1] Encoding labels...")\nlabel_encoder = LabelEncoder()\nlabels_encoded = label_encoder.fit_transform(labels)\nprint(f"✅ Original classes: {label_encoder.classes_}")\nprint(f"   Encoded as: {np.unique(labels_encoded)}")\n\n# Split data\nprint("\\n[2] Splitting data (80% train, 20% test)...")\nX_train, X_test, y_train, y_test = train_test_split(\n    features, labels_encoded, test_size=0.2, random_state=42, stratify=labels_encoded\n)\nprint(f"✅ Training samples: {len(X_train)}")\nprint(f"   Testing samples: {len(X_test)}")\n\n# Train XGBoost on GPU\nprint("\\n[3] Training XGBoost classifier on RTX 4060 GPU...")\nprint("   GPU Settings: device=\'cuda:0\'")\n\nxgb_model = XGBClassifier(\n    n_estimators=100,\n    max_depth=20,\n    learning_rate=0.1,\n    device=\'cuda:0\',           # Use GPU (updated from deprecated gpu_id)\n    tree_method=\'hist\',        # Use hist with device for GPU training\n    random_state=42,\n    eval_metric=\'mlogloss\',    # Multi-class log loss\n    verbosity=1                # Show GPU training progress\n)\n\nxgb_model.fit(X_train, y_train)\nprint(f"✅ Model trained on GPU")\n\n# Evaluate\nprint("\\n[4] Evaluating model...")\ntrain_score = xgb_model.score(X_train, y_train)\ntest_score = xgb_model.score(X_test, y_test)\nprint(f"✅ Training accuracy: {train_score:.4f}")\nprint(f"   Testing accuracy: {test_score:.4f}")\n\n# Classification report\nprint("\\n[5] Classification Report:")\ny_pred = xgb_model.predict(X_test)\nprint(classification_report(y_test, y_pred, target_names=[str(c) for c in label_encoder.classes_]))\n\n# Confusion matrix\nprint("\\n[6] Confusion Matrix:")\nfig, ax = plt.subplots(figsize=(10, 8))\ncm = confusion_matrix(y_test, y_pred)\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=label_encoder.classes_)\ndisp.plot(ax=ax, cmap=\'Blues\', values_format=\'d\')\nplt.title(\'Confusion Matrix - XGBoost GPU Model (RTX 4060)\')\nplt.tight_layout()\nplt.show()\n\nprint("="*70)\n')
+
+
+# In[23]:
+
+
+get_ipython().run_cell_magic('time', '', '# 💾 SAVE MODEL AND CLEANUP\nprint("="*70)\nprint("SAVING MODEL & CLEANUP")\nprint("="*70)\n\nimport joblib\nfrom datetime import datetime\n\n# Save model and label encoder\nmodel_filename = f"model_train/model_xgboost_gpu_{datetime.now().strftime(\'%Y%m%d_%H%M%S\')}.joblib"\nprint(f"\\n[1] Saving model to: {model_filename}")\njoblib.dump({\'model\': xgb_model, \'label_encoder\': label_encoder}, model_filename)\nprint(f"✅ Model and label encoder saved")\n\n# Save model info\ninfo = {\n    "timestamp": datetime.now().isoformat(),\n    "data_source": "Microsoft Planetary Computer STAC",\n    "collections": ["sentinel-2-l2a", "sentinel-1-rtc"],\n    "features": ["NDVI_mean", "VH_dB_mean", "VV_dB_mean"],\n    "training_samples": len(X_train),\n    "testing_samples": len(X_test),\n    "train_accuracy": float(train_score),\n    "test_accuracy": float(test_score),\n    "model_type": "XGBClassifier",\n    "device": "cuda:0",\n    "gpu_device": "RTX 4060",\n    "tree_method": "hist",\n    "n_estimators": 100,\n    "max_depth": 20,\n    "learning_rate": 0.1\n}\n\nimport json\ninfo_filename = model_filename.replace(\'.joblib\', \'_info.json\')\nwith open(info_filename, \'w\') as f:\n    json.dump(info, f, indent=2)\nprint(f"✅ Model info saved to: {info_filename}")\n\n# No cleanup needed (Dask removed)\nprint("\\n[2] Cleanup complete")\n\nprint("="*70)\n\nprint("\\n" + "="*70)\n\nprint("🎉 TRAINING COMPLETE!")\n')
+

02.predict_ODC.py

@@ -0,0 +1,191 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+get_ipython().run_cell_magic('time', '', '%matplotlib inline\n\nimport importlib\nimport new_import_ODC  \n\nimportlib.reload(new_import_ODC)\n\nfrom new_import_ODC import *\n')
+
+
+# In[2]:
+
+
+get_ipython().run_cell_magic('time', '', '# Dask gateway\ncluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))\ndc = datacube.Datacube()\n\n# Configure s3 access\nconfigure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n\nclient\n')
+
+
+# In[3]:
+
+
+## cấu hình thời gian lấy ảnh và tọa độ
+date_range = ('2022-09-01', '2023-10-01')
+longtitude_range = (105.86575, 105.94120)
+latitude_range = (9.65070, 9.69850)
+
+
+# In[4]:
+
+
+## truy vấn ảnh vệ tinh sen2
+data = load_data(dc, date_range, longtitude_range, latitude_range)
+notebook_utils.heading(notebook_utils.xarray_object_size(data))
+display(data)
+
+
+# In[5]:
+
+
+get_ipython().run_cell_magic('time', '', '# Tiến hành loại bỏ các vị trí bị mây ảnh hưởng\nresult = mask_clean(data)\nprogress(result)\n')
+
+
+# In[6]:
+
+
+# Tiến hành tính toán NDVI
+ds1 = calculate_indices(result, index='NDVI', satellite_mission='s2')
+ndvi = ds1["NDVI"]
+display(ndvi)
+
+
+# In[7]:
+
+
+## ảnh NDVI chưa điền mây (fill nan)
+plt.imshow(ndvi.isel(time=50))
+
+
+# In[8]:
+
+
+# đặt thời gian các mùa
+time_split = [slice('2022-09-01', '2023-01-01'), 
+               slice('2023-01-01', '2023-05-01'),
+               slice('2023-05-01', '2023-07-01'),
+               slice('2023-07-01', '2023-10-01')]
+
+# Điền mây ở các vị trí mang giá trị nan (fill nan)
+fill_nan_ndvi = fill_nan(ndvi, time_split)
+
+# In kết quả ảnh ndvi đã điền mây (đã fill nan)
+plt.imshow(fill_nan_ndvi.isel(time=50))
+
+
+# In[9]:
+
+
+get_ipython().run_cell_magic('time', '', "## tính ndvi theo tháng\naverage_ndvi = fill_nan_ndvi.resample(time='1M').mean().persist()\nprogress(average_ndvi)\n")
+
+
+# In[10]:
+
+
+# compute average_ndvi
+average_ndvi = average_ndvi.compute()
+
+
+# In[11]:
+
+
+# load dữ liệu sen1
+coordinates = (longtitude_range, latitude_range)
+dsvh, dsvv = load_data_sen1(dc, date_range, coordinates)
+average_vv = calculate_average(dsvv, time_pattern='1M')
+average_vh = calculate_average(dsvh, time_pattern='1M')
+
+
+# In[12]:
+
+
+# load model RF
+loaded_model = joblib.load(os.path.join("model_train", "model_odc.joblib"))
+
+# dự đoán
+data_array = predict(loaded_model, data.rio.crs, average_ndvi, average_vh, average_vv)
+
+
+# In[13]:
+
+
+# cấu hình màu cho các loại đất
+colors = [
+    "#abcee9",
+    "#ffef44",
+    "#c4ff9e",
+    "#ffd6a8",
+    "#93ddda",
+    "#1aeef7",
+    "#ffa7f2",
+    "#33ee33"
+]
+labels = [
+    "Lúa tôm",
+    "Lúa",
+    "CHN",
+    "CLN",
+    "TS",
+    "Sông",
+    "Đất xây dựng",
+    "Rừng"
+]
+# hiển thị phân loại sử dụng đất
+cmap = ListedColormap(colors)
+img = data_array.plot(cmap=cmap, add_colorbar=False)
+cbar = plt.colorbar(img)
+cbar.ax.set_yticklabels(labels)
+plt.title("Phân loại sử dụng đất")
+plt.axis('off')
+plt.show()
+
+
+# In[14]:
+
+
+## cấu hình shapefile ranh giới thuận hòa và vh vv file
+thuanhoa_path = "ThuanHoa/region/ST_ThuanHoa_Boundaryofficially.shp"
+
+# cắt theo ranh giới xã thuận hòa
+region_result = cut_according_shp(thuanhoa_path, average_ndvi, data_array)
+
+
+# In[15]:
+
+
+# hiển thị kết quả phân loại sử dụng đất
+colorval = list(range(len(colors)))
+options = {
+    'title': 'Phân loại sử dụng đất',
+    'cmap': colors,
+    'clim': (0, 8),
+    'aspect': 'equal',
+    'colorbar_opts': {
+        'major_label_overrides': dict(zip(colorval, labels)),
+        'major_label_text_align': 'left',
+        'ticker': FixedTicker(ticks=colorval),
+    },
+}
+                                      
+region_result.hvplot(
+    rasterize = True,                  # Use Datashader, particularly useful for dask arrays
+    aggregator = reductions.mode(),    # Datashader selects mode value, requires 'hv.Image'
+).options(opts.Image(**options))
+
+
+# In[16]:
+
+
+# Lưu lại kết quả
+region_result.rio.to_raster("KetQuaPhanLoaiDatODC.tif")
+
+
+# In[17]:
+
+
+# đóng client, cluster
+client.close()
+cluster.close()
+
+
+# In[ ]:
+
+
+
+

03.compare_ODC.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+# Khai báo các thư viện cần thiết
+from new_import_ODC import *
+
+# Khai báo đường dẫn đến kết quả phân loại và dữ liệu của địa phương
+KD_path = "ThuanHoa/KhoanhDat/ThuanHoa_TKDD2022.shp"
+KetQuaPhanLoaiDat = "KetQuaPhanLoaiDatODC.tif"
+
+
+# In[2]:
+
+
+# khai báo các loại đất từ dữ liệu kiểm kê ứng với các hiện trạng được phân loại từ viễn thám
+CODE_MAP = {
+    "BHK": 2,
+    "CLN": 3,
+    "DGD": 6,
+    "DGT": 6,
+    "DNL": 6,
+    "DRA": 6,
+    "DSH": 6,
+    "DTL": 5,
+    "DTS": 6,
+    "DYT": 6,
+    "LUC": 1,
+    "NKH": 3,
+    "NTD": 6,
+    "NTS": 4,
+    "ONT": 6,
+    "SKC": 6,
+    "SKX": 6,
+    "SON": 5,
+    "TMD": 6,
+    "TON": 6,
+    "TSC": 6,
+}
+
+# Khai báo các nhãn phân loại đất ứng với 3 loại đất chính
+HT_MAP = {
+    "NN": {"name": "Đất Nông Nghiệp", "data": [1, 2, 3, 4]},
+    "PNN": {"name": "Đất Phi Nông Nghiệp", "data": [6]},
+    "TQ": {"name": "Đất Thổ Quả", "data": [15]},
+}
+
+
+# In[3]:
+
+
+# Tiến hành chồng lắp
+result = compare(KD_path, KetQuaPhanLoaiDat, CODE_MAP, HT_MAP)
+
+
+# In[4]:
+
+
+# cấu hình màu cho các loại sử dụng đất
+colors = [
+    "#abcee9",
+    "#ffffc0",
+    "#c4ff9e",
+    "#ffd6a8",
+    "#93ddda",
+    "#1aeef7",
+    "#ffa7f2",
+    "#33ee33",
+]
+labels = ["Lúa tôm", "Lúa", "CHN", "CLN", "TS", "Sông", "Đất xây dựng", "Rừng"]
+
+
+# In[5]:
+
+
+# Lưu kết quả
+save_result(result, HT_MAP)
+
+
+# In[6]:
+
+
+# hiển thị kết quả
+xx = []
+
+for k, v in result.items():
+    rs = merge_arrays(v, nodata=np.nan)
+    xx.append(rs.squeeze(drop=True))
+xx = xr.concat(xx, pd.Index([HT_MAP[x]["name"] for x in HT_MAP], name="name"))
+
+colorval = list(range(len(colors)))
+options = {
+    "cmap": colors,
+    "clim": (0, 8),
+    "aspect": "equal",
+    "height": 400,
+    "colorbar_opts": {
+        "major_label_overrides": dict(zip(colorval, labels)),
+        "major_label_text_align": "left",
+        "ticker": FixedTicker(ticks=colorval),
+    },
+}
+
+xx.hvplot(
+    groupby="name",
+    rasterize=True,  # Use Datashader, particularly useful for dask arrays
+    aggregator=reductions.mode(),  # Datashader selects mode value, requires 'hv.Image'
+).options(opts.Image(**options))
+
+
+# In[ ]:
+
+
+
+

GEMINI_PROJECT_CONTEXT.md

@@ -1,638 +0,0 @@
-# GEMINI PROJECT CONTEXT - Land Classification & Remote Sensing System
-
-**Last Updated**: March 26, 2026  
-**Project Location**: `/home/x79/remote-sensing`  
-**Purpose**: Complete land classification and environmental monitoring system using satellite remote sensing for Vietnam
-
----
-
-## 📋 PROJECT OVERVIEW
-
-### High-Level Purpose & Problem Domain
-- **Core Task**: Classify land use/land cover (8 land classes) in Vietnam using multispectral Sentinel-2 and radar Sentinel-1 data from Microsoft Planetary Computer
-- **Geographic Focus**: Vietnam provinces/regions with bounding-box (bbox) based Area-of-Interest (AOI) selection
-- **Key Capabilities**:
-  - Dynamic training with user-selected regions and time periods
-  - Pixel-wise inference (prediction) on new regions
-  - Cloud removal using 7 different strategies
-  - NDVI time-series forecasting and change detection workflows
-  - Auto-generated HTML reports with visualizations
-  - Batch processing of multiple regions
-  - Model lifecycle management (save, load, validate, delete)
-
-### Data Pipeline
-```
-Sentinel-2 (optical) + Sentinel-1 (SAR) 
-    ↓
-[Feature Extraction: 4 modes - simple (3) / temporal (39) / extended (15) / odc (8)]
-    ↓
-[Model Training: XGBoost, RF, SVM, CNN, Swin-UNet, MobileNet-LRASPP]
-    ↓
-[Prediction: Pixel-wise classification]
-    ↓
-[Output: GeoTIFF + PNG preview + HTML report + JSON metadata]
-```
-
----
-
-## 🏗️ SYSTEM ARCHITECTURE
-
-### Core Technology Stack
-- **Backend**: FastAPI (~4200 lines in `api_server.py`)
-- **ML Training**: scikit-learn (XGBoost, RF, SVM, DT) + PyTorch (CNN, Swin-UNet, MobileNet)
-- **Geospatial**: rasterio, rioxarray, geopandas, xarray, odc.stac
-- **Data Access**: Microsoft Planetary Computer STAC API (Sentinel-2 L2A, Sentinel-1 RTC)
-- **Frontend**: HTML + Leaflet.js (map drawing) + Fetch API + Chart.js
-- **GPU Support**: PyTorch with CUDA 12.x (optional fallback to CPU)
-
-### Folder Structure
-```
-remote-sensing/
-├── Core Backend
-│   ├── api_server.py                 # FastAPI app (~4200 LOC, 70+ endpoints)
-│   ├── train_module.py               # Training pipeline engine
-│   ├── feature_extractor.py          # Unified feature extraction (4 modes)
-│   ├── model_manager.py              # Model lifecycle management
-│   ├── cloud_removal.py              # 7 cloud removal strategies
-│   ├── report_generator.py           # Auto HTML/PNG report generation
-│   ├── generate_previews.py          # GeoTIFF → PNG conversion
-│   │
-├── Utilities & Lookup
-│   ├── vietnam_provinces.py          # Province bboxes & metadata
-│   ├── vietnam_provinces_merged.py   # 32-province variant
-│   ├── utils.py                      # Geospatial helper functions
-│   ├── create_odc_metadata.py        # Metadata generator utility
-│   │
-├── Frontend Pages (HTML)
-│   ├── index.html                    # Main dashboard hub
-│   ├── training_interface.html       # Training UI
-│   ├── prediction_interface.html     # Prediction UI
-│   ├── batch_interface.html          # Batch processing UI
-│   ├── ndvi_interface.html           # NDVI time-series UI
-│   ├── dashboard.html                # Analytics dashboard
-│   ├── reports_interface.html        # Reports management
-│   ├── change_detection_interface.html # Change detection UI
-│   ├── cloud_training_interface.html # Cloud removal training UI
-│   │
-├── Tests & Notebooks
-│   ├── test_*.py                     # Unit & integration tests
-│   ├── 01.train_ODC*.ipynb           # Training notebooks
-│   ├── 02.predict_ODC.ipynb          # Prediction notebooks
-│   ├── cloud_removal_train.ipynb     # Cloud removal training
-│   │
-├── Model Storage & Caches
-│   ├── model_train/                  # Trained models (*.joblib, *.pth)
-│   │   ├── model_odc.joblib          # Legacy GridSearchCV model
-│   │   ├── model_*_info.json         # Metadata sidecar files
-│   ├── cloud_removal_model/          # Cloud removal U-Net models (.pth)
-│   ├── predictions/                  # Prediction output (GeoTIFF + PNG)
-│   ├── reports/                      # Generated HTML reports
-│   ├── dataset_cache/                # Cached Sentinel data (optional)
-│   │
-├── Config & Documentation
-│   ├── requirement.txt               # Python dependencies
-│   ├── requirements_api.txt          # API-specific deps
-│   ├── IMPLEMENTATION_SUMMARY.md     # Model manager summary
-│   ├── MODEL_MANAGER_GUIDE.md        # Full model management guide
-│   ├── NDVI_FORECAST_METHODOLOGY.md  # NDVI algorithm docs
-│   ├── CLOUD_TRAINING_GUIDE.md       # Cloud removal training guide
-│   └── [Other guides & docs]
-```
-
----
-
-## 🔧 MAIN MODULES & RESPONSIBILITIES
-
-| **Module** | **File(s)** | **Key Responsibility** |
-|---|---|---|
-| **API Server** | `api_server.py` | FastAPI app with 70+ endpoints; routes all training, prediction, batch, cloud removal, dashboard, reports, model management tasks |
-| **Training Engine** | `train_module.py` | Complete training pipeline: fetch data → feature extraction → train/test split → model training → evaluation → save with metadata |
-| **Feature Extraction** | `feature_extractor.py` | Standardized feature extraction with 4 modes: simple, temporal, extended, odc; used by both training and prediction |
-| **Model Manager** | `model_manager.py` | Lifecycle management: list, load, save, validate, delete models; handles metadata JSON; auto-detects CNN/PyTorch models |
-| **Cloud Removal** | `cloud_removal.py` | 7 cloud removal strategies: classic (3-step), temporal_only, median_composite, none, speckle filter, ML inpainting, deep learning U-Net |
-| **Report Generator** | `report_generator.py` | Auto-generates HTML/PNG reports with confusion matrices, class distributions, accuracy trends |
-| **Preview Generator** | `generate_previews.py` | Converts GeoTIFF outputs to PNG previews (NDVI or classification rasters) |
-| **Province Lookup** | `vietnam_provinces*.py` | Lookup tables for 32+ Vietnamese provinces with bboxes and region grouping |
-| **Utilities** | `utils.py` | Geospatial helper functions (load GeoDataFrames, etc.) |
-
----
-
-## 📊 END-TO-END WORKFLOWS
-
-### 1. TRAINING WORKFLOW
-```
-User Input → Training Configuration
-    ↓
-API Endpoint: POST /api/training/start
-    ↓
-train_module.py: train_model()
-    1. Fetch Sentinel-2 & Sentinel-1 from Planetary Computer STAC
-    2. Apply cloud mask (SCL band: clouds, shadows, cirrus masked)
-    3. Extract features via FeatureExtractor (mode: simple/temporal/extended/odc)
-    4. Train/test split (default 0.2)
-    5. Train selected model type (XGBoost, RF, CNN, Swin-UNet, MobileNet)
-    6. Evaluate: accuracy, precision, recall, F1, confusion matrix
-    ↓
-model_manager.py: Save model + JSON metadata
-    ↓
-report_generator.py: Auto-generate HTML training report
-    ↓
-Return: {model_filename, accuracy_metrics, training_time}
-```
-
-**Key Metadata Saved**:
-```json
-{
-  "timestamp": "2026-03-26T14:30:00",
-  "model_type": "xgboost",
-  "feature_mode": "temporal",
-  "n_features": 39,
-  "n_classes": 8,
-  "features": ["NDVI_t1", "NDVI_t2", ..., "NDWI_t1", ...],
-  "test_accuracy": 0.85,
-  "train_accuracy": 0.92,
-  "bbox": [105.6, 9.3, 106.2, 9.8],
-  "time_range": "2023-03-01/2023-05-31",
-  "resolution": 20,
-  "data_source": "Microsoft Planetary Computer STAC"
-}
-```
-
-### 2. PREDICTION WORKFLOW
-```
-User Input → Prediction Configuration (model_filename, bbox, time_range, cloud_strategy)
-    ↓
-API Endpoint: POST /api/predict or POST /api/predict/with-ndvi
-    ↓
-run_prediction() function:
-    1. Load model via model_manager.py (retrieves metadata, feature requirements)
-    2. Fetch Sentinel-2 & Sentinel-1 for new region
-    3. Apply chosen cloud_removal_method (classic/temporal_only/median_composite/none/deep_learning)
-    4. Extract features matching model's metadata requirements
-    5. Auto-adjust if feature count mismatch (pad/trim)
-    6. Predict land class for each pixel
-    7. (Optional) Calculate NDVI: (NIR - Red) / (NIR + Red)
-    8. Save outputs: GeoTIFF + PNG preview
-    ↓
-generate_previews.py: Create PNG from GeoTIFF
-    ↓
-report_generator.py: Generate prediction report
-    ↓
-Return: {prediction_file, ndvi_file, class_distribution, statistics}
-```
-
-### 3. BATCH PROCESSING WORKFLOW
-```
-User uploads CSV with multiple regions:
-  (name, min_lon, min_lat, max_lon, max_lat, start_date, end_date, max_scenes, cloud_cover, resolution)
-    ↓
-API Endpoint: POST /api/batch/start
-    ↓
-Enqueue all regions; process sequentially
-    ↓
-For each region: Run same prediction workflow
-    ↓
-Track status per region: Queued → Running → Completed/Failed
-    ↓
-UI shows progress bar, auto-retry on failure (max 3 retries)
-    ↓
-Return: Bulk results with per-region status & output files
-```
-
-### 4. CLOUD REMOVAL WORKFLOW
-```
-User selects cloud_removal_method in prediction config:
-    ↓
-cloud_removal.py: process_cloud_removal()
-    ↓
-Strategy Selection:
-  • 'classic': temporal interpolation → median composite → spatial interpolation (3-step)
-  • 'temporal_only': ffill + bfill across time dimension (fast, good for many scenes)
-  • 'median_composite': Prioritize median across scenes (best for noise reduction)
-  • 'none': Keep original, just fill NaN with 0
-  • 'deep': Use trained U-Net model (S2 cloudy + S1 → clean S2)
-  • 'ml_inpainting': KNN or Random Forest based inpainting
-  • 'speckle_filter': Reduce radar noise
-    ↓
-Return cleaned Sentinel-2 data for subsequent feature extraction
-```
-
-### 5. NDVI TIME-SERIES WORKFLOW
-```
-User requests NDVI calculation (bbox + time_range + aggregation)
-    ↓
-API Endpoint: POST /api/ndvi/timeseries or /api/ndvi/predict-timeseries
-    ↓
-Load Sentinel-2 (B04 Red, B08 NIR)
-    ↓
-Calculate NDVI = (NIR - Red) / (NIR + Red + 0.00001)
-    ↓
-Resample to monthly or user-defined aggregation
-    ↓
-Export as GeoTIFF + PNG visualization
-    ↓
-Show time-series graph & statistics (mean, min, max, std, trend)
-```
-
-### 6. CHANGE DETECTION WORKFLOW
-```
-User selects: model + current_period + prediction_period
-    ↓
-API Endpoint: POST /api/change-detection/compare-periods
-    ↓
-Run prediction for both time periods
-    ↓
-Compute difference map (current - prediction)
-    ↓
-Classify changes: increased vegetation, decreased vegetation, stable
-    ↓
-Generate change map GeoTIFF + report with statistics
-```
-
----
-
-## 🌐 API ENDPOINTS SUMMARY (70+ endpoints)
-
-### Model Management
-- `GET /api/models/list` - List all trained models with metadata
-- `GET /api/models/{filename}/info` - Get model details
-- `GET /api/models/{filename}/validate` - Validate model integrity
-- `DELETE /api/models/{filename}` - Delete model file
-
-### Training APIs
-- `POST /api/training/start` - Start land classification training
-- `GET /api/training/status` - Get training progress
-- `POST /api/training/stop` - Cancel ongoing training
-- `POST /api/cloud-removal/train` - Train cloud removal U-Net
-
-### Prediction APIs
-- `POST /api/predict` - Standard prediction (classification only)
-- `POST /api/predict/with-ndvi` - Prediction with NDVI export
-- `POST /api/change-detection/compare-periods` - Change detection
-- `GET /api/prediction/status` - Check prediction progress
-- `GET /api/predictions/list` - List prediction outputs
-- `GET /api/predictions/download/{filename}` - Download prediction file
-- `GET /api/predictions/preview/{filename}` - View PNG preview
-
-### Batch Processing
-- `POST /api/batch/start` - Enqueue multiple predictions from CSV
-- `GET /api/batch/status` - Check batch queue
-- `GET /api/batch/results/{batch_id}` - Retrieve batch results
-- `POST /api/batch/cancel/{batch_id}` - Cancel batch job
-
-### Cloud Removal
-- `GET /api/cloud-removal/methods` - List available strategies
-- `GET /api/cloud-removal/models` - List trained .pth models
-- `POST /api/cloud-removal/upload` - Upload .pth cloud removal model
-- `DELETE /api/cloud-removal/models/{filename}` - Delete cloud removal model
-
-### Dashboard & Reports
-- `GET /api/dashboard/statistics` - Overall system stats
-- `GET /api/dashboard/accuracy-trends` - Accuracy over time
-- `GET /api/dashboard/class-distribution/{model_filename}` - Class distribution
-- `GET /api/reports/list` - List generated reports
-- `GET /api/reports/view/{filename}` - View HTML report
-- `GET /api/reports/download/{filename}` - Download report
-- `DELETE /api/reports/delete/{filename}` - Delete report
-
-### Provinces & Utilities
-- `GET /api/provinces/list` - List all Vietnamese provinces
-- `GET /api/provinces/by-region` - Group provinces by region
-- `GET /api/provinces/{province_name}/bbox` - Get province bbox
-- `GET /api/provinces/search/{query}` - Search province by name
-- `GET /api/provinces-32/*` - Alternative 32-province variant
-- `GET /api/network/check` - Check connectivity to Planetary Computer
-- `GET /api/cache/info` - Show cache statistics
-- `POST /api/cache/clear` - Clear local cache
-
-### NDVI & Time-Series
-- `POST /api/ndvi/timeseries` - Calculate NDVI time-series
-- `POST /api/ndvi/predict-timeseries` - NDVI prediction/forecast
-- `POST /api/ndvi/forecast` - NDVI forecasting
-
-### File Management
-- `GET /api/training/files` - List training files
-- `GET /api/overlay/shapefiles` - List available shapefiles
-- `GET /api/training/shapefile/{filename}/labels` - Get shapefile labels
-- `POST /api/land-classification/upload` - Upload custom model
-- `POST /api/cloud-removal/upload` - Upload cloud removal model
-
-### Frontend Routes (Serve HTML)
-- `GET /` - Main dashboard
-- `GET /training` - Training interface
-- `GET /prediction` - Prediction interface
-- `GET /dashboard` - Analytics dashboard
-- `GET /batch` - Batch processing UI
-- `GET /ndvi` - NDVI time-series UI
-- `GET /reports` - Reports management
-- `GET /cloud-training` - Cloud removal training
-- `GET /change-detection` - Change detection UI
-
----
-
-## 💾 DATA INPUTS / OUTPUTS & FOLDER CONVENTIONS
-
-### Input Data Sources
-- **Sentinel-2 L2A** from Microsoft Planetary Computer STAC API
-  - Bands: B02 (blue), B03 (green), B04 (red), B08 (NIR), B11 (SWIR), SCL (cloud mask)
-  - Resolution: 10m or 20m (user selectable)
-  - Collection: `sentinel-2-l2a`
-
-- **Sentinel-1 RTC** from Planetary Computer
-  - Bands: VH, VV (radar polarizations)
-  - Converted to dB scale: `10 * log10(intensity)`
-  - Collection: `sentinel-1-rtc`
-
-- **Training Labels**: User-provided shapefiles with pixel-level class labels
-
-### Output File Structure
-```
-predictions/
-  ├── prediction_YYYYMMDD_HHMMSS.tif         # Classification GeoTIFF
-  ├── prediction_YYYYMMDD_HHMMSS.png         # PNG preview
-  ├── ndvi_YYYYMMDD_HHMMSS.tif              # NDVI raster
-  ├── ndvi_YYYYMMDD_HHMMSS.png              # NDVI preview
-
-reports/
-  ├── training_report_*.html                 # Auto training reports
-  ├── prediction_report_*.html                # Auto prediction reports
-
-model_train/
-  ├── model_odc.joblib                      # Legacy model
-  ├── model_odc_info.json                    # Metadata
-  ├── model_xgboost_*.joblib                 # XGBoost models
-  ├── model_xgboost_*_info.json              # Metadata
-  ├── model_cnn_*.joblib                     # CNN models
-  ├── model_cnn_*_info.json                  # Metadata
-
-cloud_removal_model/
-  ├── cloud_removal_unet_best.pth            # Trained U-Net
-  ├── *.pth                                  # Custom models
-  ├── *.json                                 # Model metadata
-```
-
----
-
-## 🔌 EXTERNAL DEPENDENCIES & PLATFORMS
-
-### Critical External Services
-- **Microsoft Planetary Computer** (STAC API)
-  - Hosts Sentinel-2 L2A and Sentinel-1 RTC archives
-  - URL: `https://planetarycomputer.microsoft.com/api/stac/v1`
-  - Auto-signed access tokens via `planetary_computer.sign_inplace`
-  - Network connectivity check: `GET /api/network/check`
-
-### Key Python Libraries
-- **Geospatial**: rasterio, rioxarray, geopandas, shapely, Cartopy, folium, ipyleaflet
-- **Data Processing**: numpy, pandas, xarray, dask
-- **ML**: scikit-learn, xgboost
-- **Deep Learning**: torch, torchvision
-- **Web**: fastapi, uvicorn, pydantic
-- **Visualization**: matplotlib, Pillow (PIL)
-- **Document Gen**: markdown, Pillow
-
-### GPU Support
-- PyTorch with CUDA 12.x (optional; falls back to CPU)
-- Benefits Swin-UNet and CNN models (10-100x speedup)
-- CPU training for XGBoost/RF typically <1 hour; deep models need GPU for reasonable speed
-
----
-
-## ⚙️ FEATURE EXTRACTION MODES (CRITICAL)
-
-Train and prediction **MUST** use same feature mode and dimension; metadata auto-detects this.
-
-| Mode | # Features | Description | Best For | Training Time |
-|---|---|---|---|---|
-| **simple** | 3 | NDVI_mean, VH_db_mean, VV_db_mean | Fast iteration, baseline | ~5-10 min |
-| **temporal** | 39 | NDVI/NDWI/NDBI across 13 months + radar stats | High accuracy (~85%+) | ~30-60 min |
-| **extended** | 15 | NDVI/NDWI/NDBI stats (mean/std/min/max) + radar | Balanced speed/accuracy | ~15-30 min |
-| **odc** | 8 | NDVI stats + NDWI/NDBI/EVI mean (legacy ODC mode) | Legacy compatibility | ~10-20 min |
-
-**Critical**: If feature mode = "temporal" (39 features) at training, prediction MUST extract 39 features. System auto-detects from metadata but will fail if mismatched.
-
----
-
-## 🎯 OPERATIONAL NOTES & CONSTRAINTS
-
-### Performance Limits
-1. **Planetary Computer Timeout Issues**
-   - Large bbox (>10km × 10km) + long time range (>1 month) + high max_scenes → timeouts
-   - **Solution**: Progressive loading (subdivide bbox), reduce time window, reduce max_scenes
-   - **Safe Settings**: bbox ≤ 10km × 10km, time ≤ 1 month, max_scenes ≤ 12
-
-2. **Memory Usage**
-   - Temporal mode (39 features) requires ~2-3x RAM vs simple mode
-   - Large regions: reduce resolution (10m → 20m) or split into sub-tiles
-   - Batch processing: sequential (one region at a time due to API limits)
-
-3. **GPU Training**
-   - Swin-UNet: ~15-60 min on GPU vs ~2-4 hours on CPU
-   - CNN: ~10-30 min on GPU vs ~1-2 hours on CPU
-   - XGBoost/RF: CPU-bound; GPU not beneficial
-
-### Data Quality Issues
-1. **Cloud Cover**
-   - SCL band values: 3=cloud shadow, 8=cloud medium, 9=cloud high, 10=cirrus
-   - Recommend multiple scenes (≥5) for temporal aggregation
-   - Cloud removal strategy critical—test different approaches
-
-2. **Radar Data (Sentinel-1)**
-   - Not always available for all regions/dates
-   - System gracefully falls back to zeros if unavailable
-   - Safe for "extended" & "odc" modes that have radar fallback
-
-3. **Feature Mode Mismatch**
-   - Model trained with "temporal" (39 features) needs 39-dim input
-   - System auto-adjusts (pads/trims) from metadata but may degrade accuracy
-   - **Best Practice**: Align feature mode explicitly; don't mix
-
-### Known Caveats
-1. **Legacy Model (model_odc.joblib)**: Hardcoded 39 temporal features; auto-detected via `model_odc_info.json`
-2. **Metadata Consistency**: Old models may lack `.json` sidecar; system generates default (may be incorrect)
-3. **Batch Processing**: Sequential only; large batches (100+ regions) take hours
-4. **Change Detection**: Simple differencing approach; requires same model & feature mode for both periods
-5. **Rate Limiting**: Planetary Computer may rate-limit if too many concurrent requests
-
-### Recommended Best Practices
-- Test model on small bbox first (2km × 2km, 1 week, 3 scenes)
-- Use "simple" mode for fast iteration, "temporal" for best accuracy (85%+)
-- Store metadata JSON alongside model file (sidecar pattern)
-- Version control: record feature_mode & n_features in every training
-- Monitor training accuracy; retrain if <70% accuracy
-- Cache Sentinel data locally to avoid repeated downloads
-- Use "median_composite" cloud strategy if >5 scenes; "temporal_only" if 3-4 scenes
-
----
-
-## 🔍 TEST COVERAGE MAP
-
-| Test File | Coverage | Status |
-|---|---|---|
-| `test_model_manager.py` | ModelManager lifecycle (list, load, validate) | ✅ Well-tested |
-| `test_feature_extractor.py` | All 4 feature extraction modes | ✅ Well-tested |
-| `test_training_api.py` | Training API endpoints | ✅ Partial |
-| `test_cloud_removal.py` | 7 cloud removal strategies | ✅ Well-tested |
-| `test_cloud_training.py` | U-Net cloud removal training | ✅ Partial |
-| `test_shapefile_api.py` | Shapefile overlay feature | ✅ Partial |
-| `test_planetary_computer.py` | Planetary Computer STAC access | ✅ Well-tested |
-| `test_new_features.py` | Recent feature releases | ✅ Partial |
-| Jupyter Notebooks | Training & prediction workflows | ✅ Mix of unit/integration/notebooks |
-
-**Coverage Notes**: Model management, feature extraction, and cloud removal well-tested; Dashboard UI, change detection, NDVI time-series mostly tested via notebooks.
-
----
-
-## 📚 FILE REFERENCE MAP
-
-### Core Execution
-- `api_server.py` — Main FastAPI application (~4200 LOC)
-- `train_module.py` — Training logic (data fetch → feature extraction → training)
-- `run_prediction_new.py` — Prediction execution function
-- `feature_extractor.py` — Unified feature extraction (4 modes)
-- `model_manager.py` — Model lifecycle (load/save/validate/list)
-- `cloud_removal.py` — Cloud removal strategies (7 methods)
-- `report_generator.py` — HTML/PNG report auto-generation
-- `generate_previews.py` — GeoTIFF → PNG conversion
-
-### Data & Config
-- `vietnam_provinces.py` — 32+ province lookup tables & bboxes
-- `vietnam_provinces_merged.py` — Alternative 32-province variant
-- `utils.py` — Geospatial utility functions
-- `create_odc_metadata.py` — Legacy metadata generator
-
-### Frontend
-- `index.html` — Main dashboard hub (tab navigation)
-- `training_interface.html` — Training configuration UI
-- `prediction_interface.html` — Prediction configuration UI
-- `batch_interface.html` — Batch processing (CSV upload)
-- `ndvi_interface.html` — NDVI time-series visualization
-- `dashboard.html` — Analytics & model performance dashboard
-- `reports_interface.html` — Report management & viewing
-- `change_detection_interface.html` — Change detection visualization
-- `cloud_training_interface.html` — Cloud removal U-Net training
-
-### Documentation
-- `IMPLEMENTATION_SUMMARY.md` — Model manager & system overview
-- `MODEL_MANAGER_GUIDE.md` — Complete model management guide
-- `NDVI_FORECAST_METHODOLOGY.md` — NDVI algorithm documentation
-- `CLOUD_TRAINING_GUIDE.md` — Cloud removal training guide
-- `NDVI_PREDICTION_GUIDE.md` — NDVI prediction workflow
-- `CLOUD_PROCESSING.md` — Cloud processing notes
-- `UPDATE_SUMMARY.md` — Recent updates & features
-
----
-
-## 🚀 BOOTSTRAP PROMPT FOR GEMINI
-
-### System Context (Copy & Paste for Gemini)
-
-```
-You are assisting a remote-sensing land-classification project for Vietnam.
-
-## ARCHITECTURE SNAPSHOT
-- **Backend**: FastAPI (~4200 LOC, 70+ endpoints) for orchestrating training, prediction, batch, cloud removal, reporting
-- **Data Source**: Microsoft Planetary Computer STAC API (Sentinel-2 L2A + Sentinel-1 RTC)
-- **Training**: scikit-learn (XGBoost/RF/SVM/DT) + PyTorch (CNN/Swin-UNet/MobileNet)
-- **Feature Extraction**: 4 modes (simple 3-feat / temporal 39-feat / extended 15-feat / odc 8-feat)
-- **Cloud Removal**: 7 strategies (classic, temporal_only, median_composite, none, ML inpainting, deep U-Net)
-- **Output**: GeoTIFF + PNG + HTML report + JSON metadata
-
-## CORE FILES TO UNDERSTAND (Priority Order)
-1. api_server.py — Main API server (training, prediction, batch, models, reports)
-2. train_module.py — Training pipeline (data fetch → feature extraction → train → save)
-3. feature_extractor.py — Unified feature extraction with auto mode detection
-4. model_manager.py — Model lifecycle (load/save/validate/list)
-5. cloud_removal.py — Cloud removal strategies (7 methods)
-6. report_generator.py — Auto-generate HTML reports
-7. run_prediction_new.py — Prediction execution
-8. vietnam_provinces.py — Province lookup & bbox tables
-
-## CRITICAL CONSTRAINTS & GOTCHAS
-1. **Feature Mode Consistency**: Training & prediction MUST use same mode (simple/temporal/extended/odc)
-   → Auto-detected from metadata JSON
-   → Mismatch causes dimension error or accuracy degradation
-
-2. **Planetary Computer Limits**:
-   → Timeout if bbox >10km×10km OR time range >1 month OR max_scenes >12
-   → Solution: subdivide bbox, reduce time window, limit scenes
-
-3. **Cloud Strategy Selection**:
-   → ≥5 scenes → use "median_composite" (best noise reduction)
-   → 3-4 scenes → use "temporal_only" (fast temporal interp)
-   → <3 scenes → use "none" (skip cloud removal)
-
-4. **Radar Data Fallback**:
-   → Sentinel-1 may be unavailable for some regions
-   → System gracefully falls back to zeros (safe for all modes)
-
-5. **Model Metadata**:
-   → Always stored as `model_name_info.json` sidecar file
-   → Contains: n_features, feature_mode, features list, accuracy, bbox, time_range
-   → Missing metadata → system uses defaults (may be incorrect)
-
-6. **Legacy Model (model_odc.joblib)**:
-   → Hardcoded 39 temporal features
-   → Metadata in model_odc_info.json
-
-## REASONING CHECKLIST (before answering)
-□ Is feature_mode consistent between train and prediction?
-□ Is metadata.json present and correct?
-□ Does bbox exceed 10km×10km? (Planetary Computer timeout risk)
-□ Is cloud_removal_strategy appropriate for # of scenes?
-□ Is Sentinel-1 data available for this region/date?
-□ Is model a joblib (scikit-learn) or .pth (PyTorch) file?
-□ Is GPU available for deep models (CNN, Swin-UNet)?
-□ Does memory allow temporal feature extraction (39-feat)?
-
-## RESPONSE FORMAT
-- Always cite api_server.py endpoint, function name, or module being discussed
-- Verify feature_mode & n_features from metadata JSON
-- Suggest cloud_removal_strategy based on # of scenes available
-- For unknown issues: offer alternative approaches (reduce bbox, cache results, use simpler model)
-- Explain reasoning using checklist above
-
-## DATA FLOW SUMMARY
-Sentinel-2/S1 → [Cloud Remove] → [Feature Extract] → [Train/Predict] → [GeoTIFF + PNG + Report]
-```
-
----
-
-## 📞 QUICK REFERENCE CHECKLIST
-
-### Before Troubleshooting Any Issue
-- [ ] Check feature_mode consistency (metadata JSON)
-- [ ] Verify metadata.json exists for the model
-- [ ] Check Planetary Computer connectivity (`GET /api/network/check`)
-- [ ] Review cloud_removal_method choice (≥5 scenes = median_composite)
-- [ ] Confirm Sentinel-1 availability (or fallback to zeros if missing)
-- [ ] Validate bbox size (≤10km×10km for safety)
-- [ ] Check memory usage for temporal feature mode
-- [ ] Verify GPU if using CNN/Swin-UNet models
-
-### Common Issues & Solutions
-| Issue | Likely Cause | Solution |
-|---|---|---|
-| Training timeout | Large bbox / long time / many scenes | Subdivide bbox, reduce time window, max_scenes ≤ 12 |
-| Feature dimension mismatch | Different feature_mode between train & predict | Check metadata.json, ensure same mode |
-| Low prediction accuracy | Cloud cover, poor training data, feature mode too simple | Use "temporal" mode, increase training data, try cloud removal |
-| Out of memory | Temporal features + large region | Reduce resolution (20m), split into sub-tiles, increase RAM |
-| Model not found | Wrong filename or model_train/ path issue | `GET /api/models/list` to verify, check file path |
-| Planetary Computer error | Network issue or API rate limit | Check DNS, retry later, reduce concurrent requests |
-| Cloud removal failing | Strategy not suitable for scene count | Try "none" or "median_composite" depending on scenes |
-
----
-
-## 🎓 LEARNING RESOURCES IN REPO
-
-- **Notebooks**: `01.train_ODC.ipynb`, `02.predict_ODC.ipynb`, `cloud_removal_train.ipynb`
-- **Tests**: `test_*.py` files for unit test patterns
-- **Docs**: All `*.md` files for detailed guides and methodology
-- **Code Comments**: API server and modules heavily commented
-
----
-
-**Generated**: March 26, 2026  
-**For Use By**: Gemini, Claude, GPT, or any AI system needing project context  
-**Maintainer**: Remote-Sensing Project Team
-

api_server.py

@@ -224,9 +224,6 @@ class PredictionConfig(BaseModel):
     # Cloud removal strategy
     cloud_removal_method: str = "classic"
     cloud_removal_model: Optional[str] = None  # Optional: .pth model filename for deep learning cloud removal
-    
-    # Shapefile overlay for visualization
-    shapefile_overlay: Optional[str] = None  # Path to shapefile for overlaying boundaries
 
 
 class TrainingStatus(BaseModel):
@@ -289,7 +286,6 @@ class PredictionWithNDVIConfig(BaseModel):
     export_classification: bool = True  # Export classification raster
     cloud_removal_method: str = "classic"
     cloud_removal_model: Optional[str] = None  # Optional: .pth model filename for deep learning cloud removal
-    shapefile_overlay: Optional[str] = None  # Path to shapefile for overlaying boundaries
 
 
 class CloudRemovalTrainingConfig(BaseModel):
@@ -1232,87 +1228,6 @@ async def list_training_files():
     }
 
 
-@app.get("/api/overlay/shapefiles")
-async def list_overlay_shapefiles():
-    """Liệt kê các shapefile có sẵn cho overlay trên prediction"""
-    overlay_dirs = ["region", "ChauThanh", "ThuanHoa"]
-    shapefiles = []
-    
-    for overlay_dir in overlay_dirs:
-        dir_path = Path(overlay_dir)
-        if not dir_path.exists():
-            continue
-            
-        # Find all .shp files in this directory and subdirectories
-        for shp_file in dir_path.rglob("*.shp"):
-            try:
-                file_size = shp_file.stat().st_size
-                file_modified = datetime.fromtimestamp(shp_file.stat().st_mtime).isoformat()
-                
-                # Try to read shapefile to get feature count and bbox
-                import geopandas as gpd
-                gdf = gpd.read_file(str(shp_file))
-                feature_count = len(gdf)
-                
-                # Calculate bbox (always in EPSG:4326 for consistency)
-                bbox = None
-                if not gdf.empty and gdf.crs:
-                    try:
-                        # Reproject to EPSG:4326 if needed
-                        if gdf.crs != "EPSG:4326":
-                            gdf_4326 = gdf.to_crs("EPSG:4326")
-                        else:
-                            gdf_4326 = gdf
-                        
-                        # Get total bounds [minx, miny, maxx, maxy]
-                        bounds = gdf_4326.total_bounds
-                        if len(bounds) == 4:
-                            bbox = [
-                                float(bounds[0]),  # min_lon
-                                float(bounds[1]),  # min_lat
-                                float(bounds[2]),  # max_lon
-                                float(bounds[3])   # max_lat
-                            ]
-                    except Exception as bbox_error:
-                        print(f"[WARNING] Cannot calculate bbox for {shp_file}: {bbox_error}")
-                
-                # Get relative path from workspace root
-                relative_path = str(shp_file)
-                
-                shapefiles.append({
-                    "filename": shp_file.name,
-                    "path": relative_path,
-                    "directory": overlay_dir,
-                    "size_bytes": file_size,
-                    "size_mb": round(file_size / 1024 / 1024, 2),
-                    "modified": file_modified,
-                    "feature_count": feature_count,
-                    "crs": str(gdf.crs) if gdf.crs else "Unknown",
-                    "bbox": bbox  # [min_lon, min_lat, max_lon, max_lat] in EPSG:4326
-                })
-            except Exception as e:
-                # If cannot read shapefile, just add basic info
-                file_size = shp_file.stat().st_size
-                file_modified = datetime.fromtimestamp(shp_file.stat().st_mtime).isoformat()
-                relative_path = str(shp_file)
-                
-                shapefiles.append({
-                    "filename": shp_file.name,
-                    "path": relative_path,
-                    "directory": overlay_dir,
-                    "size_bytes": file_size,
-                    "size_mb": round(file_size / 1024 / 1024, 2),
-                    "modified": file_modified,
-                    "error": f"Cannot read shapefile: {str(e)}"
-                })
-    
-    return {
-        "shapefiles": shapefiles,
-        "count": len(shapefiles),
-        "directories": overlay_dirs
-    }
-
-
 @app.get("/api/training/shapefile/{filename}/labels")
 async def get_shapefile_labels(filename: str):
     """Lấy các label từ một shapefile cụ thể"""
@@ -1814,85 +1729,6 @@ def update_progress(message: str):
     print(f"[PROGRESS] {message}")
 
 
-def rasterize_shapefile_overlay(shapefile_path, reference_raster, boundary_value=255):
-    """
-    Rasterize shapefile boundaries to overlay on prediction result.
-    
-    Args:
-        shapefile_path: Path to shapefile
-        reference_raster: xarray DataArray to match dimensions and CRS
-        boundary_value: Value to use for boundaries (default 255 for white)
-    
-    Returns:
-        numpy array with boundaries, same shape as reference_raster
-    """
-    try:
-        import geopandas as gpd
-        from rasterio.features import rasterize
-        import numpy as np
-        
-        # Read shapefile
-        gdf = gpd.read_file(shapefile_path)
-        print(f"[OVERLAY] Loaded shapefile with {len(gdf)} features, CRS: {gdf.crs}")
-        
-        # Ensure CRS matches
-        target_crs = reference_raster.rio.crs
-        if gdf.crs != target_crs:
-            print(f"[OVERLAY] Reprojecting from {gdf.crs} to {target_crs}")
-            gdf = gdf.to_crs(target_crs)
-        
-        # Get raster dimensions and transform
-        height, width = reference_raster.shape
-        transform = reference_raster.rio.transform()
-        print(f"[OVERLAY] Raster dimensions: {height}x{width}")
-        print(f"[OVERLAY] Transform: {transform}")
-        
-        # Calculate appropriate buffer size based on pixel resolution
-        # Get pixel size from transform (transform[0] is x resolution)
-        pixel_size = abs(transform[0])  # in CRS units
-        # Very thin boundary - only 0.2 pixels wide for 1px line
-        buffer_distance = pixel_size * 0.2
-        print(f"[OVERLAY] Pixel size: {pixel_size}, Buffer distance: {buffer_distance} (thin 1px line)")
-        
-        # Create boundary geometries with minimal buffering
-        boundary_geoms = []
-        for idx, geom in enumerate(gdf.geometry):
-            if geom is not None and geom.is_valid:
-                # Get boundary of each polygon
-                boundary = geom.boundary
-                if boundary is not None:
-                    # Minimal buffer for 1-pixel thin line
-                    buffered = boundary.buffer(buffer_distance)
-                    boundary_geoms.append((buffered, boundary_value))
-        
-        if not boundary_geoms:
-            print(f"[WARNING] No valid boundary geometries found in {shapefile_path}")
-            return np.zeros((height, width), dtype=np.uint8)
-        
-        print(f"[OVERLAY] Rasterizing {len(boundary_geoms)} boundaries...")
-        
-        # Rasterize boundaries
-        boundary_mask = rasterize(
-            shapes=boundary_geoms,
-            out_shape=(height, width),
-            transform=transform,
-            fill=0,  # Background
-            dtype=np.uint8
-        )
-        
-        boundary_count = np.count_nonzero(boundary_mask)
-        print(f"[OVERLAY] Boundary pixels: {boundary_count} / {height*width} ({boundary_count/(height*width)*100:.2f}%)")
-        
-        if boundary_count == 0:
-            print(f"[OVERLAY WARNING] No boundary pixels were rasterized! Check CRS and geometry overlap.")
-        
-        return boundary_mask
-        
-    except Exception as e:
-        print(f"[ERROR] Failed to rasterize shapefile {shapefile_path}: {e}")
-        return None
-
-
 def update_prediction_progress(message: str):
     """Cập nhật prediction progress message"""
     global prediction_status
@@ -2247,33 +2083,6 @@ async def run_prediction(config: PredictionConfig):
         
         prediction_da.rio.to_raster(str(output_file), driver="GTiff")
         
-        # ============ SHAPEFILE OVERLAY ============
-        overlay_mask = None
-        if config.shapefile_overlay:
-            prediction_status["progress"] = f"Đang overlay shapefile: {config.shapefile_overlay}..."
-            print(f"[OVERLAY] Shapefile overlay requested: {config.shapefile_overlay}")
-            
-            # Validate shapefile path exists
-            shapefile_path = Path(config.shapefile_overlay)
-            if not shapefile_path.exists():
-                print(f"[OVERLAY ERROR] Shapefile not found: {shapefile_path}")
-                print(f"[OVERLAY ERROR] Absolute path: {shapefile_path.absolute()}")
-                print(f"[OVERLAY ERROR] Current working directory: {Path.cwd()}")
-            else:
-                print(f"[OVERLAY] Shapefile exists: {shapefile_path.absolute()}")
-                try:
-                    overlay_mask = rasterize_shapefile_overlay(str(shapefile_path), prediction_da)
-                    if overlay_mask is not None and np.count_nonzero(overlay_mask) > 0:
-                        print(f"[OVERLAY] Successfully rasterized shapefile boundaries ({np.count_nonzero(overlay_mask)} pixels)")
-                    else:
-                        print(f"[OVERLAY WARNING] Shapefile rasterized but no boundary pixels found")
-                except Exception as overlay_error:
-                    print(f"[OVERLAY ERROR] Exception: {overlay_error}")
-                    import traceback
-                    traceback.print_exc()
-        else:
-            print(f"[OVERLAY] No shapefile overlay requested")
-        
         # Generate PNG preview for web display
         prediction_status["progress"] = "Đang tạo PNG preview..."
         png_file = output_dir / f"prediction_{timestamp}.png"
@@ -2288,42 +2097,6 @@ async def run_prediction(config: PredictionConfig):
             
             # Plot prediction with colormap
             im = ax.imshow(predictions_2d, cmap='tab20', interpolation='nearest')
-            
-            # Overlay shapefile boundaries if available
-            if overlay_mask is not None and np.count_nonzero(overlay_mask) > 0:
-                print(f"[PNG OVERLAY] Overlaying {np.count_nonzero(overlay_mask)} boundary pixels")
-                
-                # Create a mask for boundaries (where overlay_mask > 0)
-                boundary_mask = overlay_mask > 0
-                
-                # Method: Direct overlay with high-contrast colors
-                # Create RGBA overlay image
-                overlay_rgba = np.zeros((*predictions_2d.shape, 4))
-                overlay_rgba[boundary_mask, 0] = 1.0  # Red = 1.0 (white)
-                overlay_rgba[boundary_mask, 1] = 1.0  # Green = 1.0 (white)
-                overlay_rgba[boundary_mask, 2] = 1.0  # Blue = 1.0 (white)
-                overlay_rgba[boundary_mask, 3] = 1.0  # Alpha = 1.0 (fully opaque)
-                
-                # Overlay on top of prediction
-                ax.imshow(overlay_rgba, interpolation='nearest')
-                
-                # Also add a black outline for better contrast
-                from scipy import ndimage
-                boundary_dilated = ndimage.binary_dilation(boundary_mask, iterations=1)
-                boundary_outline = boundary_dilated & ~boundary_mask
-                
-                outline_rgba = np.zeros((*predictions_2d.shape, 4))
-                outline_rgba[boundary_outline, 0] = 0.0  # Black outline
-                outline_rgba[boundary_outline, 1] = 0.0
-                outline_rgba[boundary_outline, 2] = 0.0
-                outline_rgba[boundary_outline, 3] = 0.8
-                
-                ax.imshow(outline_rgba, interpolation='nearest')
-                
-                print(f"[PNG OVERLAY] Added shapefile boundaries to visualization (direct overlay method)")
-            else:
-                print(f"[PNG OVERLAY] No overlay mask or empty mask (pixels: {np.count_nonzero(overlay_mask) if overlay_mask is not None else 0})")
-            
             ax.set_title(f'Land Classification - {timestamp}', fontsize=16, fontweight='bold', pad=20)
             ax.set_xlabel('X (pixels)', fontsize=11)
             ax.set_ylabel('Y (pixels)', fontsize=11)
@@ -2403,9 +2176,7 @@ async def run_prediction(config: PredictionConfig):
             "n_features": features.shape[1],
             "feature_mode": feature_mode,
             "used_radar": use_radar,
-            "model_used": config.model_filename,
-            "shapefile_overlay": config.shapefile_overlay,
-            "overlay_applied": overlay_mask is not None
+            "model_used": config.model_filename
         }
         
         # Auto generate prediction report
@@ -4584,69 +4355,8 @@ async def predict_with_ndvi(config: PredictionWithNDVIConfig, background_tasks:
                 import matplotlib.pyplot as plt
                 from matplotlib.patches import Patch
                 
-                # ============ SHAPEFILE OVERLAY ============
-                overlay_mask = None
-                if config.shapefile_overlay:
-                    print(f"[OVERLAY] Shapefile overlay requested: {config.shapefile_overlay}")
-                    
-                    # Validate shapefile path exists
-                    shapefile_path = Path(config.shapefile_overlay)
-                    if not shapefile_path.exists():
-                        print(f"[OVERLAY ERROR] Shapefile not found: {shapefile_path}")
-                        print(f"[OVERLAY ERROR] Absolute path: {shapefile_path.absolute()}")
-                    else:
-                        print(f"[OVERLAY] Shapefile exists: {shapefile_path.absolute()}")
-                        try:
-                            # Import rioxarray for rio accessor
-                            import rioxarray
-                            
-                            # Create temporary DataArray for rasterization
-                            temp_da = xr.DataArray(
-                                prediction_raster,
-                                coords={
-                                    "y": np.linspace(bbox[3], bbox[1], height),
-                                    "x": np.linspace(bbox[0], bbox[2], width)
-                                },
-                                dims=["y", "x"]
-                            )
-                            temp_da.rio.write_crs("EPSG:4326", inplace=True)
-                            temp_da.rio.write_transform(transform, inplace=True)
-                            
-                            overlay_mask = rasterize_shapefile_overlay(str(shapefile_path), temp_da)
-                            if overlay_mask is not None and np.count_nonzero(overlay_mask) > 0:
-                                print(f"[OVERLAY] Successfully rasterized shapefile boundaries ({np.count_nonzero(overlay_mask)} pixels)")
-                            else:
-                                print(f"[OVERLAY WARNING] Shapefile rasterized but no boundary pixels found")
-                        except Exception as overlay_error:
-                            print(f"[OVERLAY ERROR] Exception: {overlay_error}")
-                            import traceback
-                            traceback.print_exc()
-                else:
-                    print(f"[OVERLAY] No shapefile overlay requested")
-                
                 fig, ax = plt.subplots(figsize=(14, 10), dpi=150)
                 im = ax.imshow(prediction_raster, cmap='tab20', interpolation='nearest')
-                
-                # Overlay shapefile boundaries if available
-                if overlay_mask is not None and np.count_nonzero(overlay_mask) > 0:
-                    print(f"[PNG OVERLAY] Overlaying {np.count_nonzero(overlay_mask)} boundary pixels (1px thin line)")
-                    
-                    # Create a mask for boundaries
-                    boundary_mask = overlay_mask > 0
-                    
-                    # Create RGBA overlay image - thin 1px white line only
-                    overlay_rgba = np.zeros((*prediction_raster.shape, 4))
-                    overlay_rgba[boundary_mask, 0] = 1.0  # White (R=1)
-                    overlay_rgba[boundary_mask, 1] = 1.0  # White (G=1)
-                    overlay_rgba[boundary_mask, 2] = 1.0  # White (B=1)
-                    overlay_rgba[boundary_mask, 3] = 1.0  # Fully opaque
-                    
-                    ax.imshow(overlay_rgba, interpolation='nearest')
-                    
-                    print(f"[PNG OVERLAY] Added thin 1px shapefile boundaries to visualization")
-                else:
-                    print(f"[PNG OVERLAY] No overlay mask or empty mask")
-                
                 ax.set_title(f'Land Classification - {timestamp}', fontsize=16, fontweight='bold', pad=20)
                 ax.set_xlabel('X (pixels)', fontsize=11)
                 ax.set_ylabel('Y (pixels)', fontsize=11)

fix_all.py

@@ -0,0 +1,25 @@
+import glob, json
+
+changed_files = []
+for file_path in glob.glob('*.ipynb'):
+    with open(file_path, 'r', encoding='utf-8') as f:
+        nb = json.load(f)
+        
+    changed = False
+    for cell in nb.get('cells', []):
+        if cell.get('cell_type') == 'code':
+            source = cell.get('source', [])
+            for i, line in enumerate(source):
+                if 'time=50' in line:
+                    source[i] = line.replace('time=50', 'time=0')
+                    changed = True
+                if 'load_data_sen1(dc,' in line:
+                    source[i] = line.replace('load_data_sen1(dc,', 'load_data_sen1(None,')
+                    changed = True
+                    
+    if changed:
+        with open(file_path, 'w', encoding='utf-8') as f:
+            json.dump(nb, f, indent=1)
+        changed_files.append(file_path)
+
+print('Fixed issues in:', changed_files)

fix_new_train.py

@@ -0,0 +1,20 @@
+import json
+
+def fix_import(file_path):
+    with open(file_path, 'r', encoding='utf-8') as f:
+        nb = json.load(f)
+    changed = False
+    for cell in nb.get('cells', []):
+        if cell.get('cell_type') == 'code':
+            source = cell.get('source', [])
+            if isinstance(source, list):
+                for i, line in enumerate(source):
+                    if "from new_import import *" in line:
+                        source[i] = line.replace("from new_import import *", "from new_import_ODC import *")
+                        changed = True
+    if changed:
+        with open(file_path, 'w', encoding='utf-8') as f:
+            json.dump(nb, f, indent=1)
+        print(f"Fixed {file_path}")
+
+fix_import('new_train.ipynb')

fix_typo.py

@@ -0,0 +1,22 @@
+import json
+
+def fix_filename(file_path):
+    with open(file_path, 'r', encoding='utf-8') as f:
+        nb = json.load(f)
+    changed = False
+    for cell in nb.get('cells', []):
+        if cell.get('cell_type') == 'code':
+            source = cell.get('source', [])
+            if isinstance(source, list):
+                for i, line in enumerate(source):
+                    if "ST_training data_updated_1130points.shp" in line:
+                        source[i] = line.replace("ST_training data_updated_1130points.shp", "ST_training_data_updated_1130points.shp")
+                        changed = True
+    if changed:
+        with open(file_path, 'w', encoding='utf-8') as f:
+            json.dump(nb, f, indent=1)
+        print(f"Fixed typo in {file_path}")
+
+import glob
+for nb in glob.glob("*.ipynb"):
+    fix_filename(nb)

inspect_cells_train.py

@@ -0,0 +1,7 @@
+import json
+nb = json.load(open('01.train_ODC.ipynb'))
+for idx, cell in enumerate(nb['cells']):
+    if cell['cell_type'] == 'code':
+        print(f"Cell {idx}:")
+        print("".join(cell['source'][:3]))
+        print("-" * 20)

microsoft_planetary_migration.md

@@ -0,0 +1,105 @@
+# Hướng dẫn Chuyển đổi dữ liệu vệ tinh sang Microsoft Planetary Computer STAC
+
+Tài liệu này ghi chú lại các bước chuẩn hóa và các đoạn code mẫu để chuyển đổi việc tải dữ liệu vệ tinh (Sentinel-1, Sentinel-2) từ kho lưu trữ đóng (như AWS S3 yêu cầu xác thực) sang nền tảng mở **Microsoft Planetary Computer STAC API**. Bạn có thể dùng tài liệu này làm context (ngữ cảnh) gửi cho các AI khác để chúng hiểu cách thực hiện tương tự.
+
+---
+
+## 1. Mục đích
+- Bỏ qua các lỗi liên quan đến xác thực đám mây (VD: `RasterioIOError: AWS_SECRET_ACCESS_KEY not defined`).
+- Tải dữ liệu miễn phí, trực tiếp từ kho dữ liệu mở của Microsoft Planetary Computer.
+- Đảm bảo đầu ra (output) của dữ liệu STAC giống hệt với định dạng của ảnh TIF gốc tải bằng `rioxarray` để không làm hỏng các luồng xử lý Machine Learning ở phía sau.
+
+## 2. Các thư viện bắt buộc (Dependencies)
+Đảm bảo môi trường Python có cài đặt các thư viện sau:
+```python
+import pystac_client
+import planetary_computer
+import odc.stac
+import xarray as xr
+import rioxarray
+```
+
+## 3. Các bước thực hiện chi tiết
+
+### Bước 1: Kết nối đến STAC API và truy vấn dữ liệu
+Thay vì dùng `rioxarray.open_rasterio("s3://...")`, chúng ta khởi tạo STAC Client và tìm kiếm dữ liệu theo tọa độ (`bbox`) và thời gian (`datetime`).
+
+```python
+# 1. Kết nối STAC Client có kèm chữ ký xác thực (sign_inplace) của Microsoft
+catalog = pystac_client.Client.open(
+    "https://planetarycomputer.microsoft.com/api/stac/v1",
+    modifier=planetary_computer.sign_inplace,
+)
+
+# 2. Định nghĩa toạ độ và thời gian
+bbox = [105.5, 9.2, 106.4, 10.0] # [min_lon, min_lat, max_lon, max_lat]
+datetime = "2022-09-01/2023-10-01"
+
+# 3. Tìm kiếm Items
+# Thay "sentinel-1-rtc" bằng "sentinel-2-l2a" nếu tải ảnh quang học
+search = catalog.search(
+    collections=["sentinel-1-rtc"],
+    bbox=bbox,
+    datetime=datetime,
+)
+items = list(search.items())
+```
+
+### Bước 2: Tải dữ liệu xuống xarray bằng `odc.stac`
+Thay vì tải thủ công từng link URL, `odc.stac.load` sẽ tự động tải, cắt ảnh theo `bbox`, đổi hệ tọa độ (reproject) và ghép lại thành một khối dữ liệu không gian - thời gian (DataCube).
+
+```python
+# Tải dữ liệu thành xarray Dataset
+ds_s1 = odc.stac.load(
+    items,
+    bands=["vv", "vh"],           # Tên các band cần tải
+    bbox=bbox,
+    crs="EPSG:32648",             # Ép về hệ toạ độ đích (VD: UTM Zone 48N cho VN)
+    resolution=10,                # Độ phân giải (10 mét)
+    chunks={"x": 2048, "y": 2048, "time": 1} # Dùng Dask chunking để tránh tràn RAM
+)
+```
+
+### Bước 3: Nén trục thời gian (Temporal Compositing)
+Dữ liệu từ STAC sẽ có 3 chiều: `(time, y, x)`. Do ảnh TIF gốc cũ thường là ảnh đã được nén (ví dụ trung bình của 1 năm), ta cần dùng phép tính trung vị (`median`) hoặc trung bình (`mean`) để triệt tiêu trục `time`, biến dữ liệu thành dạng 2D `(y, x)`.
+
+```python
+# Tính giá trị trung vị theo thời gian
+ds_median = ds_s1.median(dim="time").compute()
+
+# Tách riêng các DataArray
+vv = ds_median["vv"]
+vh = ds_median["vh"]
+```
+
+### Bước 4: Khôi phục cấu trúc DataArray gốc (Mimic rioxarray)
+Hàm `rioxarray.open_rasterio` gốc luôn trả về dữ liệu có trục `band` (kích thước = 1). Để code Machine Learning bên dưới không bị lỗi "out of bounds" hay "missing dimension", ta phải thêm trục `band` giả và gán lại thông tin `crs`.
+
+```python
+# Thêm chiều 'band' để giống hệt rioxarray
+vv = vv.expand_dims(dim="band")
+vh = vh.expand_dims(dim="band")
+
+# Phục hồi metadata về toạ độ
+vv = vv.rio.write_crs("EPSG:32648")
+vh = vh.rio.write_crs("EPSG:32648")
+```
+
+### Bước 5: Quét và sửa các đoạn code "Hardcode" kích thước
+Do lưới tọa độ của STAC tự sinh (dựa trên bounding box) có thể lệch vài pixel so với lưới của file TIF đã cắt tay trên S3 (VD: S3 là `8874 x 9902`, STAC là `8870 x 9900`), **phải tìm và xóa bỏ toàn bộ các con số fix cứng trong mảng**.
+
+*Code cũ sai lầm:*
+```python
+tmp = np.ones((8874, 9902))
+final_label = final_label.reshape(8874, 9902)
+```
+
+*Code chuẩn hóa:*
+```python
+# Lấy linh động theo shape thực tế của xarray
+tmp = np.ones((ds_vhvv.shape[1], ds_vhvv.shape[2]))
+final_label = final_label.reshape(ds_vhvv.shape[1], ds_vhvv.shape[2])
+```
+
+## 4. Tổng kết
+Chỉ cần cung cấp tài liệu này cho bất kỳ AI nào, yêu cầu: *"Hãy refactor (viết lại) hàm load file TIF của tôi theo đúng 5 bước trong tài liệu Microsoft Planetary Computer này"*, AI đó sẽ có đủ toàn bộ tư duy và code mẫu để hoàn thành công việc một cách mượt mà nhất.

model_xgboost_info.json

@@ -0,0 +1,31 @@
+{
+  "model_type": "XGBoost",
+  "num_classes": 8,
+  "classes": [
+    "Lua tom",
+    "Lua",
+    "CHN",
+    "CLN",
+    "TS",
+    "Song",
+    "Dat xay dung",
+    "Rung"
+  ],
+  "num_features": 3,
+  "params": {
+    "objective": "multi:softmax",
+    "num_class": 8,
+    "max_depth": 6,
+    "learning_rate": 0.1,
+    "n_estimators": 200,
+    "subsample": 0.8,
+    "colsample_bytree": 0.8,
+    "random_state": 42,
+    "n_jobs": -1,
+    "eval_metric": "mlogloss"
+  },
+  "accuracy": 0.28761061946902655,
+  "precision": 0.35339400643604185,
+  "recall": 0.28761061946902655,
+  "f1_score": 0.23460742664282486
+}

new_import_ODC.py

@@ -1,8 +1,16 @@
+TEST_MODE = True
+RESOLUTION = 1000 if TEST_MODE else 10
 import matplotlib.pyplot as plt
 
 # Common imports and settings
 import os, sys
 os.environ['USE_PYGEOS'] = '0'
+os.environ["GDAL_HTTP_MAX_RETRY"] = "5"
+os.environ["GDAL_HTTP_RETRY_DELAY"] = "2"
+os.environ["GDAL_HTTP_CONNECTION_TIMEOUT"] = "10"
+os.environ["GDAL_HTTP_TIMEOUT"] = "30"
+os.environ["CPL_VSIL_CURL_ALLOWED_EXTENSIONS"] = ".tif,.tiff"
+os.environ["GDAL_DISABLE_READDIR_ON_OPEN"] = "YES"
 from IPython.display import Markdown
 import pandas as pd
 pd.set_option("display.max_rows", None)
@@ -13,15 +21,13 @@ import datacube
 from datacube.utils.rio import configure_s3_access
 from datacube.utils import masking
 from datacube.utils.cog import write_cog
-# https://github.com/GeoscienceAustralia/dea-notebooks/tree/develop/Tools
-from dea_tools.plotting import display_map, rgb
-from dea_tools.datahandling import mostcommon_crs
+# removed deafrica_tools imports to avoid ipyleaflet error
 
 # EASI defaults
-easinotebooksrepo = '/home/jovyan/easi-notebooks'
+easinotebooksrepo = '/home/x79/CSIROBoeingPhase4-Vietnam'
 if easinotebooksrepo not in sys.path: sys.path.append(easinotebooksrepo)
 from easi_tools import EasiDefaults, xarray_object_size, notebook_utils, unset_cachingproxy
-from easi_tools.load_s2l2a import load_s2l2a_with_offset
+# from easi_tools.load_s2l2a import load_s2l2a_with_offset
 from dask.distributed import progress
 
 # Data tools
@@ -31,7 +37,7 @@ from datetime import datetime
 # Datacube
 from datacube.utils import masking  # https://github.com/opendatacube/datacube-core/blob/develop/datacube/utils/masking.py
 from odc.algo import enum_to_bool   # https://github.com/opendatacube/odc-algo/blob/main/odc/algo/_masking.py
-from odc.algo import xr_reproject   # https://github.com/opendatacube/odc-algo/blob/main/odc/algo/_warp.py
+# removed xr_reproject
 from datacube.utils.geometry import GeoBox, box  # https://github.com/opendatacube/datacube-core/blob/develop/datacube/utils/geometry/_base.py
 
 # Holoviews, Datashader and Bokeh
@@ -83,56 +89,73 @@ import joblib
 
 def load_data(dc, date_range, longtitude_range, latitude_range):
     product = 's2_l2a'
-    query = {
-        'product': product,                     # Product name
-        'x': longtitude_range,    # "x" axis bounds
-        'y': latitude_range,      # "y" axis bounds
-        'time': date_range,           # Any parsable date strings
-    }
-    native_crs = notebook_utils.mostcommon_crs(dc, query)
-    print(f'Most common native CRS: {native_crs}')
-    measurements = ['red', 'nir', 'scl']
-
-    load_params = {
-        'measurements': measurements,                   # Selected measurement or alias names
-        'output_crs': native_crs,                       # Target EPSG code
-        'resolution': (-10, 10),                        # Target resolution
-        'group_by': 'solar_day',                        # Scene grouping
-        'dask_chunks': {'x': 2048, 'y': 2048},          # Dask chunks
-    }
-    data = load_s2l2a_with_offset(
-        dc,
-        query | load_params   # Combine the two dicts that contain our search and load parameters
+    bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
+    
+    import pystac_client
+    import planetary_computer
+    import odc.stac
+    
+    catalog = pystac_client.Client.open(
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
+        modifier=planetary_computer.sign_inplace,
     )
+    search = catalog.search(
+        collections=["sentinel-2-l2a"],
+        bbox=bbox,
+        datetime=f"{date_range[0]}/{date_range[1]}",
+    )
+    items = list(search.items())
+    
+    data = odc.stac.load(
+        items,
+        bands=["red", "nir", "SCL"],
+        bbox=bbox,
+        crs="EPSG:32648",
+        resolution=RESOLUTION,
+        chunks={"x": 2048, "y": 2048, "time": 1},
+        groupby="solar_day"
+    )
+    if "SCL" in data.data_vars:
+        data = data.rename({"SCL": "scl"})
     return data
 
 
 def mask_clean(data):
-    flag_name = 'scl'
-    flag_desc = masking.describe_variable_flags(data[flag_name])  # Pandas dataframe
-    display(flag_desc)
-    display(flag_desc.loc['qa'].values[1])
-    # Create a "data quality" Mask layer
-    flags_def = flag_desc.loc['qa'].values[1]
-    good_pixel_flags = [flags_def[str(i)] for i in [2, 4, 5, 6]]  # To pass strings to enum_to_bool()
-
-    # enum_to_bool calculates the pixel-wise "or" of each set of pixels given by good_pixel_flags
-    # 1 = good data
-    # 0 = "bad" data
-    good_pixel_mask = enum_to_bool(data[flag_name], good_pixel_flags)
+    # For Sentinel-2 L2A SCL: 
+    # 2: Dark Area Pixels, 4: Vegetation, 5: Not Vegetated, 6: Water
+    good_pixel_mask = data['scl'].isin([2, 4, 5, 6])
     data_layer_names = [x for x in data.data_vars if x != 'scl']
-    # Apply good pixel mask to blue, green, red and nir.
+    # Apply good pixel mask
     result = data[data_layer_names].where(good_pixel_mask).persist()
     return result
 
 
 def fill_nan(ndvi, time_split):
+    if len(ndvi.time) == 0:
+        return ndvi
+    
+    # If the total time duration is less than 90 days, skip seasonal splitting
+    try:
+        total_days = (ndvi.time[-1] - ndvi.time[0]).dt.days.item()
+        if total_days < 90:
+            return ndvi.bfill(dim="time").ffill(dim="time")
+    except Exception:
+        pass
+        
     rs = []
     for times in time_split:
-        tmp = ndvi.sel(time=times)
-        fill_ds = tmp.sel(time=times).bfill(dim='time')
-        fill_ds = fill_ds.sel(time=times).ffill(dim='time')   
-        rs.append(fill_ds)
+        try:
+            tmp = ndvi.sel(time=times)
+            if len(tmp.time) == 0:
+                continue
+            fill_ds = tmp.bfill(dim='time').ffill(dim='time')   
+            rs.append(fill_ds)
+        except Exception:
+            continue
+            
+    if len(rs) == 0:
+        return ndvi.bfill(dim="time").ffill(dim="time")
+        
     merged_ndvi = xr.concat([i for i in rs], dim="time")
     fill_m = merged_ndvi.bfill(dim="time")
     fill_m = fill_m.ffill(dim="time")
@@ -144,10 +167,49 @@ def load_train_data(train_path):
     return train
 
 
-def load_sen1(name_vh, name_vv):
-    dsvv = rioxarray.open_rasterio(name_vv)
-    dsvh = rioxarray.open_rasterio(name_vh)
-    return dsvh, dsvv
+def load_sen1(bbox, time_range):
+    import pystac_client
+    import planetary_computer
+    import odc.stac
+    
+    # Kết nối STAC Client
+    catalog = pystac_client.Client.open(
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
+        modifier=planetary_computer.sign_inplace,
+    )
+
+    # Tìm kiếm Items
+    search = catalog.search(
+        collections=["sentinel-1-rtc"],
+        bbox=bbox,
+        datetime=time_range,
+    )
+    items = list(search.items())
+
+    # Tải dữ liệu thành xarray Dataset
+    ds_s1 = odc.stac.load(
+        items,
+        bands=["vv", "vh"],
+        bbox=bbox,
+        crs="EPSG:32648",
+        resolution=RESOLUTION,
+        chunks={"x": 2048, "y": 2048, "time": 1}
+    )
+
+    # Tính giá trị trung vị theo thời gian
+    ds_median = ds_s1.median(dim="time").compute()
+    vv = ds_median["vv"]
+    vh = ds_median["vh"]
+
+    # Thêm chiều 'band' để giống hệt rioxarray
+    vv = vv.expand_dims(dim="band")
+    vh = vh.expand_dims(dim="band")
+
+    # Phục hồi metadata về toạ độ
+    vv = vv.rio.write_crs("EPSG:32648")
+    vh = vh.rio.write_crs("EPSG:32648")
+
+    return vh, vv
 
 
 def get_data_sen1_and_sen2(train, average_ndvi, dsvh, dsvv):
@@ -267,6 +329,10 @@ def save_model(name_file, model, metadata=None, label_encoder=None):
     
     
 def predict(model, data_crs, ndvi, vh, vv):
+    # Unpack model if it is wrapped in a dictionary (from ModelManager)
+    if isinstance(model, dict) and 'model' in model:
+        model = model['model']
+        
     data_predict = []
     for i in range(ndvi.shape[1]):
         ndvi_tmp = ndvi.isel(y=i).values
@@ -361,21 +427,35 @@ def save_result(result, HT_MAP):
         
 def load_data_sen1(dc, date_range, coordinates):
     longtitude_range, latitude_range = coordinates
-    data_sen1 = dc.load(
-        product="sentinel1_grd_gamma0_10m",
-        x=longtitude_range,
-        y=latitude_range,
-        time=date_range,
-        measurements=["vv", "vh"],
-        output_crs="EPSG:32648",
-        resolution=(-10,10),
-        dask_chunks={"x":2048, "y":2048},
-        skip_broken_datasets=True,
-        group_by='solar_day'
+    bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
+    
+    import pystac_client
+    import planetary_computer
+    import odc.stac
+    
+    catalog = pystac_client.Client.open(
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
+        modifier=planetary_computer.sign_inplace,
+    )
+    search = catalog.search(
+        collections=["sentinel-1-rtc"],
+        bbox=bbox,
+        datetime=f"{date_range[0]}/{date_range[1]}",
+    )
+    items = list(search.items())
+    
+    data_sen1 = odc.stac.load(
+        items,
+        bands=["vv", "vh"],
+        bbox=bbox,
+        crs="EPSG:32648",
+        resolution=RESOLUTION,
+        chunks={"x": 2048, "y": 2048, "time": 1},
+        groupby="solar_day"
     )
     
-    notebook_utils.heading(notebook_utils.xarray_object_size(data_sen1))
-    display(data_sen1)
+    # notebook_utils.heading(notebook_utils.xarray_object_size(data_sen1))
+    # display(data_sen1)
     dsvh = data_sen1.vh
     dsvv = data_sen1.vv
     
@@ -387,46 +467,42 @@ def calculate_average(data, time_pattern='1M'):
 
 def load_data_sen2(dc, date_range, coordinates):
     longtitude_range, latitude_range = coordinates
-    product = 's2_l2a'
-    query = {
-        'product': product,                     # Product name
-        'x': longtitude_range,    # "x" axis bounds
-        'y': latitude_range,      # "y" axis bounds
-        'time': date_range,           # Any parsable date strings
-    }
-    native_crs = notebook_utils.mostcommon_crs(dc, query)
-    print(f'Most common native CRS: {native_crs}')
+    bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
     
-    # measurements = ['red','green', 'blue', 'nir', 'scl']
-    measurements = ['red', 'nir', 'scl']
-
-    load_params = {
-        'measurements': measurements,                   # Selected measurement or alias names
-        'output_crs': native_crs,                       # Target EPSG code
-        'resolution': (-10, 10),                        # Target resolution
-        'group_by': 'solar_day',                        # Scene grouping
-        'dask_chunks': {'x': 2048, 'y': 2048},          # Dask chunks
-    }
-    data = load_s2l2a_with_offset(
-        dc,
-        query | load_params   # Combine the two dicts that contain our search and load parameters
+    import pystac_client
+    import planetary_computer
+    import odc.stac
+    
+    catalog = pystac_client.Client.open(
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
+        modifier=planetary_computer.sign_inplace,
     )
+    search = catalog.search(
+        collections=["sentinel-2-l2a"],
+        bbox=bbox,
+        datetime=f"{date_range[0]}/{date_range[1]}",
+    )
+    items = list(search.items())
+    
+    data = odc.stac.load(
+        items,
+        bands=["red", "nir", "SCL"],
+        bbox=bbox,
+        crs="EPSG:32648",
+        resolution=RESOLUTION,
+        chunks={"x": 2048, "y": 2048, "time": 1},
+        groupby="solar_day"
+    )
+    if "SCL" in data.data_vars:
+        data = data.rename({"SCL": "scl"})
     return data
 
 def mask_cloud(data):
-    flag_name = 'scl'
-    flag_desc = masking.describe_variable_flags(data[flag_name])  # Pandas dataframe
-    display(flag_desc.loc['qa'].values[1])
-    # Create a "data quality" Mask layer
-    flags_def = flag_desc.loc['qa'].values[1]
-    good_pixel_flags = [flags_def[str(i)] for i in [2, 4, 5, 6]]  # To pass strings to enum_to_bool()
-
-    # enum_to_bool calculates the pixel-wise "or" of each set of pixels given by good_pixel_flags
-    # 1 = good data
-    # 0 = "bad" data
-    good_pixel_mask = enum_to_bool(data[flag_name], good_pixel_flags)
+    # For Sentinel-2 L2A SCL: 
+    # 2: Dark Area Pixels, 4: Vegetation, 5: Not Vegetated, 6: Water
+    good_pixel_mask = data['scl'].isin([2, 4, 5, 6])
     data_layer_names = [x for x in data.data_vars if x != 'scl']
-    # Apply good pixel mask to blue, green, red and nir.
+    # Apply good pixel mask
     result = data[data_layer_names].where(good_pixel_mask).persist()
     return result
 

new_train.py

@@ -0,0 +1,264 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[49]:
+
+
+get_ipython().run_cell_magic('time', '', '%matplotlib inline\nfrom new_import import *\n')
+
+
+# In[2]:
+
+
+get_ipython().run_cell_magic('time', '', '# Cấu hình Daskgateway\ncluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,10))\n# Khai báo 1 Datacube là dc\ndc = datacube.Datacube()\n\n# Cấu hình truy cập dịch vụ S3\nconfigure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n\nclient\n')
+
+
+# LOAD VH, VV
+
+# In[47]:
+
+
+## cấu hình thời gian lấy ảnh và tọa độ
+date_range = ('2022-09-01', '2023-10-01')
+longtitude_range = (105.5, 106.4)
+latitude_range = (9.2, 10.0) 
+
+
+# In[3]:
+
+
+## cấu hình dữ liệu train và vh vv file
+train_path = "train/ST_training data_updated_1130points.shp"  # đường dẫn shp file train
+name_vh = "vh-0922_0923-full_ST.tif"
+name_vv = "vv-0922_0923-full_ST.tif"
+
+
+train = load_train_data(train_path)
+
+
+# In[4]:
+
+
+# %%time
+# ## tải về dữ liệu sen1
+# import os
+# if not os.path.exists(name_vh):
+#     !aws s3 cp s3://easi-asia-dc-data/staging/ctu/sentinel-1/vh-0922_0923-full_ST.tif vh-0922_0923-full_ST.tif
+# if not os.path.exists(name_vv):
+#     !aws s3 cp s3://easi-asia-dc-data/staging/ctu/sentinel-1/vv-0922_0923-full_ST.tif vv-0922_0923-full_ST.tif
+
+
+# In[5]:
+
+
+
+
+
+# In[38]:
+
+
+ds = dc.load(
+    product="sentinel1_grd_gamma0_20m",
+    x=(105.5, 106.4),
+    y=(9.2, 10.0),
+    time=("2022-09-01", "2023-10-01"),
+    measurements=["vv", "vh"],
+    output_crs="EPSG:32648",
+    resolution=(-10,10),
+    dask_chunks={"x":2048, "y":2048},
+    skip_broken_datasets=True,
+    group_by="solar_day"
+)
+notebook_utils.heading(notebook_utils.xarray_object_size(ds))
+ds
+
+
+# In[43]:
+
+
+vv_data = ds.vv
+vv_data
+
+
+# In[44]:
+
+
+bbox = [105.5, 9.2, 106.4, 10.0]
+time_range = "2022-09-01/2023-10-01"
+dsvh, dsvv = load_sen1(bbox, time_range)
+dsvv
+
+
+# LOAD SENTINEL 2
+# 
+# 
+
+# In[50]:
+
+
+data = load_data(dc, date_range, longtitude_range, latitude_range)
+notebook_utils.heading(notebook_utils.xarray_object_size(data))
+display(data)
+
+
+# In[8]:
+
+
+get_ipython().run_cell_magic('time', '', '# Tiến hành loại bỏ các vị trí bị mây ảnh hưởng\nresult = mask_clean(data)\nprogress(result)\n')
+
+
+# CALCULATING THE MEAN VALUE AND FILL TO NAN POINT
+
+# In[9]:
+
+
+ds1 = calculate_indices(result, index='NDVI', satellite_mission='s2')
+ndvi = ds1["NDVI"]
+average_ndvi = ndvi.resample(time='1M').mean().persist()  ## tính mean cho từng tháng -> time = 12
+progress(average_ndvi)
+
+
+# In[10]:
+
+
+dsvh.shape
+
+
+# In[11]:
+
+
+average_ndvi = average_ndvi.compute()
+average_ndvi = average_ndvi[:, :dsvh.shape[1], :dsvh.shape[2]]
+
+
+# In[12]:
+
+
+get_ipython().run_cell_magic('time', '', "filled_ds = average_ndvi.bfill(dim='time')\nfilled_ds = filled_ds.ffill(dim='time')\n")
+
+
+# FIND NAN POINT AFTER FILLING AND FILLING AGAIN WITH LINEARREGRESSION ALGORITHM
+
+# In[13]:
+
+
+nan_mask = filled_ds.isnull()
+
+# Print the NaN mask
+# print(nan_mask)
+
+# Count the number of NaNs
+num_nans = nan_mask.sum()
+print(f'Number of NaNs: {num_nans.values}')
+
+
+# In[14]:
+
+
+from sklearn.preprocessing import PolynomialFeatures
+from sklearn.linear_model import LinearRegression
+from sklearn.ensemble import RandomForestRegressor
+
+mask = ~np.isnan(filled_ds)
+X_train = np.stack([dsvh.values[mask], dsvv.values[mask]], axis=1)
+y_train = filled_ds.values[mask]
+
+
+# In[15]:
+
+
+model = LinearRegression()
+model.fit(X_train, y_train)
+
+
+# In[16]:
+
+
+X_pred = np.stack([dsvh.values[~mask], dsvv.values[~mask]], axis=1)
+filled_ds.values[~mask] = model.predict(X_pred)
+
+
+# MATCH LABEL TO DATASET
+
+# In[17]:
+
+
+get_ipython().run_cell_magic('time', '', '\n# Takes 1 minute to complete.\nloaded_datasets = {}\nfor idx, point in train.iterrows():\n    key = f"point_{idx + 1}"\n    try:\n        ndvi_data = filled_ds.sel(x=point.geometry.x, y=point.geometry.y, method=\'nearest\').values\n        vh_data = dsvh.sel(x=point.geometry.x, y=point.geometry.y, method=\'nearest\').values\n        vv_data = dsvv.sel(x=point.geometry.x, y=point.geometry.y, method=\'nearest\').values\n        loaded_datasets[key] = {\n            "data": np.concatenate((ndvi_data, vh_data, vv_data)),\n            "label": point.HT_code\n                               }\n    except Exception as e:\n        # loaded_datasets[key] = None\n        print(e)\n')
+
+
+# In[18]:
+
+
+label_mapping = {
+    "Lua tom": "0",
+    "Lua": "1",
+    "CHN": "2",
+    "CLN": "3",
+    "TS": "4",
+    "Song": "5",
+    "Dat xay dung": "6",
+    "Rung": "7"
+}
+label_encoder = LabelEncoder()
+
+# Fit and transform the labels
+labels = train.Hientrang.values
+numeric_labels = label_encoder.fit_transform([label_mapping[label] for label in labels])
+
+
+# In[19]:
+
+
+X = []
+x_new = []
+lb_new = []
+for k, v in loaded_datasets.items():
+    X.append(v)
+for i in range(len(X)):
+    if X[i] is not None:
+        x_new.append(X[i]["data"])
+        lb_new.append(numeric_labels[i])
+
+
+# BUILDING DATASETS 
+
+# In[20]:
+
+
+X_train, X_temp, y_train, y_temp= train_test_split(x_new, lb_new, test_size=0.4, random_state=42)
+X_val, X_test, y_val, y_test = train_test_split(X_temp, y_temp, test_size=0.5, random_state=42)
+
+
+# TRAIN MODEL
+
+# In[21]:
+
+
+get_ipython().run_cell_magic('time', '', 'from sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.svm import SVC\nfrom sklearn.metrics import accuracy_score\n\n# Define the models\nrf_model = RandomForestClassifier(random_state=42, n_jobs=-1)\nknn_model = KNeighborsClassifier()\nnb_model = GaussianNB()\nsvm_model = SVC()\n\n# Create a pipeline\npipeline = Pipeline([\n    (\'scaler\', StandardScaler()),  # Apply scaling\n    (\'classifier\', rf_model)       # Placeholder, will be set by param_grid\n])\n\n# Define the parameter grid for each classifier\nparam_grid = [\n    # RandomForest\n    {\n        \'classifier\': [rf_model],\n        \'classifier__n_estimators\': [100, 300, 500, 700],\n        \'classifier__max_depth\': [6, 8, 10, 15],\n        \'classifier__criterion\': [\'gini\', \'entropy\'],\n    },\n    # KNeighborsClassifier\n    {\n        \'classifier\': [knn_model],\n        \'classifier__n_neighbors\': [3, 5, 7, 9],\n        \'classifier__weights\': [\'uniform\', \'distance\'],\n        \'classifier__metric\': [\'euclidean\', \'manhattan\']\n    },\n    # Naive Bayes (GaussianNB doesn\'t have hyperparameters to tune here)\n    {\n        \'classifier\': [nb_model],\n    },\n    # SVM\n    {\n        \'classifier\': [svm_model],\n        \'classifier__C\': [0.1, 1, 10, 100],\n        \'classifier__kernel\': [\'linear\', \'rbf\'],\n        \'classifier__gamma\': [\'scale\', \'auto\']\n    }\n]\n\n# Use GridSearchCV to find the best classifier and hyperparameters\ngrid_search = GridSearchCV(pipeline, param_grid, cv=5, scoring=\'accuracy\', n_jobs=-1)\ngrid_search.fit(X_train, y_train)\n\n# Print out the best parameters and classifier\nbest_params = grid_search.best_params_\nprint("Best Parameters:", best_params)\n\n# Make predictions on the validation set\ny_pred = grid_search.predict(X_val)\n\n# Evaluate the results\naccuracy = accuracy_score(y_val, y_pred)\nprint(f"Accuracy: {round(accuracy, 2)*100} %")\n')
+
+
+# In[22]:
+
+
+## check accuracy score
+
+y_pred_test = grid_search.predict(X_test)
+test_accuracy = accuracy_score(y_test, y_pred_test)
+print(f"Accuracy for test data {round(test_accuracy, 2)*100} %")
+
+
+# In[23]:
+
+
+dir_save_model = "model_train"
+if not os.path.exists(dir_save_model):
+    os.mkdir(dir_save_model)
+joblib.dump(grid_search, os.path.join(dir_save_model, "model_new2.joblib"))
+
+
+# In[24]:
+
+
+client.close()
+cluster.close()
+

patch_load_sen1.py

@@ -0,0 +1,29 @@
+import json
+import glob
+
+def fix_load_sen1(file_path):
+    with open(file_path, 'r', encoding='utf-8') as f:
+        nb = json.load(f)
+        
+    changed = False
+    for cell in nb.get('cells', []):
+        if cell.get('cell_type') == 'code':
+            source = cell.get('source', [])
+            for i, line in enumerate(source):
+                if 'load_sen1(name_vh, name_vv)' in line:
+                    indent = line[:len(line) - len(line.lstrip())]
+                    replacement = (
+                        f"{indent}bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]\n"
+                        f"{indent}time_range = f'{{date_range[0]}}/{{date_range[1]}}'\n"
+                        f"{indent}{line.lstrip().replace('load_sen1(name_vh, name_vv)', 'load_sen1(bbox, time_range)')}"
+                    )
+                    source[i] = replacement
+                    changed = True
+                    
+    if changed:
+        with open(file_path, 'w', encoding='utf-8') as f:
+            json.dump(nb, f, indent=1)
+        print(f"Patched load_sen1 in {file_path}")
+
+for nb in glob.glob("*.ipynb"):
+    fix_load_sen1(nb)

patch_notebooks.py

@@ -0,0 +1,43 @@
+import json
+import glob
+
+def patch_notebook(file_path):
+    with open(file_path, 'r', encoding='utf-8') as f:
+        nb = json.load(f)
+        
+    changed = False
+    for cell in nb.get('cells', []):
+        if cell.get('cell_type') == 'code':
+            source = cell.get('source', [])
+            
+            # Check if this cell should be fully commented out
+            full_source = ''.join(source)
+            if 'dc.load(' in full_source or 'ds.vv' in full_source:
+                for i in range(len(source)):
+                    if not source[i].startswith('#'):
+                        source[i] = '# ' + source[i]
+                changed = True
+                continue
+            
+            # Otherwise, do line-by-line replacements
+            for i, line in enumerate(source):
+                if 'ST_training data_updated_1130points.shp' in line:
+                    source[i] = line.replace('ST_training data_updated_1130points.shp', 'ST_training_data_updated_1130points.shp')
+                    changed = True
+                if 'from new_import import *' in line:
+                    source[i] = line.replace('from new_import import *', 'from new_import_ODC import *')
+                    changed = True
+                if 'dc = datacube.Datacube()' in line:
+                    source[i] = line.replace('dc = datacube.Datacube()', 'dc = None')
+                    changed = True
+                if 'load_data(dc,' in line:
+                    source[i] = line.replace('load_data(dc,', 'load_data(None,')
+                    changed = True
+                    
+    if changed:
+        with open(file_path, 'w', encoding='utf-8') as f:
+            json.dump(nb, f, indent=1)
+        print(f"Patched {file_path}")
+
+for nb in glob.glob("*.ipynb"):
+    patch_notebook(nb)

prediction_interface.html

@@ -774,23 +774,6 @@
                             </div>
                         </label>
                     </div>
-                    
-                    <!-- Shapefile Overlay Option -->
-                    <div class="form-group" style="margin-bottom: 15px;">
-                        <label style="font-weight: 600; color: #c2410c; margin-bottom: 8px; display: block;">
-                            🗺️ Overlay Shapefile (Hiển thị ranh giới lô đất)
-                        </label>
-                        <select id="shapefileOverlay" onchange="onShapefileSelected(event)" style="width: 100%; padding: 10px; border: 2px solid #fdba74; border-radius: 8px; font-size: 14px; background: white;">
-                            <option value="">-- Không overlay --</option>
-                            <!-- Shapefiles will be loaded here -->
-                        </select>
-                        <div style="font-size: 0.85em; color: #9a3412; margin-top: 4px; line-height: 1.4;">
-                            <b>🎯 Tự động cập nhật vùng prediction:</b><br>
-                            ✅ Khi chọn shapefile → <b>Bbox trên bản đồ tự động thay đổi</b> theo vùng shapefile<br>
-                            ✅ <b>CRS sẽ tự động chuyển đổi</b> - không cần lo về EPSG:4326/9209/32648<br>
-                            💡 Không chọn shapefile → Dùng bbox tùy chỉnh do bạn vẽ trên bản đồ
-                        </div>
-                    </div>
                 </div>
                 
                 <button class="btn btn-primary" onclick="startPrediction()" id="predictBtn" style="margin-top: 5px; width: 100%; font-size: 1.1em; padding: 16px;">
@@ -1668,29 +1651,6 @@
                 }
             }
             
-            // Get shapefile overlay option
-            const shapefileOverlay = document.getElementById('shapefileOverlay').value;
-            
-            // Warn if no shapefile selected (optional but recommended)
-            if (!shapefileOverlay) {
-                const confirmWithoutShapefile = confirm(
-                    '⚠️ CẢNH BÁO: Bạn chưa chọn shapefile!\n\n' +
-                    '❌ Kết quả sẽ KHÔNG có đường ranh giới lô đất.\n\n' +
-                    '💡 Để có đường phân lô trên ảnh kết quả:\n' +
-                    '   - Hủy bỏ\n' +
-                    '   - Chọn shapefile trong dropdown "Overlay Shapefile"\n' +
-                    '   - Chạy lại prediction\n\n' +
-                    'Bạn có muốn tiếp tục KHÔNG CÓ ranh giới lô đất không?'
-                );
-                
-                if (!confirmWithoutShapefile) {
-                    console.log('[PREDICTION] User cancelled to select shapefile');
-                    return;
-                }
-            } else {
-                console.log(`[PREDICTION] Shapefile selected: ${shapefileOverlay}`);
-            }
-            
             const config = {
                 model_filename: modelFilename,
                 min_lon: selectedBbox.min_lon,
@@ -1706,8 +1666,7 @@
                 export_ndvi: exportNDVI,
                 export_classification: true,
                 cloud_removal_method: cloudRemovalConfig.method,
-                cloud_removal_model: cloudRemovalConfig.model_filename || null,
-                shapefile_overlay: shapefileOverlay || null
+                cloud_removal_model: cloudRemovalConfig.model_filename || null
             };
             
             try {
@@ -2639,7 +2598,6 @@
             loadPredProvinces();  // Load provinces list
             loadNDVIProvinces();  // Load NDVI provinces list
             loadNDVIModels();  // Load models for NDVI
-            loadOverlayShapefiles();  // Load shapefiles for overlay
             
             // Add event listener for model selection
             document.getElementById('modelSelect').addEventListener('change', updateModelInfo);
@@ -3035,153 +2993,6 @@
             
             alert(`✅ Đã áp dụng preset: ${config.name}\n\nBbox: [${config.bbox.join(', ')}]\nThời gian: ${config.start_date} → ${config.end_date}\nSample points: ${config.sample_points}`);
         }
-        
-        // Load overlay shapefiles
-        async function loadOverlayShapefiles() {
-            try {
-                const response = await fetch('/api/overlay/shapefiles');
-                const data = await response.json();
-                
-                const select = document.getElementById('shapefileOverlay');
-                select.innerHTML = '<option value="">-- Không overlay --</option>';
-                
-                if (data.shapefiles && data.shapefiles.length > 0) {
-                    data.shapefiles.forEach(shp => {
-                        const option = document.createElement('option');
-                        option.value = shp.path;
-                        
-                        // Build detailed label with CRS and bbox info
-                        let label = `${shp.filename} - ${shp.feature_count} features`;
-                        
-                        // Add CRS info (important for matching!)
-                        if (shp.crs) {
-                            const crsCode = shp.crs.split(':').pop(); // Extract code from "EPSG:4326"
-                            label += ` | CRS: ${crsCode}`;
-                        }
-                        
-                        // Add bbox info for easy matching with prediction area
-                        if (shp.bbox && shp.bbox.length === 4) {
-                            const [minLon, minLat, maxLon, maxLat] = shp.bbox;
-                            label += ` | Vùng: [${minLon.toFixed(2)}, ${minLat.toFixed(2)}, ${maxLon.toFixed(2)}, ${maxLat.toFixed(2)}]`;
-                        }
-                        
-                        option.textContent = label;
-                        
-                        // Store full shapefile info as data attributes for later use
-                        option.dataset.crs = shp.crs || '';
-                        option.dataset.bbox = JSON.stringify(shp.bbox || []);
-                        option.dataset.featureCount = shp.feature_count;
-                        
-                        select.appendChild(option);
-                    });
-                    
-                    console.log(`[Overlay Shapefiles] Loaded ${data.shapefiles.length} shapefiles`);
-                } else {
-                    console.log('[Overlay Shapefiles] No shapefiles found');
-                }
-                
-                // Add event listener for shapefile selection change (OUTSIDE the if block)
-                // Remove old listener first to prevent duplicates
-                select.removeEventListener('change', onShapefileSelected);
-                select.addEventListener('change', onShapefileSelected);
-                console.log('[Overlay Shapefiles] Event listener attached');
-                
-            } catch (error) {
-                console.error('[Overlay Shapefiles] Error loading shapefiles:', error);
-                const select = document.getElementById('shapefileOverlay');
-                select.innerHTML = '<option value="">Error loading shapefiles</option>';
-            }
-        }
-        
-        // Handle shapefile selection - auto update bbox on map
-        function onShapefileSelected(event) {
-            console.log('[Shapefile Select] Event triggered');
-            console.log('[Shapefile Select] map exists:', typeof map !== 'undefined');
-            console.log('[Shapefile Select] drawnItems exists:', typeof drawnItems !== 'undefined');
-            
-            const selectedOption = event.target.selectedOptions[0];
-            
-            // If no shapefile selected (empty value), keep current bbox
-            if (!selectedOption || !selectedOption.value) {
-                console.log('[Shapefile Select] No shapefile selected, keeping current bbox');
-                return;
-            }
-            
-            console.log('[Shapefile Select] Selected shapefile:', selectedOption.value);
-            
-            // Get bbox from data attribute
-            const bboxData = selectedOption.dataset.bbox;
-            console.log('[Shapefile Select] Bbox data:', bboxData);
-            
-            if (!bboxData || bboxData === '[]') {
-                console.warn('[Shapefile Select] Selected shapefile has no bbox data');
-                return;
-            }
-            
-            try {
-                const bbox = JSON.parse(bboxData);
-                console.log('[Shapefile Select] Parsed bbox:', bbox);
-                
-                if (bbox.length !== 4) {
-                    console.warn('[Shapefile Select] Invalid bbox format:', bbox);
-                    return;
-                }
-                
-                const [minLon, minLat, maxLon, maxLat] = bbox;
-                
-                // Validate bbox
-                if (minLon < -180 || maxLon > 180 || minLat < -90 || maxLat > 90) {
-                    alert('❌ Bbox của shapefile không hợp lệ!');
-                    return;
-                }
-                
-                console.log('[Shapefile Select] Creating rectangle with bounds:', [[minLat, minLon], [maxLat, maxLon]]);
-                
-                // Update prediction map bbox
-                const bounds = [
-                    [minLat, minLon],
-                    [maxLat, maxLon]
-                ];
-                
-                const rectangle = L.rectangle(bounds, {
-                    color: '#667eea',
-                    weight: 3,
-                    fillOpacity: 0.2
-                });
-                
-                // Clear old bbox and add new one
-                console.log('[Shapefile Select] Clearing old layers...');
-                drawnItems.clearLayers();
-                console.log('[Shapefile Select] Adding new rectangle...');
-                drawnItems.addLayer(rectangle);
-                console.log('[Shapefile Select] Fitting map to bounds...');
-                map.fitBounds(bounds, { padding: [50, 50] });
-                
-                // Update selected bbox variable
-                selectedBbox = {
-                    min_lon: minLon,
-                    min_lat: minLat,
-                    max_lon: maxLon,
-                    max_lat: maxLat
-                };
-                
-                // Save to localStorage
-                localStorage.setItem('prediction_bbox', JSON.stringify(selectedBbox));
-                
-                console.log(`[Shapefile Select] Auto-updated bbox from shapefile:`, selectedBbox);
-                
-                // Show notification
-                const crs = selectedOption.dataset.crs || 'Unknown';
-                alert(`✅ Đã tự động cập nhật vùng prediction theo shapefile!\n\n` +
-                      `📍 Bbox: [${minLon.toFixed(4)}, ${minLat.toFixed(4)}, ${maxLon.toFixed(4)}, ${maxLat.toFixed(4)}]\n` +
-                      `🗺️ CRS: ${crs}\n\n` +
-                      `💡 Bạn có thể điều chỉnh lại bằng cách vẽ lại trên bản đồ nếu muốn.`);
-                
-            } catch (e) {
-                console.error('[Shapefile Select] Error parsing bbox:', e);
-                alert(`❌ Lỗi khi xử lý bbox: ${e.message}`);
-            }
-        }
     </script>
 </body>
 </html>

processMask-test.ipynb

@@ -1132,7 +1132,7 @@
    "source": [
     "%%time\n",
     "%matplotlib inline\n",
-    "from new_import import *"
+    "from new_import_ODC import *"
    ]
   },
   {
@@ -1216,7 +1216,7 @@
     "%%time\n",
     "# Dask gateway\n",
     "cluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))\n",
-    "dc = datacube.Datacube()\n",
+    "dc = None\n",
     "\n",
     "# Configure s3 access\n",
     "configure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n",
@@ -1233,11 +1233,11 @@
    },
    "outputs": [],
    "source": [
-    "## cấu hình thời gian lấy ảnh và tọa độ\n",
-    "# date_range = ('2022-09-01', '2023-10-01')\n",
+    "## c\u1ea5u h\u00ecnh th\u1eddi gian l\u1ea5y \u1ea3nh v\u00e0 t\u1ecda \u0111\u1ed9\n",
+    "# date_range = ('2022-09-01', '2022-10-01')\n",
     "# longtitude_range = (105.86575, 105.94120)\n",
     "# latitude_range = (9.65070, 9.69850)\n",
-    "date_range = ('2022-09-01', '2023-10-01')\n",
+    "date_range = ('2022-09-01', '2022-10-01')\n",
     "longtitude_range = (105.5, 106.4)\n",
     "latitude_range = (9.2, 10.0) "
    ]
@@ -1396,7 +1396,7 @@
        "\n",
        ".xr-section-summary-in + label:before {\n",
        "  display: inline-block;\n",
-       "  content: '►';\n",
+       "  content: '\u25ba';\n",
        "  font-size: 11px;\n",
        "  width: 15px;\n",
        "  text-align: center;\n",
@@ -1407,7 +1407,7 @@
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label:before {\n",
-       "  content: '▼';\n",
+       "  content: '\u25bc';\n",
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label > span {\n",
@@ -2424,8 +2424,8 @@
     }
    ],
    "source": [
-    "## truy vấn ảnh vệ tinh sen2\n",
-    "data = load_data(dc, date_range, longtitude_range, latitude_range)\n",
+    "## truy v\u1ea5n \u1ea3nh v\u1ec7 tinh sen2\n",
+    "data = load_data(None, date_range, longtitude_range, latitude_range)\n",
     "notebook_utils.heading(notebook_utils.xarray_object_size(data))\n",
     "display(data)"
    ]
@@ -2440,10 +2440,10 @@
    "outputs": [],
    "source": [
     "# Specify the start and end times \n",
-    "min_date = '2022-09-01' # Thời gian bắt đầu lấy data cho quá trình train\n",
-    "max_date = '2023-10-01' # Thời gian kết thúc lấy data cho quá trình train\n",
+    "min_date = '2022-09-01' # Th\u1eddi gian b\u1eaft \u0111\u1ea7u l\u1ea5y data cho qu\u00e1 tr\u00ecnh train\n",
+    "max_date = '2022-10-01' # Th\u1eddi gian k\u1ebft th\u00fac l\u1ea5y data cho qu\u00e1 tr\u00ecnh train\n",
     "# Just do 1 month for testing\n",
-    "# max_date = '2022-10-01' # Thời gian kết thúc lấy data cho quá trình train\n",
+    "# max_date = '2022-10-01' # Th\u1eddi gian k\u1ebft th\u00fac l\u1ea5y data cho qu\u00e1 tr\u00ecnh train\n",
     "\n",
     "# Specify a spatail region to search using latitude/longitude cooridinates\n",
     "min_longitude, max_longitude = (105.5, 106.4)\n",
@@ -2657,7 +2657,7 @@
        "\n",
        ".xr-section-summary-in + label:before {\n",
        "  display: inline-block;\n",
-       "  content: '►';\n",
+       "  content: '\u25ba';\n",
        "  font-size: 11px;\n",
        "  width: 15px;\n",
        "  text-align: center;\n",
@@ -2668,7 +2668,7 @@
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label:before {\n",
-       "  content: '▼';\n",
+       "  content: '\u25bc';\n",
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label > span {\n",
@@ -3423,23 +3423,23 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "CPU times: user 1.57 s, sys: 177 µs, total: 1.57 s\n",
+      "CPU times: user 1.57 s, sys: 177 \u00b5s, total: 1.57 s\n",
       "Wall time: 1.66 s\n"
      ]
     }
    ],
    "source": [
-    "%%time\n",
+    "# %%time\n",
     "# The replacement \"dc.load()\" function for this product\n",
-    "data = load_s2l2a_with_offset(\n",
-    "    dc,\n",
-    "    query | load_params   # Combine the two dicts that contain our search and load parameters\n",
-    ")\n",
-    "\n",
+    "# data = load_s2l2a_with_offset(\n",
+    "#     dc,\n",
+    "#     query | load_params   # Combine the two dicts that contain our search and load parameters\n",
+    "# )\n",
+    "# \n",
     "# This line prints the total size of the dataset hat was loaded\n",
-    "notebook_utils.heading(notebook_utils.xarray_object_size(data))\n",
-    "\n",
-    "display(data)"
+    "# notebook_utils.heading(notebook_utils.xarray_object_size(data))\n",
+    "# \n",
+    "# display(data)"
    ]
   },
   {
@@ -3546,9 +3546,9 @@
    ],
    "source": [
     "# %%time\n",
-    "# # Tiến hành loại bỏ các vị trí bị mây ảnh hưởng\n",
+    "# # Ti\u1ebfn h\u00e0nh lo\u1ea1i b\u1ecf c\u00e1c v\u1ecb tr\u00ed b\u1ecb m\u00e2y \u1ea3nh h\u01b0\u1edfng\n",
     "# result = mask_clean(data)\n",
-    "# progress(result)"
+    "# # progress(result)"
    ]
   },
   {
@@ -3684,7 +3684,7 @@
        "\n",
        ".xr-section-summary-in + label:before {\n",
        "  display: inline-block;\n",
-       "  content: '►';\n",
+       "  content: '\u25ba';\n",
        "  font-size: 11px;\n",
        "  width: 15px;\n",
        "  text-align: center;\n",
@@ -3695,7 +3695,7 @@
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label:before {\n",
-       "  content: '▼';\n",
+       "  content: '\u25bc';\n",
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label > span {\n",
@@ -4174,7 +4174,7 @@
     }
    ],
    "source": [
-    "# Tiến hành tính toán NDVI\n",
+    "# Ti\u1ebfn h\u00e0nh t\u00ednh to\u00e1n NDVI\n",
     "ds1 = calculate_indices(result, index='NDVI', satellite_mission='s2')\n",
     "ndvi = ds1[\"NDVI\"]\n",
     "display(ndvi)"
@@ -4213,9 +4213,9 @@
    ],
    "source": [
     "%%time\n",
-    "## tính ndvi theo tháng\n",
+    "## t\u00ednh ndvi theo th\u00e1ng\n",
     "average_ndvi = ndvi.resample(time='1M').mean().persist()\n",
-    "progress(average_ndvi)"
+    "# progress(average_ndvi)"
    ]
   },
   {
@@ -4253,12 +4253,12 @@
     }
    ],
    "source": [
-    "# cấu hình vh vv file\n",
+    "# c\u1ea5u h\u00ecnh vh vv file\n",
     "# name_vh = \"ThuanHoa/ThuanHoa_VH.tif\"\n",
     "# name_vv = \"ThuanHoa/ThuanHoa_VV.tif\"\n",
     "\n",
-    "# load dữ liệu sen1\n",
-    "# dsvh, dsvv = load_sen1(name_vh, name_vv)\n",
+    "# load d\u1eef li\u1ec7u sen1\n",
+    "bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]\ntime_range = f'{date_range[0]}/{date_range[1]}'\n# dsvh, dsvv = load_sen1(bbox, time_range)\n",
     "\n",
     "name_vh = \"vh-0922_0923-full_ST.tif\"\n",
     "name_vv = \"vv-0922_0923-full_ST.tif\"\n",
@@ -4268,7 +4268,7 @@
     "if not os.path.exists(name_vv):\n",
     "    !aws s3 cp s3://easi-asia-dc-data/staging/ctu/sentinel-1/vv-0922_0923-full_ST.tif vv-0922_0923-full_ST.tif\n",
     "    \n",
-    "dsvh, dsvv = load_sen1(name_vh, name_vv)"
+    "bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]\ntime_range = f'{date_range[0]}/{date_range[1]}'\ndsvh, dsvv = load_sen1(bbox, time_range)"
    ]
   },
   {
@@ -4326,7 +4326,7 @@
     {
      "data": {
       "text/html": [
-       "<style>#sk-container-id-1 {color: black;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>LinearRegression()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LinearRegression</label><div class=\"sk-toggleable__content\"><pre>LinearRegression()</pre></div></div></div></div></div>"
+       "<style>#sk-container-id-1 {color: black;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"\u25b8\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"\u25be\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>LinearRegression()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LinearRegression</label><div class=\"sk-toggleable__content\"><pre>LinearRegression()</pre></div></div></div></div></div>"
       ],
       "text/plain": [
        "LinearRegression()"
@@ -4397,7 +4397,7 @@
     }
    ],
    "source": [
-    "plt.imshow(average_ndvi_filled.isel(time=6))"
+    "plt.imshow(average_ndvi_filled.isel(time=0))"
    ]
   },
   {
@@ -4430,7 +4430,7 @@
     }
    ],
    "source": [
-    "plt.imshow(average_ndvi.isel(time=6))"
+    "plt.imshow(average_ndvi.isel(time=0))"
    ]
   },
   {
@@ -4440,7 +4440,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "train_path = \"train/ST_training data_updated_1130points.shp\""
+    "train_path = \"train/ST_training_data_updated_1130points.shp\""
    ]
   },
   {
@@ -4476,7 +4476,7 @@
    },
    "outputs": [],
    "source": [
-    "# cấu hình nhãn dữ liệu\n",
+    "# c\u1ea5u h\u00ecnh nh\u00e3n d\u1eef li\u1ec7u\n",
     "label_mapping = {\n",
     "    \"Lua tom\": \"0\",\n",
     "    \"Lua\": \"1\",\n",
@@ -4488,7 +4488,7 @@
     "    \"Rung\": \"7\"\n",
     "}\n",
     "\n",
-    "# chia tập dữ liệu train, val, test\n",
+    "# chia t\u1eadp d\u1eef li\u1ec7u train, val, test\n",
     "X_train, X_val, X_test, y_train, y_val, y_test = split_train_data(train, label_mapping, datasets)"
    ]
   },
@@ -4510,7 +4510,7 @@
     }
    ],
    "source": [
-    "# Huấn luyện mô hình\n",
+    "# Hu\u1ea5n luy\u1ec7n m\u00f4 h\u00ecnh\n",
     "grid_search = train_with_rf(X_train, X_val, y_train, y_val)"
    ]
   },
@@ -4531,7 +4531,7 @@
     }
    ],
    "source": [
-    "# kiểm tra độ chính xác với tập test\n",
+    "# ki\u1ec3m tra \u0111\u1ed9 ch\u00ednh x\u00e1c v\u1edbi t\u1eadp test\n",
     "y_pred_test = grid_search.predict(X_test)\n",
     "test_accuracy = accuracy_score(y_test, y_pred_test)\n",
     "print(f\"Accuracy for test data {round(test_accuracy, 2)*100} %\")"
@@ -4554,7 +4554,7 @@
     }
    ],
    "source": [
-    "# Lưu mô hình huấn luyện\n",
+    "# L\u01b0u m\u00f4 h\u00ecnh hu\u1ea5n luy\u1ec7n\n",
     "save_model(\"model_new.joblib\", grid_search)"
    ]
   },
@@ -4567,7 +4567,7 @@
    },
    "outputs": [],
    "source": [
-    "# đóng client, cluster\n",
+    "# \u0111\u00f3ng client, cluster\n",
     "client.close()\n",
     "cluster.close()"
    ]
@@ -4602,4 +4602,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 5
-}
+}

processMask-test.py

@@ -0,0 +1,272 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+get_ipython().run_cell_magic('time', '', '%matplotlib inline\nfrom new_import import *\n')
+
+
+# In[2]:
+
+
+get_ipython().run_cell_magic('time', '', '# Dask gateway\ncluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))\ndc = datacube.Datacube()\n\n# Configure s3 access\nconfigure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n\nclient\n')
+
+
+# In[3]:
+
+
+## cấu hình thời gian lấy ảnh và tọa độ
+# date_range = ('2022-09-01', '2023-10-01')
+# longtitude_range = (105.86575, 105.94120)
+# latitude_range = (9.65070, 9.69850)
+date_range = ('2022-09-01', '2023-10-01')
+longtitude_range = (105.5, 106.4)
+latitude_range = (9.2, 10.0) 
+
+
+# In[4]:
+
+
+## truy vấn ảnh vệ tinh sen2
+data = load_data(dc, date_range, longtitude_range, latitude_range)
+notebook_utils.heading(notebook_utils.xarray_object_size(data))
+display(data)
+
+
+# In[5]:
+
+
+# Specify the start and end times 
+min_date = '2022-09-01' # Thời gian bắt đầu lấy data cho quá trình train
+max_date = '2023-10-01' # Thời gian kết thúc lấy data cho quá trình train
+# Just do 1 month for testing
+# max_date = '2022-10-01' # Thời gian kết thúc lấy data cho quá trình train
+
+# Specify a spatail region to search using latitude/longitude cooridinates
+min_longitude, max_longitude = (105.5, 106.4)
+min_latitude, max_latitude = (9.2, 10.0)
+
+# Specify the product. In this case we want to use Sentinel-2 Level-2A data
+product = 's2_l2a'
+
+# Construct the search query dictionary
+query = {
+    'product': product,                     # Product name
+    'x': (min_longitude, max_longitude),    # "x" axis bounds
+    'y': (min_latitude, max_latitude),      # "y" axis bounds
+    'time': (min_date, max_date),           # Any parsable date strings
+}
+
+
+# In[6]:
+
+
+# Most common CRS
+native_crs = notebook_utils.mostcommon_crs(dc, query)
+print(f'Most common native CRS: {native_crs}')
+
+
+# In[7]:
+
+
+# Specify the spectral band measurements we want to use for a classification algorithm
+measurements = ['red', 'nir', 'scl']
+
+load_params = {
+    'measurements': measurements,                   # Selected measurement or alias names
+    'output_crs': native_crs,                       # Target EPSG code
+    'resolution': (-10, 10),                        # Target resolution
+    'group_by': 'solar_day',                        # Scene grouping
+    'dask_chunks': {'x': 2048, 'y': 2048},          # Dask chunks
+}
+
+
+# In[8]:
+
+
+get_ipython().run_cell_magic('time', '', '# The replacement "dc.load()" function for this product\ndata = load_s2l2a_with_offset(\n    dc,\n    query | load_params   # Combine the two dicts that contain our search and load parameters\n)\n\n# This line prints the total size of the dataset hat was loaded\nnotebook_utils.heading(notebook_utils.xarray_object_size(data))\n\ndisplay(data)\n')
+
+
+# In[9]:
+
+
+# %%time
+# # Tiến hành loại bỏ các vị trí bị mây ảnh hưởng
+# result = mask_clean(data)
+# progress(result)
+
+
+# In[10]:
+
+
+# Tiến hành tính toán NDVI
+ds1 = calculate_indices(result, index='NDVI', satellite_mission='s2')
+ndvi = ds1["NDVI"]
+display(ndvi)
+
+
+# In[11]:
+
+
+get_ipython().run_cell_magic('time', '', "## tính ndvi theo tháng\naverage_ndvi = ndvi.resample(time='1M').mean().persist()\nprogress(average_ndvi)\n")
+
+
+# In[12]:
+
+
+# compute average_ndvi
+average_ndvi = average_ndvi.compute()
+
+
+# In[13]:
+
+
+# cấu hình vh vv file
+# name_vh = "ThuanHoa/ThuanHoa_VH.tif"
+# name_vv = "ThuanHoa/ThuanHoa_VV.tif"
+
+# load dữ liệu sen1
+bbox = [105.5, 9.2, 106.4, 10.0]
+time_range = '2022-09-01/2023-10-01'
+# dsvh, dsvv = load_sen1(bbox, time_range)
+
+name_vh = "vh-0922_0923-full_ST.tif"
+name_vv = "vv-0922_0923-full_ST.tif"
+
+if not os.path.exists(name_vh):
+    get_ipython().system('aws s3 cp s3://easi-asia-dc-data/staging/ctu/sentinel-1/vh-0922_0923-full_ST.tif vh-0922_0923-full_ST.tif')
+if not os.path.exists(name_vv):
+    get_ipython().system('aws s3 cp s3://easi-asia-dc-data/staging/ctu/sentinel-1/vv-0922_0923-full_ST.tif vv-0922_0923-full_ST.tif')
+    
+bbox = [105.5, 9.2, 106.4, 10.0]
+time_range = '2022-09-01/2023-10-01'
+dsvh, dsvv = load_sen1(bbox, time_range)
+
+
+# In[27]:
+
+
+from sklearn.preprocessing import PolynomialFeatures
+from sklearn.linear_model import LinearRegression
+from sklearn.ensemble import RandomForestRegressor
+
+
+# In[28]:
+
+
+average_ndvi = average_ndvi[:, :7680, :8687]
+mask = ~np.isnan(average_ndvi)
+print(average_ndvi.shape)
+print(dsvh.shape)
+print(dsvv.shape)
+print(mask.shape)
+X_train = np.stack([dsvh.values[mask], dsvv.values[mask]], axis=1)
+y_train = average_ndvi.values[mask]
+
+
+# In[29]:
+
+
+model = LinearRegression()
+model.fit(X_train, y_train)
+
+
+# In[30]:
+
+
+X_pred = np.stack([dsvh.values[~mask], dsvv.values[~mask]], axis=1)
+average_ndvi.values[~mask] = model.predict(X_pred)
+
+
+# In[31]:
+
+
+average_ndvi_filled = xr.DataArray(average_ndvi, dims=average_ndvi.dims)
+
+
+# In[32]:
+
+
+plt.imshow(average_ndvi_filled.isel(time=6))
+
+
+# In[65]:
+
+
+plt.imshow(average_ndvi.isel(time=6))
+
+
+# In[33]:
+
+
+train_path = "train/ST_training data_updated_1130points.shp"
+
+
+# In[34]:
+
+
+train = load_train_data(train_path)
+
+
+# In[37]:
+
+
+datasets = get_data_sen1_and_sen2(train, average_ndvi_filled, dsvh, dsvv)
+
+
+# In[39]:
+
+
+# cấu hình nhãn dữ liệu
+label_mapping = {
+    "Lua tom": "0",
+    "Lua": "1",
+    "CHN": "2",
+    "CLN": "3",
+    "TS": "4",
+    "Song": "5",
+    "Dat xay dung": "6",
+    "Rung": "7"
+}
+
+# chia tập dữ liệu train, val, test
+X_train, X_val, X_test, y_train, y_val, y_test = split_train_data(train, label_mapping, datasets)
+
+
+# In[40]:
+
+
+# Huấn luyện mô hình
+grid_search = train_with_rf(X_train, X_val, y_train, y_val)
+
+
+# In[41]:
+
+
+# kiểm tra độ chính xác với tập test
+y_pred_test = grid_search.predict(X_test)
+test_accuracy = accuracy_score(y_test, y_pred_test)
+print(f"Accuracy for test data {round(test_accuracy, 2)*100} %")
+
+
+# In[42]:
+
+
+# Lưu mô hình huấn luyện
+save_model("model_new.joblib", grid_search)
+
+
+# In[43]:
+
+
+# đóng client, cluster
+client.close()
+cluster.close()
+
+
+# In[ ]:
+
+
+
+

processMask.ipynb

@@ -1132,7 +1132,7 @@
    "source": [
     "%%time\n",
     "%matplotlib inline\n",
-    "from new_import import *"
+    "from new_import_ODC import *"
    ]
   },
   {
@@ -1216,7 +1216,7 @@
     "%%time\n",
     "# Dask gateway\n",
     "cluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))\n",
-    "dc = datacube.Datacube()\n",
+    "dc = None\n",
     "\n",
     "# Configure s3 access\n",
     "configure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n",
@@ -1233,8 +1233,8 @@
    },
    "outputs": [],
    "source": [
-    "## cấu hình thời gian lấy ảnh và tọa độ\n",
-    "date_range = ('2022-09-01', '2023-10-01')\n",
+    "## c\u1ea5u h\u00ecnh th\u1eddi gian l\u1ea5y \u1ea3nh v\u00e0 t\u1ecda \u0111\u1ed9\n",
+    "date_range = ('2022-09-01', '2022-10-01')\n",
     "longtitude_range = (105.86575, 105.94120)\n",
     "latitude_range = (9.65070, 9.69850)"
    ]
@@ -1393,7 +1393,7 @@
        "\n",
        ".xr-section-summary-in + label:before {\n",
        "  display: inline-block;\n",
-       "  content: '►';\n",
+       "  content: '\u25ba';\n",
        "  font-size: 11px;\n",
        "  width: 15px;\n",
        "  text-align: center;\n",
@@ -1404,7 +1404,7 @@
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label:before {\n",
-       "  content: '▼';\n",
+       "  content: '\u25bc';\n",
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label > span {\n",
@@ -2333,8 +2333,8 @@
     }
    ],
    "source": [
-    "## truy vấn ảnh vệ tinh sen2\n",
-    "data = load_data(dc, date_range, longtitude_range, latitude_range)\n",
+    "## truy v\u1ea5n \u1ea3nh v\u1ec7 tinh sen2\n",
+    "data = load_data(None, date_range, longtitude_range, latitude_range)\n",
     "notebook_utils.heading(notebook_utils.xarray_object_size(data))\n",
     "display(data)"
    ]
@@ -2443,9 +2443,9 @@
    ],
    "source": [
     "%%time\n",
-    "# Tiến hành loại bỏ các vị trí bị mây ảnh hưởng\n",
+    "# Ti\u1ebfn h\u00e0nh lo\u1ea1i b\u1ecf c\u00e1c v\u1ecb tr\u00ed b\u1ecb m\u00e2y \u1ea3nh h\u01b0\u1edfng\n",
     "result = mask_clean(data)\n",
-    "progress(result)"
+    "# progress(result)"
    ]
   },
   {
@@ -2581,7 +2581,7 @@
        "\n",
        ".xr-section-summary-in + label:before {\n",
        "  display: inline-block;\n",
-       "  content: '►';\n",
+       "  content: '\u25ba';\n",
        "  font-size: 11px;\n",
        "  width: 15px;\n",
        "  text-align: center;\n",
@@ -2592,7 +2592,7 @@
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label:before {\n",
-       "  content: '▼';\n",
+       "  content: '\u25bc';\n",
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label > span {\n",
@@ -3047,7 +3047,7 @@
     }
    ],
    "source": [
-    "# Tiến hành tính toán NDVI\n",
+    "# Ti\u1ebfn h\u00e0nh t\u00ednh to\u00e1n NDVI\n",
     "ds1 = calculate_indices(result, index='NDVI', satellite_mission='s2')\n",
     "ndvi = ds1[\"NDVI\"]\n",
     "display(ndvi)"
@@ -3086,9 +3086,9 @@
    ],
    "source": [
     "%%time\n",
-    "## tính ndvi theo tháng\n",
+    "## t\u00ednh ndvi theo th\u00e1ng\n",
     "average_ndvi = ndvi.resample(time='1M').mean().persist()\n",
-    "progress(average_ndvi)"
+    "# progress(average_ndvi)"
    ]
   },
   {
@@ -3113,12 +3113,12 @@
    },
    "outputs": [],
    "source": [
-    "# cấu hình vh vv file\n",
+    "# c\u1ea5u h\u00ecnh vh vv file\n",
     "name_vh = \"ThuanHoa/ThuanHoa_VH.tif\"\n",
     "name_vv = \"ThuanHoa/ThuanHoa_VV.tif\"\n",
     "\n",
-    "# load dữ liệu sen1\n",
-    "dsvh, dsvv = load_sen1(name_vh, name_vv)"
+    "# load d\u1eef li\u1ec7u sen1\n",
+    "bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]\ntime_range = f'{date_range[0]}/{date_range[1]}'\ndsvh, dsvv = load_sen1(bbox, time_range)"
    ]
   },
   {
@@ -3160,7 +3160,7 @@
     {
      "data": {
       "text/html": [
-       "<style>#sk-container-id-1 {color: black;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>LinearRegression()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LinearRegression</label><div class=\"sk-toggleable__content\"><pre>LinearRegression()</pre></div></div></div></div></div>"
+       "<style>#sk-container-id-1 {color: black;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"\u25b8\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"\u25be\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>LinearRegression()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LinearRegression</label><div class=\"sk-toggleable__content\"><pre>LinearRegression()</pre></div></div></div></div></div>"
       ],
       "text/plain": [
        "LinearRegression()"
@@ -3297,4 +3297,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 5
-}
+}

processMask.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+get_ipython().run_cell_magic('time', '', '%matplotlib inline\nfrom new_import import *\n')
+
+
+# In[2]:
+
+
+get_ipython().run_cell_magic('time', '', '# Dask gateway\ncluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))\ndc = datacube.Datacube()\n\n# Configure s3 access\nconfigure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n\nclient\n')
+
+
+# In[3]:
+
+
+## cấu hình thời gian lấy ảnh và tọa độ
+date_range = ('2022-09-01', '2023-10-01')
+longtitude_range = (105.86575, 105.94120)
+latitude_range = (9.65070, 9.69850)
+
+
+# In[4]:
+
+
+## truy vấn ảnh vệ tinh sen2
+data = load_data(dc, date_range, longtitude_range, latitude_range)
+notebook_utils.heading(notebook_utils.xarray_object_size(data))
+display(data)
+
+
+# In[5]:
+
+
+get_ipython().run_cell_magic('time', '', '# Tiến hành loại bỏ các vị trí bị mây ảnh hưởng\nresult = mask_clean(data)\nprogress(result)\n')
+
+
+# In[6]:
+
+
+# Tiến hành tính toán NDVI
+ds1 = calculate_indices(result, index='NDVI', satellite_mission='s2')
+ndvi = ds1["NDVI"]
+display(ndvi)
+
+
+# In[17]:
+
+
+get_ipython().run_cell_magic('time', '', "## tính ndvi theo tháng\naverage_ndvi = ndvi.resample(time='1M').mean().persist()\nprogress(average_ndvi)\n")
+
+
+# In[18]:
+
+
+# compute average_ndvi
+average_ndvi = average_ndvi.compute()
+
+
+# In[9]:
+
+
+# cấu hình vh vv file
+name_vh = "ThuanHoa/ThuanHoa_VH.tif"
+name_vv = "ThuanHoa/ThuanHoa_VV.tif"
+
+# load dữ liệu sen1
+bbox = [105.5, 9.2, 106.4, 10.0]
+time_range = '2022-09-01/2023-10-01'
+dsvh, dsvv = load_sen1(bbox, time_range)
+
+
+# In[10]:
+
+
+from sklearn.preprocessing import PolynomialFeatures
+from sklearn.linear_model import LinearRegression
+from sklearn.ensemble import RandomForestRegressor
+
+
+# In[11]:
+
+
+mask = ~np.isnan(average_ndvi)
+X_train = np.stack([dsvh.values[mask], dsvv.values[mask]], axis=1)
+y_train = average_ndvi.values[mask]
+
+
+# In[12]:
+
+
+model = LinearRegression()
+model.fit(X_train, y_train)
+
+
+# In[13]:
+
+
+X_pred = np.stack([dsvh.values[~mask], dsvv.values[~mask]], axis=1)
+average_ndvi.values[~mask] = model.predict(X_pred)
+
+
+# In[14]:
+
+
+average_ndvi_filled = xr.DataArray(average_ndvi, dims=average_ndvi.dims)
+
+
+# In[16]:
+
+
+plt.imshow(average_ndvi_filled.isel(time=1))
+
+
+# In[19]:
+
+
+plt.imshow(average_ndvi.isel(time=1))
+
+
+# In[ ]:
+
+
+
+

remove_datacube.py

@@ -0,0 +1,35 @@
+import json
+import glob
+
+def fix_notebook(file_path):
+    with open(file_path, 'r', encoding='utf-8') as f:
+        nb = json.load(f)
+    changed = False
+    for cell in nb.get('cells', []):
+        if cell.get('cell_type') == 'code':
+            source = cell.get('source', [])
+            if isinstance(source, list):
+                for i, line in enumerate(source):
+                    if "dc = datacube.Datacube()" in line:
+                        source[i] = "dc = None\n"
+                        changed = True
+                    if "ds = dc.load(" in line:
+                        source[i] = "ds = None\n"
+                        changed = True
+                    if "data = dc.load(" in line:
+                        source[i] = "data = None\n"
+                        changed = True
+                    # If ds is None, ds.vv will fail
+                    if "vv_data = ds.vv" in line:
+                        source[i] = "vv_data = None\n"
+                        changed = True
+                    if "notebook_utils.xarray_object_size(ds)" in line:
+                        source[i] = line.replace("notebook_utils.xarray_object_size(ds)", "'ds is None'")
+                        changed = True
+    if changed:
+        with open(file_path, 'w', encoding='utf-8') as f:
+            json.dump(nb, f, indent=1)
+        print(f"Removed datacube from {file_path}")
+
+for nb in glob.glob("*.ipynb"):
+    fix_notebook(nb)

requirement.txt

@@ -1,3 +1,4 @@
+affine @ file:///home/conda/feedstock_root/build_artifacts/affine_1733762038348/work
 aiobotocore==2.25.0
 aiohappyeyeballs==2.6.1
 aiohttp==3.12.15
@@ -6,63 +7,162 @@ aiosignal==1.4.0
 alembic==1.16.5
 annotated-doc==0.0.4
 annotated-types==0.7.0
+antimeridian @ file:///home/conda/feedstock_root/build_artifacts/antimeridian_1753706324394/work
+anyio @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_anyio_1758634638/work
+argon2-cffi @ file:///home/conda/feedstock_root/build_artifacts/argon2-cffi_1749017159514/work
+argon2-cffi-bindings @ file:///home/conda/feedstock_root/build_artifacts/argon2-cffi-bindings_1649500328244/work
+arrow @ file:///home/conda/feedstock_root/build_artifacts/arrow_1733584251875/work
 asciitree==0.3.3
+asttokens @ file:///home/conda/feedstock_root/build_artifacts/asttokens_1733250440834/work
+async-lru @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_async-lru_1742153708/work
+async-timeout==3.0.1
+attrs @ file:///home/conda/feedstock_root/build_artifacts/attrs_1741918516150/work
+babel @ file:///home/conda/feedstock_root/build_artifacts/babel_1738490167835/work
+beautifulsoup4 @ file:///home/conda/feedstock_root/build_artifacts/beautifulsoup4_1759146011391/work
+bleach @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_bleach_1737382993/work
 blinker==1.9.0
 bokeh==3.7.3
 boto3==1.40.18
 botocore==1.40.49
+Bottleneck @ file:///croot/bottleneck_1731058641041/work
+branca @ file:///croot/branca_1675157607453/work
+Brotli @ file:///croot/brotli-split_1736182456865/work
+brotlicffi @ file:///croot/brotlicffi_1736182461069/work
+cached-property @ file:///home/conda/feedstock_root/build_artifacts/cached_property_1615209429212/work
 cachetools==6.2.0
 Cartopy==0.25.0
+certifi @ file:///home/conda/feedstock_root/build_artifacts/certifi_1759648874697/work/certifi
+cffi @ file:///croot/cffi_1736182485317/work
+cftime @ file:///home/conda/feedstock_root/build_artifacts/cftime_1649636873066/work
+chardet @ file:///home/conda/feedstock_root/build_artifacts/chardet_1649184137891/work
+charset-normalizer @ file:///croot/charset-normalizer_1721748349566/work
 ciso8601==2.3.3
+click @ file:///home/conda/feedstock_root/build_artifacts/click_1747811314515/work
+click-plugins @ file:///home/conda/feedstock_root/build_artifacts/click-plugins_1750848229740/work
+cligj @ file:///home/conda/feedstock_root/build_artifacts/cligj_1733749956636/work
+cloudpickle @ file:///home/conda/feedstock_root/build_artifacts/cloudpickle_1736947526808/work
 colorama==0.4.6
 colorcet==3.1.0
+comm @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_comm_1753453984/work
+contourpy @ file:///croot/contourpy_1732540045555/work
+cycler @ file:///tmp/build/80754af9/cycler_1637851556182/work
+cytoolz==0.11.2
+dask @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_dask-core_1760473436/work
+dask-gateway @ file:///Users/runner/miniforge3/conda-bld/bld/rattler-build_dask-gateway_1744370153/work/dask-gateway
+dask-glm @ file:///home/conda/feedstock_root/build_artifacts/dask-glm_1701346265909/work
 dask-image==2024.5.3
-datacube==1.9.4
+dask-ml @ file:///home/conda/feedstock_root/build_artifacts/dask-ml_1679705292494/work
+datacube==1.8.15
 datacube_ows==1.9.4
 datashader==0.18.2
 dea-tools==0.3.0
+debugpy @ file:///home/task_175706711740264/conda-bld/debugpy_1757067131873/work
+decorator @ file:///home/conda/feedstock_root/build_artifacts/decorator_1740384970518/work
 deepdiff==8.6.1
+defusedxml @ file:///home/conda/feedstock_root/build_artifacts/defusedxml_1615232257335/work
+deprecat @ file:///home/conda/feedstock_root/build_artifacts/deprecat_1734684036993/work
+distributed @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_distributed_1760476147/work
 eo-tides==0.8.2
+exceptiongroup @ file:///home/conda/feedstock_root/build_artifacts/exceptiongroup_1746947292760/work
+executing @ file:///home/conda/feedstock_root/build_artifacts/executing_1756729339227/work
 fastapi==0.124.3
+fasteners @ file:///home/conda/feedstock_root/build_artifacts/fasteners_1734943108928/work
+fastjsonschema @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_python-fastjsonschema_1755304154/work/dist
 filelock==3.19.1
 fiona==1.10.1
 Flask==3.1.2
 flask-babel==4.0.0
 flatbuffers==25.2.10
 folium==0.20.0
+fonttools @ file:///croot/fonttools_1737039080035/work
+fqdn @ file:///home/conda/feedstock_root/build_artifacts/fqdn_1733327382592/work/dist
 frozenlist==1.7.0
+fsspec @ file:///home/conda/feedstock_root/build_artifacts/fsspec_1756908513222/work
+GDAL @ file:///croot/gdal-split_1734448174900/work/build/swig/python
+GeoAlchemy2 @ file:///home/conda/feedstock_root/build_artifacts/geoalchemy2_1753372953474/work
 geographiclib==2.1
 geojson==3.2.0
 geomad==1.0.0
+geopandas @ file:///croot/geopandas-split_1755761494241/work
 geopy==2.4.1
-git-filter-repo==2.47.0
+greenlet @ file:///home/conda/feedstock_root/build_artifacts/greenlet_1648882383677/work
+h11 @ file:///home/conda/feedstock_root/build_artifacts/h11_1745526374115/work
+h2 @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_h2_1756364871/work
 h3==4.3.1
 hdstats==0.2.1
 holoviews==1.21.0
+hpack @ file:///home/conda/feedstock_root/build_artifacts/hpack_1737618293087/work
+httpcore @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_httpcore_1745602916/work
+httpx @ file:///home/conda/feedstock_root/build_artifacts/httpx_1733663348460/work
 hvplot==0.12.1
+hyperframe @ file:///home/conda/feedstock_root/build_artifacts/hyperframe_1737618333194/work
 idna==3.10
 imagecodecs==2025.3.30
 imageio==2.37.0
+importlib_metadata @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_importlib-metadata_1747934053/work
+ipykernel @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_ipykernel_1760459840/work
 ipyleaflet==0.20.0
+ipython @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_ipython_1748711175/work
 ipywidgets==8.1.7
 iso8601==2.1.0
+isoduration @ file:///home/conda/feedstock_root/build_artifacts/isoduration_1733493628631/work/dist
 itsdangerous==2.2.0
+jedi @ file:///home/conda/feedstock_root/build_artifacts/jedi_1733300866624/work
+Jinja2 @ file:///croot/jinja2_1741710844255/work
+jmespath @ file:///home/conda/feedstock_root/build_artifacts/jmespath_1733229141657/work
+joblib @ file:///home/conda/feedstock_root/build_artifacts/joblib_1756321760188/work
+json5 @ file:///home/conda/feedstock_root/build_artifacts/json5_1755034879854/work
+jsonpointer @ file:///home/conda/feedstock_root/build_artifacts/jsonpointer_1756754132747/work
+jsonschema @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_jsonschema_1755595646/work
 jsonschema-specifications==2025.4.1
+jupyter-events @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_jupyter_events_1738765986/work
 jupyter-leaflet==0.20.0
+jupyter-lsp @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_jupyter-lsp_1756388269/work/jupyter-lsp
 jupyter-ui-poll==1.0.0
+jupyter_client @ file:///home/conda/feedstock_root/build_artifacts/jupyter_client_1733440914442/work
+jupyter_core @ file:///home/conda/feedstock_root/build_artifacts/jupyter_core_1748333051527/work
+jupyter_server @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_jupyter_server_1755870522/work
+jupyter_server_terminals @ file:///home/conda/feedstock_root/build_artifacts/jupyter_server_terminals_1733427956852/work
+jupyterlab @ file:///home/conda/feedstock_root/build_artifacts/jupyterlab_1758913905644/work
+jupyterlab_pygments @ file:///home/conda/feedstock_root/build_artifacts/jupyterlab_pygments_1733328101776/work
+jupyterlab_server @ file:///home/conda/feedstock_root/build_artifacts/jupyterlab_server_1733599573484/work
 jupyterlab_widgets==3.0.15
+kiwisolver @ file:///croot/kiwisolver_1737039087198/work
 lark==1.2.2
 lark-parser==0.12.0
 lazy_loader==0.4
 linkify-it-py==2.0.3
+llvmlite @ file:///croot/llvmlite_1741209858218/work
+locket @ file:///home/conda/feedstock_root/build_artifacts/locket_1650660393415/work
 lxml==5.4.0
+lz4 @ file:///croot/lz4_1736366683208/work
+Mako @ file:///home/conda/feedstock_root/build_artifacts/mako_1744317760971/work
+mapclassify @ file:///croot/mapclassify_1675157730177/work
 Markdown==3.9
 markdown-it-py==4.0.0
+MarkupSafe @ file:///croot/markupsafe_1738584038848/work
 matplotlib==3.10.5
+matplotlib-inline @ file:///home/conda/feedstock_root/build_artifacts/matplotlib-inline_1733416936468/work
 mdit-py-plugins==0.5.0
 mdurl==0.1.2
+mistune @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_mistune_1756495311/work
 mpmath==1.3.0
+msgpack @ file:///home/conda/feedstock_root/build_artifacts/msgpack-python_1648745999384/work
+multidict @ file:///home/conda/feedstock_root/build_artifacts/multidict_1648882415384/work
+multipledispatch @ file:///home/conda/feedstock_root/build_artifacts/multipledispatch_1721907546485/work
 narwhals==2.3.0
+nbclient @ file:///home/conda/feedstock_root/build_artifacts/nbclient_1734628800805/work
+nbconvert @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_nbconvert-core_1738067871/work
+nbformat @ file:///home/conda/feedstock_root/build_artifacts/nbformat_1733402752141/work
+nest_asyncio @ file:///home/conda/feedstock_root/build_artifacts/nest-asyncio_1733325553580/work
+netCDF4 @ file:///croot/netcdf4_1743512888672/work
+networkx @ file:///croot/networkx_1737039604450/work
+notebook @ file:///home/conda/feedstock_root/build_artifacts/notebook_1759152069573/work
+notebook_shim @ file:///home/conda/feedstock_root/build_artifacts/notebook-shim_1733408315203/work
+numba @ file:///croot/numba_1750798165355/work
+numcodecs @ file:///croot/numcodecs_1707513121886/work
+numexpr @ file:///croot/numexpr_1755766469354/work
+numpy @ file:///croot/numpy_and_numpy_base_1755590845055/work/dist/numpy-1.26.4-cp310-cp310-linux_x86_64.whl#sha256=1096d33ad9a9757a1b4b46634d809e894263fc8b78780bff36801684b6e8cc88
 nvidia-cublas-cu12==12.8.4.1
 nvidia-cuda-cupti-cu12==12.8.90
 nvidia-cuda-nvrtc-cu12==12.8.93
@@ -74,67 +174,136 @@ nvidia-curand-cu12==10.3.9.90
 nvidia-cusolver-cu12==11.7.3.90
 nvidia-cusparse-cu12==12.5.8.93
 nvidia-cusparselt-cu12==0.7.1
-nvidia-nccl-cu12==2.27.5
+nvidia-nccl-cu12==2.27.3
 nvidia-nvjitlink-cu12==12.8.93
-nvidia-nvshmem-cu12==3.3.20
 nvidia-nvtx-cu12==12.8.90
 odc-algo==0.2.3
 odc-geo==0.4.10
 odc-io==0.2.2
+odc-loader @ file:///home/conda/feedstock_root/build_artifacts/odc-loader_1743656085024/work
+odc-stac @ file:///home/conda/feedstock_root/build_artifacts/odc-stac_1746136311934/work
 odc-ui==0.2.1
 orderly-set==5.5.0
+overrides @ file:///home/conda/feedstock_root/build_artifacts/overrides_1734587627321/work
 OWSLib==0.34.1
+packaging @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_packaging_1745345660/work
+pandas @ file:///home/task_175982153789305/conda-bld/pandas_1759822248912/work/dist/pandas-2.3.3-cp310-cp310-linux_x86_64.whl#sha256=0de7c83109c411cc2a74419a396c92f65e3d1e457fb4d835e5f100cfb04393a7
+pandocfilters @ file:///home/conda/feedstock_root/build_artifacts/pandocfilters_1631603243851/work
 panel==1.7.5
 param==2.2.1
+parso @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_parso_1755974222/work
+partd @ file:///home/conda/feedstock_root/build_artifacts/partd_1715026491486/work
+pexpect @ file:///home/conda/feedstock_root/build_artifacts/pexpect_1733301927746/work
+pickleshare @ file:///home/conda/feedstock_root/build_artifacts/pickleshare_1733327343728/work
+pillow @ file:///croot/pillow_1738010226202/work
 PIMS==0.7
 planetary-computer==1.0.0
+platformdirs @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_platformdirs_1756227402/work
 prometheus_client==0.22.1
 prometheus_flask_exporter==0.23.2
+prompt_toolkit @ file:///home/conda/feedstock_root/build_artifacts/prompt-toolkit_1756321756983/work
 propcache==0.3.2
+psutil @ file:///home/conda/feedstock_root/build_artifacts/psutil_1653089181607/work
+psycopg2 @ file:///croot/psycopg2_1744919787325/work
+ptyprocess @ file:///home/conda/feedstock_root/build_artifacts/ptyprocess_1733302279685/work/dist/ptyprocess-0.7.0-py2.py3-none-any.whl#sha256=92c32ff62b5fd8cf325bec5ab90d7be3d2a8ca8c8a3813ff487a8d2002630d1f
+pure_eval @ file:///home/conda/feedstock_root/build_artifacts/pure_eval_1733569405015/work
+pyarrow @ file:///home/task_175983338836370/conda-bld/pyarrow_1759833584228/work/python
+pycparser @ file:///tmp/build/80754af9/pycparser_1636541352034/work
 pyct==0.5.0
 pydantic==2.11.7
 pydantic_core==2.33.2
+Pygments @ file:///home/conda/feedstock_root/build_artifacts/pygments_1750615794071/work
+pyogrio @ file:///croot/pyogrio_1741107161422/work
 pyows==0.3.1
+pyparsing @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_pyparsing_1753873557/work
+pyproj @ file:///croot/pyproj_1739284761968/work
 PyQt6==6.7.1
+PyQt6_sip @ file:///croot/pyqt-split_1753427276959/work/pyqt_sip
 pyshp==2.3.1
+PySocks @ file:///home/builder/ci_310/pysocks_1640793678128/work
+pystac @ file:///home/conda/feedstock_root/build_artifacts/pystac_1758218055393/work
 pystac-client==0.9.0
 python-dateutil==2.9.0.post0
 python-dotenv==1.1.1
-python-multipart==0.0.21
+python-json-logger @ file:///home/conda/feedstock_root/build_artifacts/python-json-logger_1677079630776/work
 python-slugify==8.0.4
 pyTMD==2.2.8
+pytz @ file:///home/conda/feedstock_root/build_artifacts/pytz_1742920838005/work
 pyviz_comms==3.0.6
 PyYAML==6.0.2
+pyzmq @ file:///croot/pyzmq_1734687138743/work
+rasterio @ file:///croot/rasterio_1740069178893/work
 rasterstats==0.20.0
 referencing==0.36.2
 regex==2025.9.1
+requests @ file:///croot/requests_1756709366904/work
+rfc3339_validator @ file:///home/conda/feedstock_root/build_artifacts/rfc3339-validator_1733599910982/work
+rfc3986-validator @ file:///home/conda/feedstock_root/build_artifacts/rfc3986-validator_1598024191506/work
 rfc3987==1.3.8
+rfc3987-syntax @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_rfc3987-syntax_1752876729/work
+rioxarray @ file:///home/conda/feedstock_root/build_artifacts/rioxarray_1737140588464/work
+rpds-py @ file:///croot/rpds-py_1736541261634/work
+ruamel.yaml @ file:///home/conda/feedstock_root/build_artifacts/ruamel.yaml_1649033201098/work
 ruamel.yaml.clib==0.2.12
 s3fs==2025.9.0
 s3transfer==0.13.1
 scikit-image==0.25.2
 scikit-learn==1.7.1
+scipy @ file:///croot/scipy_1747238027288/work/dist/scipy-1.15.3-cp310-cp310-linux_x86_64.whl#sha256=2a791554880ad4f358fcc4cd2a982ffe1e9d472e9241011216b2be797457f1f9
 seaborn==0.13.2
+Send2Trash @ file:///home/conda/feedstock_root/build_artifacts/send2trash_1733322040660/work
 setuptools-scm==9.2.0
+shapely @ file:///croot/shapely_1754380812723/work
 simplejson==3.20.1
+sip @ file:///croot/sip_1738856193618/work
 six==1.17.0
 slicerator==1.1.0
-SQLAlchemy==2.0.0
+sniffio @ file:///home/conda/feedstock_root/build_artifacts/sniffio_1733244044561/work
+snuggs @ file:///home/conda/feedstock_root/build_artifacts/snuggs_1733818638588/work
+sortedcontainers @ file:///home/conda/feedstock_root/build_artifacts/sortedcontainers_1738440353519/work
+soupsieve @ file:///home/conda/feedstock_root/build_artifacts/soupsieve_1756330469801/work
+sparse @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_sparse_1747799051/work
+SQLAlchemy==1.4.54
+stack_data @ file:///home/conda/feedstock_root/build_artifacts/stack_data_1733569443808/work
 starlette==0.50.0
 sympy==1.14.0
+tblib @ file:///home/conda/feedstock_root/build_artifacts/tblib_1743515515538/work
+terminado @ file:///home/conda/feedstock_root/build_artifacts/terminado_1710262609923/work
 text-unidecode==1.3
+threadpoolctl @ file:///home/conda/feedstock_root/build_artifacts/threadpoolctl_1741878222898/work
 tifffile==2025.5.10
 timescale==0.0.9
 timezonefinder==8.0.0
-torch==2.9.1
-torchvision==0.24.1
+tinycss2 @ file:///home/conda/feedstock_root/build_artifacts/tinycss2_1729802851396/work
+tomli @ file:///croot/tomli_1753774587605/work
+toolz @ file:///home/conda/feedstock_root/build_artifacts/toolz_1733736030883/work
+torch==2.8.0
+tornado @ file:///croot/tornado_1748956929273/work
 tqdm==4.67.1
+traitlets @ file:///home/conda/feedstock_root/build_artifacts/traitlets_1733367359838/work
 traittypes==0.2.1
-triton==3.5.1
+triton==3.4.0
+types-python-dateutil @ file:///home/conda/feedstock_root/build_artifacts/types-python-dateutil_1759899809376/work
 typing-inspection==0.4.1
+typing_extensions @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_typing_extensions_1756220668/work
+typing_utils @ file:///home/conda/feedstock_root/build_artifacts/typing_utils_1733331286120/work
+tzdata @ file:///croot/python-tzdata_1746123641790/work
 uc-micro-py==1.0.3
+unicodedata2 @ file:///croot/unicodedata2_1736541023050/work
+uri-template @ file:///home/conda/feedstock_root/build_artifacts/uri-template_1733323593477/work/dist
+urllib3 @ file:///croot/urllib3_1750775463400/work
 uvicorn==0.38.0
+wcwidth @ file:///home/conda/feedstock_root/build_artifacts/wcwidth_1733231326287/work
+webcolors @ file:///home/conda/feedstock_root/build_artifacts/webcolors_1733359735138/work
+webencodings @ file:///home/conda/feedstock_root/build_artifacts/webencodings_1733236011802/work
+websocket-client @ file:///home/conda/feedstock_root/build_artifacts/websocket-client_1759928050786/work
 Werkzeug==3.1.3
 widgetsnbextension==4.0.14
+wrapt @ file:///home/conda/feedstock_root/build_artifacts/wrapt_1651495243689/work
+xarray @ file:///home/conda/feedstock_root/build_artifacts/xarray_1749743207754/work
 xgboost==3.1.2
+xyzservices @ file:///croot/xyzservices_1675159059961/work
 yarl==1.20.1
+zarr @ file:///home/conda/feedstock_root/build_artifacts/zarr_1733237197728/work
+zict @ file:///home/conda/feedstock_root/build_artifacts/zict_1733261551178/work
+zipp @ file:///home/conda/feedstock_root/build_artifacts/zipp_1749421620841/work

requirement.txt.backup

@@ -1,313 +0,0 @@
-affine @ file:///home/conda/feedstock_root/build_artifacts/affine_1733762038348/work
-aiobotocore==2.25.0
-aiohappyeyeballs==2.6.1
-aiohttp==3.12.15
-aioitertools==0.12.0
-aiosignal==1.4.0
-alembic==1.16.5
-annotated-doc==0.0.4
-annotated-types==0.7.0
-antimeridian @ file:///home/conda/feedstock_root/build_artifacts/antimeridian_1753706324394/work
-anyio @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_anyio_1758634638/work
-argon2-cffi @ file:///home/conda/feedstock_root/build_artifacts/argon2-cffi_1749017159514/work
-argon2-cffi-bindings @ file:///home/conda/feedstock_root/build_artifacts/argon2-cffi-bindings_1649500328244/work
-arrow @ file:///home/conda/feedstock_root/build_artifacts/arrow_1733584251875/work
-asciitree==0.3.3
-asttokens @ file:///home/conda/feedstock_root/build_artifacts/asttokens_1733250440834/work
-async-lru @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_async-lru_1742153708/work
-async-timeout==3.0.1
-attrs @ file:///home/conda/feedstock_root/build_artifacts/attrs_1741918516150/work
-babel @ file:///home/conda/feedstock_root/build_artifacts/babel_1738490167835/work
-beautifulsoup4 @ file:///home/conda/feedstock_root/build_artifacts/beautifulsoup4_1759146011391/work
-bleach @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_bleach_1737382993/work
-blinker==1.9.0
-bokeh==3.7.3
-boto3==1.40.18
-botocore==1.40.49
-Bottleneck @ file:///croot/bottleneck_1731058641041/work
-branca @ file:///croot/branca_1675157607453/work
-Brotli @ file:///croot/brotli-split_1736182456865/work
-brotlicffi @ file:///croot/brotlicffi_1736182461069/work
-cached-property @ file:///home/conda/feedstock_root/build_artifacts/cached_property_1615209429212/work
-cachetools==6.2.0
-Cartopy==0.25.0
-certifi @ file:///home/conda/feedstock_root/build_artifacts/certifi_1762976168352/work/certifi
-cffi @ file:///croot/cffi_1736182485317/work
-cftime @ file:///home/conda/feedstock_root/build_artifacts/cftime_1649636873066/work
-chardet @ file:///home/conda/feedstock_root/build_artifacts/chardet_1649184137891/work
-charset-normalizer @ file:///croot/charset-normalizer_1721748349566/work
-ciso8601==2.3.3
-click @ file:///home/conda/feedstock_root/build_artifacts/click_1747811314515/work
-click-plugins @ file:///home/conda/feedstock_root/build_artifacts/click-plugins_1750848229740/work
-cligj @ file:///home/conda/feedstock_root/build_artifacts/cligj_1733749956636/work
-cloudpickle @ file:///home/conda/feedstock_root/build_artifacts/cloudpickle_1736947526808/work
-colorama==0.4.6
-colorcet==3.1.0
-comm @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_comm_1753453984/work
-contourpy @ file:///croot/contourpy_1732540045555/work
-cycler @ file:///tmp/build/80754af9/cycler_1637851556182/work
-cytoolz==0.11.2
-dask @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_dask-core_1760473436/work
-dask-gateway @ file:///Users/runner/miniforge3/conda-bld/bld/rattler-build_dask-gateway_1744370153/work/dask-gateway
-dask-glm @ file:///home/conda/feedstock_root/build_artifacts/dask-glm_1701346265909/work
-dask-image==2024.5.3
-dask-ml @ file:///home/conda/feedstock_root/build_artifacts/dask-ml_1679705292494/work
-datacube==1.8.15
-datacube_ows==1.9.4
-datashader==0.18.2
-dea-tools==0.3.0
-debugpy @ file:///home/task_175706711740264/conda-bld/debugpy_1757067131873/work
-decorator @ file:///home/conda/feedstock_root/build_artifacts/decorator_1740384970518/work
-deepdiff==8.6.1
-defusedxml @ file:///home/conda/feedstock_root/build_artifacts/defusedxml_1615232257335/work
-deprecat @ file:///home/conda/feedstock_root/build_artifacts/deprecat_1734684036993/work
-distributed @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_distributed_1760476147/work
-eo-tides==0.8.2
-exceptiongroup @ file:///home/conda/feedstock_root/build_artifacts/exceptiongroup_1746947292760/work
-executing @ file:///home/conda/feedstock_root/build_artifacts/executing_1756729339227/work
-fastapi==0.124.3
-fasteners @ file:///home/conda/feedstock_root/build_artifacts/fasteners_1734943108928/work
-fastjsonschema @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_python-fastjsonschema_1755304154/work/dist
-filelock==3.19.1
-fiona==1.10.1
-Flask==3.1.2
-flask-babel==4.0.0
-flatbuffers==25.2.10
-folium==0.20.0
-fonttools @ file:///croot/fonttools_1737039080035/work
-fqdn @ file:///home/conda/feedstock_root/build_artifacts/fqdn_1733327382592/work/dist
-frozenlist==1.7.0
-fsspec @ file:///home/conda/feedstock_root/build_artifacts/fsspec_1756908513222/work
-GDAL @ file:///croot/gdal-split_1734448174900/work/build/swig/python
-GeoAlchemy2 @ file:///home/conda/feedstock_root/build_artifacts/geoalchemy2_1753372953474/work
-geographiclib==2.1
-geojson==3.2.0
-geomad==1.0.0
-geopandas @ file:///croot/geopandas-split_1755761494241/work
-geopy==2.4.1
-git-filter-repo==2.47.0
-greenlet @ file:///home/conda/feedstock_root/build_artifacts/greenlet_1648882383677/work
-h11 @ file:///home/conda/feedstock_root/build_artifacts/h11_1745526374115/work
-h2 @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_h2_1756364871/work
-h3==4.3.1
-hdstats==0.2.1
-holoviews==1.21.0
-hpack @ file:///home/conda/feedstock_root/build_artifacts/hpack_1737618293087/work
-httpcore @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_httpcore_1745602916/work
-httpx @ file:///home/conda/feedstock_root/build_artifacts/httpx_1733663348460/work
-hvplot==0.12.1
-hyperframe @ file:///home/conda/feedstock_root/build_artifacts/hyperframe_1737618333194/work
-idna==3.10
-imagecodecs==2025.3.30
-imageio==2.37.0
-importlib_metadata @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_importlib-metadata_1747934053/work
-ipykernel @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_ipykernel_1760459840/work
-ipyleaflet==0.20.0
-ipython @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_ipython_1748711175/work
-ipywidgets==8.1.7
-iso8601==2.1.0
-isoduration @ file:///home/conda/feedstock_root/build_artifacts/isoduration_1733493628631/work/dist
-itsdangerous==2.2.0
-jedi @ file:///home/conda/feedstock_root/build_artifacts/jedi_1733300866624/work
-Jinja2 @ file:///croot/jinja2_1741710844255/work
-jmespath @ file:///home/conda/feedstock_root/build_artifacts/jmespath_1733229141657/work
-joblib @ file:///home/conda/feedstock_root/build_artifacts/joblib_1756321760188/work
-json5 @ file:///home/conda/feedstock_root/build_artifacts/json5_1755034879854/work
-jsonpointer @ file:///home/conda/feedstock_root/build_artifacts/jsonpointer_1756754132747/work
-jsonschema @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_jsonschema_1755595646/work
-jsonschema-specifications==2025.4.1
-jupyter-events @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_jupyter_events_1738765986/work
-jupyter-leaflet==0.20.0
-jupyter-lsp @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_jupyter-lsp_1756388269/work/jupyter-lsp
-jupyter-ui-poll==1.0.0
-jupyter_client @ file:///home/conda/feedstock_root/build_artifacts/jupyter_client_1733440914442/work
-jupyter_core @ file:///home/conda/feedstock_root/build_artifacts/jupyter_core_1748333051527/work
-jupyter_server @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_jupyter_server_1755870522/work
-jupyter_server_terminals @ file:///home/conda/feedstock_root/build_artifacts/jupyter_server_terminals_1733427956852/work
-jupyterlab @ file:///home/conda/feedstock_root/build_artifacts/jupyterlab_1758913905644/work
-jupyterlab_pygments @ file:///home/conda/feedstock_root/build_artifacts/jupyterlab_pygments_1733328101776/work
-jupyterlab_server @ file:///home/conda/feedstock_root/build_artifacts/jupyterlab_server_1733599573484/work
-jupyterlab_widgets==3.0.15
-kiwisolver @ file:///croot/kiwisolver_1737039087198/work
-lark==1.2.2
-lark-parser==0.12.0
-lazy_loader==0.4
-linkify-it-py==2.0.3
-llvmlite @ file:///croot/llvmlite_1741209858218/work
-locket @ file:///home/conda/feedstock_root/build_artifacts/locket_1650660393415/work
-lxml==5.4.0
-lz4 @ file:///croot/lz4_1736366683208/work
-Mako @ file:///home/conda/feedstock_root/build_artifacts/mako_1744317760971/work
-mapclassify @ file:///croot/mapclassify_1675157730177/work
-Markdown==3.9
-markdown-it-py==4.0.0
-MarkupSafe @ file:///croot/markupsafe_1738584038848/work
-matplotlib==3.10.5
-matplotlib-inline @ file:///home/conda/feedstock_root/build_artifacts/matplotlib-inline_1733416936468/work
-mdit-py-plugins==0.5.0
-mdurl==0.1.2
-mistune @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_mistune_1756495311/work
-mpmath==1.3.0
-msgpack @ file:///home/conda/feedstock_root/build_artifacts/msgpack-python_1648745999384/work
-multidict @ file:///home/conda/feedstock_root/build_artifacts/multidict_1648882415384/work
-multipledispatch @ file:///home/conda/feedstock_root/build_artifacts/multipledispatch_1721907546485/work
-narwhals==2.3.0
-nbclient @ file:///home/conda/feedstock_root/build_artifacts/nbclient_1734628800805/work
-nbconvert @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_nbconvert-core_1738067871/work
-nbformat @ file:///home/conda/feedstock_root/build_artifacts/nbformat_1733402752141/work
-nest_asyncio @ file:///home/conda/feedstock_root/build_artifacts/nest-asyncio_1733325553580/work
-netCDF4 @ file:///croot/netcdf4_1743512888672/work
-networkx @ file:///croot/networkx_1737039604450/work
-notebook @ file:///home/conda/feedstock_root/build_artifacts/notebook_1759152069573/work
-notebook_shim @ file:///home/conda/feedstock_root/build_artifacts/notebook-shim_1733408315203/work
-numba @ file:///croot/numba_1750798165355/work
-numcodecs @ file:///croot/numcodecs_1707513121886/work
-numexpr @ file:///croot/numexpr_1755766469354/work
-numpy @ file:///croot/numpy_and_numpy_base_1755590845055/work/dist/numpy-1.26.4-cp310-cp310-linux_x86_64.whl#sha256=1096d33ad9a9757a1b4b46634d809e894263fc8b78780bff36801684b6e8cc88
-nvidia-cublas-cu12==12.8.4.1
-nvidia-cuda-cupti-cu12==12.8.90
-nvidia-cuda-nvrtc-cu12==12.8.93
-nvidia-cuda-runtime-cu12==12.8.90
-nvidia-cudnn-cu12==9.10.2.21
-nvidia-cufft-cu12==11.3.3.83
-nvidia-cufile-cu12==1.13.1.3
-nvidia-curand-cu12==10.3.9.90
-nvidia-cusolver-cu12==11.7.3.90
-nvidia-cusparse-cu12==12.5.8.93
-nvidia-cusparselt-cu12==0.7.1
-nvidia-nccl-cu12==2.27.5
-nvidia-nvjitlink-cu12==12.8.93
-nvidia-nvshmem-cu12==3.3.20
-nvidia-nvtx-cu12==12.8.90
-odc-algo==0.2.3
-odc-geo==0.4.10
-odc-io==0.2.2
-odc-loader @ file:///home/conda/feedstock_root/build_artifacts/odc-loader_1743656085024/work
-odc-stac @ file:///home/conda/feedstock_root/build_artifacts/odc-stac_1746136311934/work
-odc-ui==0.2.1
-orderly-set==5.5.0
-overrides @ file:///home/conda/feedstock_root/build_artifacts/overrides_1734587627321/work
-OWSLib==0.34.1
-packaging @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_packaging_1745345660/work
-pandas @ file:///home/task_175982153789305/conda-bld/pandas_1759822248912/work/dist/pandas-2.3.3-cp310-cp310-linux_x86_64.whl#sha256=0de7c83109c411cc2a74419a396c92f65e3d1e457fb4d835e5f100cfb04393a7
-pandocfilters @ file:///home/conda/feedstock_root/build_artifacts/pandocfilters_1631603243851/work
-panel==1.7.5
-param==2.2.1
-parso @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_parso_1755974222/work
-partd @ file:///home/conda/feedstock_root/build_artifacts/partd_1715026491486/work
-pexpect @ file:///home/conda/feedstock_root/build_artifacts/pexpect_1733301927746/work
-pickleshare @ file:///home/conda/feedstock_root/build_artifacts/pickleshare_1733327343728/work
-pillow @ file:///croot/pillow_1738010226202/work
-PIMS==0.7
-planetary-computer==1.0.0
-platformdirs @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_platformdirs_1756227402/work
-prometheus_client==0.22.1
-prometheus_flask_exporter==0.23.2
-prompt_toolkit @ file:///home/conda/feedstock_root/build_artifacts/prompt-toolkit_1756321756983/work
-propcache==0.3.2
-psutil @ file:///home/conda/feedstock_root/build_artifacts/psutil_1653089181607/work
-psycopg2 @ file:///croot/psycopg2_1744919787325/work
-ptyprocess @ file:///home/conda/feedstock_root/build_artifacts/ptyprocess_1733302279685/work/dist/ptyprocess-0.7.0-py2.py3-none-any.whl#sha256=92c32ff62b5fd8cf325bec5ab90d7be3d2a8ca8c8a3813ff487a8d2002630d1f
-pure_eval @ file:///home/conda/feedstock_root/build_artifacts/pure_eval_1733569405015/work
-pyarrow @ file:///home/task_175983338836370/conda-bld/pyarrow_1759833584228/work/python
-pycparser @ file:///tmp/build/80754af9/pycparser_1636541352034/work
-pyct==0.5.0
-pydantic==2.11.7
-pydantic_core==2.33.2
-Pygments @ file:///home/conda/feedstock_root/build_artifacts/pygments_1750615794071/work
-pyogrio @ file:///croot/pyogrio_1741107161422/work
-pyows==0.3.1
-pyparsing @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_pyparsing_1753873557/work
-pyproj @ file:///croot/pyproj_1739284761968/work
-PyQt6==6.7.1
-PyQt6_sip @ file:///croot/pyqt-split_1753427276959/work/pyqt_sip
-pyshp==2.3.1
-PySocks @ file:///home/builder/ci_310/pysocks_1640793678128/work
-pystac @ file:///home/conda/feedstock_root/build_artifacts/pystac_1758218055393/work
-pystac-client==0.9.0
-python-dateutil==2.9.0.post0
-python-dotenv==1.1.1
-python-json-logger @ file:///home/conda/feedstock_root/build_artifacts/python-json-logger_1677079630776/work
-python-multipart==0.0.21
-python-slugify==8.0.4
-pyTMD==2.2.8
-pytz @ file:///home/conda/feedstock_root/build_artifacts/pytz_1742920838005/work
-pyviz_comms==3.0.6
-PyYAML==6.0.2
-pyzmq @ file:///croot/pyzmq_1734687138743/work
-rasterio @ file:///croot/rasterio_1740069178893/work
-rasterstats==0.20.0
-referencing==0.36.2
-regex==2025.9.1
-requests @ file:///croot/requests_1756709366904/work
-rfc3339_validator @ file:///home/conda/feedstock_root/build_artifacts/rfc3339-validator_1733599910982/work
-rfc3986-validator @ file:///home/conda/feedstock_root/build_artifacts/rfc3986-validator_1598024191506/work
-rfc3987==1.3.8
-rfc3987-syntax @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_rfc3987-syntax_1752876729/work
-rioxarray @ file:///home/conda/feedstock_root/build_artifacts/rioxarray_1737140588464/work
-rpds-py @ file:///croot/rpds-py_1736541261634/work
-ruamel.yaml @ file:///home/conda/feedstock_root/build_artifacts/ruamel.yaml_1649033201098/work
-ruamel.yaml.clib==0.2.12
-s3fs==2025.9.0
-s3transfer==0.13.1
-scikit-image==0.25.2
-scikit-learn==1.7.1
-scipy @ file:///croot/scipy_1747238027288/work/dist/scipy-1.15.3-cp310-cp310-linux_x86_64.whl#sha256=2a791554880ad4f358fcc4cd2a982ffe1e9d472e9241011216b2be797457f1f9
-seaborn==0.13.2
-Send2Trash @ file:///home/conda/feedstock_root/build_artifacts/send2trash_1733322040660/work
-setuptools-scm==9.2.0
-shapely @ file:///croot/shapely_1754380812723/work
-simplejson==3.20.1
-sip @ file:///croot/sip_1738856193618/work
-six==1.17.0
-slicerator==1.1.0
-sniffio @ file:///home/conda/feedstock_root/build_artifacts/sniffio_1733244044561/work
-snuggs @ file:///home/conda/feedstock_root/build_artifacts/snuggs_1733818638588/work
-sortedcontainers @ file:///home/conda/feedstock_root/build_artifacts/sortedcontainers_1738440353519/work
-soupsieve @ file:///home/conda/feedstock_root/build_artifacts/soupsieve_1756330469801/work
-sparse @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_sparse_1747799051/work
-SQLAlchemy==1.4.54
-stack_data @ file:///home/conda/feedstock_root/build_artifacts/stack_data_1733569443808/work
-starlette==0.50.0
-sympy==1.14.0
-tblib @ file:///home/conda/feedstock_root/build_artifacts/tblib_1743515515538/work
-terminado @ file:///home/conda/feedstock_root/build_artifacts/terminado_1710262609923/work
-text-unidecode==1.3
-threadpoolctl @ file:///home/conda/feedstock_root/build_artifacts/threadpoolctl_1741878222898/work
-tifffile==2025.5.10
-timescale==0.0.9
-timezonefinder==8.0.0
-tinycss2 @ file:///home/conda/feedstock_root/build_artifacts/tinycss2_1729802851396/work
-tomli @ file:///croot/tomli_1753774587605/work
-toolz @ file:///home/conda/feedstock_root/build_artifacts/toolz_1733736030883/work
-torch==2.9.1
-torchvision==0.24.1
-tornado @ file:///croot/tornado_1748956929273/work
-tqdm==4.67.1
-traitlets @ file:///home/conda/feedstock_root/build_artifacts/traitlets_1733367359838/work
-traittypes==0.2.1
-triton==3.5.1
-types-python-dateutil @ file:///home/conda/feedstock_root/build_artifacts/types-python-dateutil_1759899809376/work
-typing-inspection==0.4.1
-typing_extensions @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_typing_extensions_1756220668/work
-typing_utils @ file:///home/conda/feedstock_root/build_artifacts/typing_utils_1733331286120/work
-tzdata @ file:///croot/python-tzdata_1746123641790/work
-uc-micro-py==1.0.3
-unicodedata2 @ file:///croot/unicodedata2_1736541023050/work
-uri-template @ file:///home/conda/feedstock_root/build_artifacts/uri-template_1733323593477/work/dist
-urllib3 @ file:///croot/urllib3_1750775463400/work
-uvicorn==0.38.0
-wcwidth @ file:///home/conda/feedstock_root/build_artifacts/wcwidth_1733231326287/work
-webcolors @ file:///home/conda/feedstock_root/build_artifacts/webcolors_1733359735138/work
-webencodings @ file:///home/conda/feedstock_root/build_artifacts/webencodings_1733236011802/work
-websocket-client @ file:///home/conda/feedstock_root/build_artifacts/websocket-client_1759928050786/work
-Werkzeug==3.1.3
-widgetsnbextension==4.0.14
-wrapt @ file:///home/conda/feedstock_root/build_artifacts/wrapt_1651495243689/work
-xarray @ file:///home/conda/feedstock_root/build_artifacts/xarray_1749743207754/work
-xgboost==3.1.2
-xyzservices @ file:///croot/xyzservices_1675159059961/work
-yarl==1.20.1
-zarr @ file:///home/conda/feedstock_root/build_artifacts/zarr_1733237197728/work
-zict @ file:///home/conda/feedstock_root/build_artifacts/zict_1733261551178/work
-zipp @ file:///home/conda/feedstock_root/build_artifacts/zipp_1749421620841/work

requirements_clean.txt

@@ -1,142 +0,0 @@
-aiobotocore==2.25.0
-aiohappyeyeballs==2.6.1
-aiohttp==3.12.15
-aioitertools==0.12.0
-aiosignal==1.4.0
-alembic==1.16.5
-annotated-doc==0.0.4
-annotated-types==0.7.0
-asciitree==0.3.3
-async-timeout==3.0.1
-blinker==1.9.0
-bokeh==3.7.3
-boto3==1.40.18
-botocore==1.40.49
-cachetools==6.2.0
-Cartopy==0.25.0
-ciso8601==2.3.3
-colorama==0.4.6
-colorcet==3.1.0
-cytoolz==0.11.2
-dask-image==2024.5.3
-datacube==1.9.4
-datacube_ows==1.9.4
-datashader==0.18.2
-dea-tools==0.3.0
-deepdiff==8.6.1
-eo-tides==0.8.2
-fastapi==0.124.3
-filelock==3.19.1
-fiona==1.10.1
-Flask==3.1.2
-flask-babel==4.0.0
-flatbuffers==25.2.10
-folium==0.20.0
-frozenlist==1.7.0
-geographiclib==2.1
-geojson==3.2.0
-geomad==1.0.0
-geopy==2.4.1
-git-filter-repo==2.47.0
-h3==4.3.1
-hdstats==0.2.1
-holoviews==1.21.0
-hvplot==0.12.1
-idna==3.10
-imagecodecs==2025.3.30
-imageio==2.37.0
-ipyleaflet==0.20.0
-ipywidgets==8.1.7
-iso8601==2.1.0
-itsdangerous==2.2.0
-jsonschema-specifications==2025.4.1
-jupyter-leaflet==0.20.0
-jupyter-ui-poll==1.0.0
-jupyterlab_widgets==3.0.15
-lark==1.2.2
-lark-parser==0.12.0
-lazy_loader==0.4
-linkify-it-py==2.0.3
-lxml==5.4.0
-Markdown==3.9
-markdown-it-py==4.0.0
-matplotlib==3.10.5
-mdit-py-plugins==0.5.0
-mdurl==0.1.2
-mpmath==1.3.0
-narwhals==2.3.0
-nvidia-cublas-cu12==12.8.4.1
-nvidia-cuda-cupti-cu12==12.8.90
-nvidia-cuda-nvrtc-cu12==12.8.93
-nvidia-cuda-runtime-cu12==12.8.90
-nvidia-cudnn-cu12==9.10.2.21
-nvidia-cufft-cu12==11.3.3.83
-nvidia-cufile-cu12==1.13.1.3
-nvidia-curand-cu12==10.3.9.90
-nvidia-cusolver-cu12==11.7.3.90
-nvidia-cusparse-cu12==12.5.8.93
-nvidia-cusparselt-cu12==0.7.1
-nvidia-nccl-cu12==2.27.5
-nvidia-nvjitlink-cu12==12.8.93
-nvidia-nvshmem-cu12==3.3.20
-nvidia-nvtx-cu12==12.8.90
-odc-algo==0.2.3
-odc-geo==0.4.10
-odc-io==0.2.2
-odc-ui==0.2.1
-orderly-set==5.5.0
-OWSLib==0.34.1
-panel==1.7.5
-param==2.2.1
-PIMS==0.7
-planetary-computer==1.0.0
-prometheus_client==0.22.1
-prometheus_flask_exporter==0.23.2
-propcache==0.3.2
-pyct==0.5.0
-pydantic==2.11.7
-pydantic_core==2.33.2
-pyows==0.3.1
-PyQt6==6.7.1
-pyshp==2.3.1
-pystac-client==0.9.0
-python-dateutil==2.9.0.post0
-python-dotenv==1.1.1
-python-multipart==0.0.21
-python-slugify==8.0.4
-pyTMD==2.2.8
-pyviz_comms==3.0.6
-PyYAML==6.0.2
-rasterstats==0.20.0
-referencing==0.36.2
-regex==2025.9.1
-rfc3987==1.3.8
-ruamel.yaml.clib==0.2.12
-s3fs==2025.9.0
-s3transfer==0.13.1
-scikit-image==0.25.2
-scikit-learn==1.7.1
-seaborn==0.13.2
-setuptools-scm==9.2.0
-simplejson==3.20.1
-six==1.17.0
-slicerator==1.1.0
-SQLAlchemy==2.0.0
-starlette==0.50.0
-sympy==1.14.0
-text-unidecode==1.3
-tifffile==2025.5.10
-timescale==0.0.9
-timezonefinder==8.0.0
-torch==2.9.1
-torchvision==0.24.1
-tqdm==4.67.1
-traittypes==0.2.1
-triton==3.5.1
-typing-inspection==0.4.1
-uc-micro-py==1.0.3
-uvicorn==0.38.0
-Werkzeug==3.1.3
-widgetsnbextension==4.0.14
-xgboost==3.1.2
-yarl==1.20.1

run_all_parallel.py

@@ -0,0 +1,82 @@
+import json
+import glob
+import subprocess
+import time
+import os
+
+NOTEBOOKS_TO_RUN = [
+    "01.train_ODC.ipynb",
+    "01.train_ODC_XGBoost.ipynb",
+    "02.predict_ODC.ipynb",
+    "new_train.ipynb"
+]
+
+def limit_time_range(file_path):
+    try:
+        with open(file_path, 'r', encoding='utf-8') as f:
+            nb = json.load(f)
+        
+        changed = False
+        for cell in nb.get('cells', []):
+            if cell.get('cell_type') == 'code':
+                source = cell.get('source', [])
+                if isinstance(source, list):
+                    for i, line in enumerate(source):
+                        # Replace 2023-12-31 with 2023-04-01
+                        if '"2023-12-31"' in line:
+                            source[i] = line.replace('"2023-12-31"', '"2023-04-01"')
+                            changed = True
+                        if "'2023-10-01'" in line:
+                            source[i] = line.replace("'2023-10-01'", "'2022-10-01'")
+                            changed = True
+                        if '"2023-10-01"' in line:
+                            source[i] = line.replace('"2023-10-01"', '"2022-10-01"')
+                            changed = True
+                        # For time_range="2022-09-01/2023-10-01"
+                        if "2022-09-01/2023-10-01" in line:
+                            source[i] = line.replace("2022-09-01/2023-10-01", "2022-09-01/2022-10-01")
+                            changed = True
+                            
+                elif isinstance(source, str):
+                    new_source = source.replace('"2023-12-31"', '"2023-04-01"')
+                    new_source = new_source.replace("'2023-10-01'", "'2022-10-01'")
+                    new_source = new_source.replace('"2023-10-01"', '"2022-10-01"')
+                    new_source = new_source.replace("2022-09-01/2023-10-01", "2022-09-01/2022-10-01")
+                    if new_source != source:
+                        cell['source'] = new_source
+                        changed = True
+        
+        if changed:
+            with open(file_path, 'w', encoding='utf-8') as f:
+                json.dump(nb, f, indent=1)
+            print(f"Limited time_range to 1 month in {file_path}")
+    except Exception as e:
+        print(f"Error on {file_path}: {e}")
+
+# 1. Modify the time ranges
+for nb_file in glob.glob("*.ipynb"):
+    limit_time_range(nb_file)
+
+# 2. Run them in parallel
+print("\nStarting parallel execution of notebooks...")
+processes = []
+for nb_file in NOTEBOOKS_TO_RUN:
+    if os.path.exists(nb_file):
+        print(f"Launching {nb_file}...")
+        cmd = f"source /home/x79/miniconda3/etc/profile.d/conda.sh && conda activate env_01 && jupyter nbconvert --execute --ExecutePreprocessor.timeout=-1 --inplace {nb_file}"
+        p = subprocess.Popen(["bash", "-c", cmd], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+        processes.append((nb_file, p))
+
+# 3. Wait and print output
+for nb_file, p in processes:
+    p.wait()
+    output = p.stdout.read().decode('utf-8')
+    if p.returncode == 0:
+        print(f"[{nb_file}] SUCCESS")
+    else:
+        print(f"[{nb_file}] FAILED (code {p.returncode})")
+        print(f"--- OUTPUT START ({nb_file}) ---")
+        print(output)
+        print(f"--- OUTPUT END ({nb_file}) ---")
+
+print("\nAll tasks finished.")

run_pipeline.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+source /home/x79/miniconda3/etc/profile.d/conda.sh
+conda activate env_01
+set -e
+
+# Configure GDAL for vsicurl stability
+export GDAL_HTTP_MAX_RETRY=5
+export GDAL_HTTP_RETRY_DELAY=2
+export GDAL_HTTP_CONNECTION_TIMEOUT=10
+export GDAL_HTTP_TIMEOUT=30
+export CPL_VSIL_CURL_ALLOWED_EXTENSIONS=.tif,.tiff
+export GDAL_DISABLE_READDIR_ON_OPEN=YES
+
+
+echo "=== [1/4] Running RF Training ==="
+jupyter nbconvert --execute --ExecutePreprocessor.timeout=-1 --inplace 01.train_ODC.ipynb
+
+echo "=== [2/4] Running XGBoost Training ==="
+jupyter nbconvert --execute --ExecutePreprocessor.timeout=-1 --inplace 01.train_ODC_XGBoost.ipynb
+
+echo "=== [3/4] Running Prediction ==="
+jupyter nbconvert --execute --ExecutePreprocessor.timeout=-1 --inplace 02.predict_ODC.ipynb
+
+echo "=== [4/4] Running New Train ==="
+jupyter nbconvert --execute --ExecutePreprocessor.timeout=-1 --inplace new_train.ipynb
+
+echo "=== ALL DONE SUCCESSFULLY ==="

test_VH_VV.ipynb

@@ -1141,12 +1141,12 @@
    "source": [
     "\n",
     "%matplotlib inline\n",
-    "from new_import import *\n",
+    "from new_import_ODC import *\n",
     "\n",
     "\n",
     "# Dask gateway\n",
     "cluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))\n",
-    "dc = datacube.Datacube()\n",
+    "dc = None\n",
     "\n",
     "\n",
     "# Configure s3 access\n",
@@ -1300,7 +1300,7 @@
        "\n",
        ".xr-section-summary-in + label:before {\n",
        "  display: inline-block;\n",
-       "  content: '►';\n",
+       "  content: '\u25ba';\n",
        "  font-size: 11px;\n",
        "  width: 15px;\n",
        "  text-align: center;\n",
@@ -1311,7 +1311,7 @@
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label:before {\n",
-       "  content: '▼';\n",
+       "  content: '\u25bc';\n",
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label > span {\n",
@@ -1880,20 +1880,20 @@
     }
    ],
    "source": [
-    "ds = dc.load(\n",
-    "    product=\"sentinel1_grd_gamma0_20m\",\n",
-    "    x=(105.5, 106.4),\n",
-    "    y=(9.2, 10.0),\n",
-    "    time=(\"2022-09-01\", \"2023-10-01\"),\n",
-    "    measurements=[\"vv\", \"vh\"],\n",
-    "    output_crs=\"EPSG:32648\",\n",
-    "    resolution=(-10,10),\n",
-    "    dask_chunks={\"x\":2048, \"y\":2048},\n",
-    "    skip_broken_datasets=True,\n",
-    "    group_by=\"solar_day\"\n",
-    ")\n",
-    "notebook_utils.heading(notebook_utils.xarray_object_size(ds))\n",
-    "ds"
+    "# ds = dc.load(\n",
+    "#     product=\"sentinel1_grd_gamma0_20m\",\n",
+    "#     x=(105.5, 106.4),\n",
+    "#     y=(9.2, 10.0),\n",
+    "#     time=(\"2022-09-01\", \"2022-10-01\"),\n",
+    "#     measurements=[\"vv\", \"vh\"],\n",
+    "#     output_crs=\"EPSG:32648\",\n",
+    "#     resolution=(-10,10),\n",
+    "#     dask_chunks={\"x\":2048, \"y\":2048},\n",
+    "#     skip_broken_datasets=True,\n",
+    "#     group_by=\"solar_day\"\n",
+    "# )\n",
+    "# notebook_utils.heading(notebook_utils.xarray_object_size(ds))\n",
+    "# ds"
    ]
   },
   {
@@ -1905,11 +1905,11 @@
    },
    "outputs": [],
    "source": [
-    "vh = ds.vh.resample(time='1M').mean().persist()\n",
-    "vh = vh.compute()\n",
-    "vv = ds.vv.resample(time='1M').mean().persist()\n",
-    "vv = vv.compute()\n",
-    "\n"
+    "# vh = ds.vh.resample(time='1M').mean().persist()\n",
+    "# vh = vh.compute()\n",
+    "# vv = ds.vv.resample(time='1M').mean().persist()\n",
+    "# vv = vv.compute()\n",
+    "# \n"
    ]
   },
   {
@@ -2045,7 +2045,7 @@
        "\n",
        ".xr-section-summary-in + label:before {\n",
        "  display: inline-block;\n",
-       "  content: '►';\n",
+       "  content: '\u25ba';\n",
        "  font-size: 11px;\n",
        "  width: 15px;\n",
        "  text-align: center;\n",
@@ -2056,7 +2056,7 @@
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label:before {\n",
-       "  content: '▼';\n",
+       "  content: '\u25bc';\n",
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label > span {\n",
@@ -2365,4 +2365,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 5
-}
+}

test_VH_VV.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[2]:
+
+
+get_ipython().run_line_magic('matplotlib', 'inline')
+from new_import import *
+
+
+# Dask gateway
+cluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))
+dc = datacube.Datacube()
+
+
+# Configure s3 access
+configure_s3_access(aws_unsigned=False, requester_pays=True, client=client)
+
+
+
+
+# In[3]:
+
+
+ds = dc.load(
+    product="sentinel1_grd_gamma0_20m",
+    x=(105.5, 106.4),
+    y=(9.2, 10.0),
+    time=("2022-09-01", "2023-10-01"),
+    measurements=["vv", "vh"],
+    output_crs="EPSG:32648",
+    resolution=(-10,10),
+    dask_chunks={"x":2048, "y":2048},
+    skip_broken_datasets=True,
+    group_by="solar_day"
+)
+notebook_utils.heading(notebook_utils.xarray_object_size(ds))
+ds
+
+
+# In[18]:
+
+
+vh = ds.vh.resample(time='1M').mean().persist()
+vh = vh.compute()
+vv = ds.vv.resample(time='1M').mean().persist()
+vv = vv.compute()
+
+
+
+# In[28]:
+
+
+vv.min()
+
+
+# In[33]:
+
+
+import matplotlib.pyplot as plt
+
+# Plot the data
+plt.imshow(vh.isel(time=0), cmap='viridis', vmin=0, vmax=1)
+plt.colorbar()  # Add colorbar for reference
+plt.show()
+
+
+# In[ ]:
+
+
+
+

test_planetary.py

@@ -0,0 +1,15 @@
+import sys
+# Thêm đường dẫn hiện tại vào PYTHONPATH để import được new_import_ODC nếu cần
+sys.path.append('.')
+import warnings
+warnings.filterwarnings('ignore')
+from new_import_ODC import load_sen1
+
+print("Testing load_sen1 with a short time range to speed up Dask compute...")
+bbox = [105.5, 9.2, 106.4, 10.0]
+time_range = "2023-01-01/2023-01-31" # Short time range for fast testing
+vh, vv = load_sen1(bbox, time_range)
+print("VH shape:", vh.shape)
+print("VV shape:", vv.shape)
+print("VH CRS:", vh.rio.crs)
+print("Success!")

test_planetary2.py

@@ -0,0 +1,49 @@
+import warnings
+warnings.filterwarnings('ignore')
+
+def load_sen1(bbox, time_range):
+    import pystac_client
+    import planetary_computer
+    import odc.stac
+    
+    catalog = pystac_client.Client.open(
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
+        modifier=planetary_computer.sign_inplace,
+    )
+
+    search = catalog.search(
+        collections=["sentinel-1-rtc"],
+        bbox=bbox,
+        datetime=time_range,
+    )
+    items = list(search.items())
+    print("Found items:", len(items))
+
+    ds_s1 = odc.stac.load(
+        items,
+        bands=["vv", "vh"],
+        bbox=bbox,
+        crs="EPSG:32648",
+        resolution=10,
+        chunks={"x": 2048, "y": 2048, "time": 1}
+    )
+
+    ds_median = ds_s1.median(dim="time").compute()
+    vv = ds_median["vv"]
+    vh = ds_median["vh"]
+
+    vv = vv.expand_dims(dim="band")
+    vh = vh.expand_dims(dim="band")
+
+    vv = vv.rio.write_crs("EPSG:32648")
+    vh = vh.rio.write_crs("EPSG:32648")
+
+    return vh, vv
+
+print("Testing load_sen1...")
+bbox = [105.5, 9.2, 106.4, 10.0]
+time_range = "2022-09-01/2023-10-01"
+vh, vv = load_sen1(bbox, time_range)
+print("VH shape:", vh.shape)
+print("VV shape:", vv.shape)
+print("Success!")

test_shapefile_api.py

@@ -1,48 +0,0 @@
-#!/usr/bin/env python3
-"""
-Test script to verify shapefile overlay API returns correct bbox data
-"""
-
-import requests
-import json
-
-def test_shapefile_api():
-    """Test /api/overlay/shapefiles endpoint"""
-    print("Testing /api/overlay/shapefiles endpoint...")
-    
-    try:
-        response = requests.get('http://localhost:8000/api/overlay/shapefiles')
-        
-        if response.status_code == 200:
-            data = response.json()
-            print(f"\n✅ API Response successful")
-            print(f"Total shapefiles: {data.get('count', 0)}")
-            
-            if data.get('shapefiles'):
-                print("\n📋 Shapefile details:")
-                for idx, shp in enumerate(data['shapefiles'], 1):
-                    print(f"\n{idx}. {shp.get('filename')}")
-                    print(f"   Path: {shp.get('path')}")
-                    print(f"   CRS: {shp.get('crs')}")
-                    print(f"   Features: {shp.get('feature_count')}")
-                    print(f"   Bbox: {shp.get('bbox')}")
-                    
-                    # Verify bbox format
-                    bbox = shp.get('bbox')
-                    if bbox and len(bbox) == 4:
-                        print(f"   ✅ Bbox format valid: [minLon, minLat, maxLon, maxLat]")
-                    else:
-                        print(f"   ❌ Bbox format invalid or missing!")
-            else:
-                print("\n⚠️ No shapefiles found")
-        else:
-            print(f"\n❌ API returned status code: {response.status_code}")
-            print(f"Response: {response.text}")
-            
-    except requests.exceptions.ConnectionError:
-        print("\n❌ Cannot connect to API server. Is it running on localhost:8000?")
-    except Exception as e:
-        print(f"\n❌ Error: {e}")
-
-if __name__ == "__main__":
-    test_shapefile_api()

test_shapefile_overlay.py

@@ -1,79 +0,0 @@
-#!/usr/bin/env python
-"""
-Test script to verify shapefile overlay functionality
-"""
-import geopandas as gpd
-import numpy as np
-from pathlib import Path
-
-# Test shapefile path
-shapefile_path = "ChauThanh/HienTrang/ChauThanh_kiemke.shp"
-
-print("=" * 70)
-print("TESTING SHAPEFILE OVERLAY")
-print("=" * 70)
-
-# Check if file exists
-shp = Path(shapefile_path)
-print(f"\n1. Checking file existence:")
-print(f"   Path: {shp}")
-print(f"   Exists: {shp.exists()}")
-print(f"   Absolute: {shp.absolute()}")
-
-if shp.exists():
-    # Read shapefile
-    print(f"\n2. Reading shapefile...")
-    gdf = gpd.read_file(str(shp))
-    print(f"   Features: {len(gdf)}")
-    print(f"   CRS: {gdf.crs}")
-    print(f"   Bounds: {gdf.total_bounds}")
-    print(f"   Columns: {list(gdf.columns)}")
-    
-    # Check geometries
-    print(f"\n3. Checking geometries...")
-    valid_count = sum(1 for geom in gdf.geometry if geom is not None and geom.is_valid)
-    print(f"   Valid geometries: {valid_count} / {len(gdf)}")
-    
-    # Sample geometry bounds
-    if len(gdf) > 0:
-        sample_geom = gdf.geometry.iloc[0]
-        print(f"   Sample geometry type: {sample_geom.geom_type}")
-        print(f"   Sample geometry bounds: {sample_geom.bounds}")
-    
-    # Test reprojection to EPSG:4326
-    print(f"\n4. Testing reprojection to EPSG:4326...")
-    try:
-        gdf_4326 = gdf.to_crs("EPSG:4326")
-        print(f"   Success!")
-        print(f"   New bounds: {gdf_4326.total_bounds}")
-    except Exception as e:
-        print(f"   ERROR: {e}")
-    
-    # Test boundary extraction
-    print(f"\n5. Testing boundary extraction...")
-    boundaries = []
-    for geom in gdf.geometry:
-        if geom is not None and geom.is_valid:
-            boundary = geom.boundary
-            if boundary is not None:
-                boundaries.append(boundary)
-    print(f"   Extracted boundaries: {len(boundaries)}")
-    
-    # Test buffering
-    print(f"\n6. Testing buffer...")
-    buffer_size = 0.001  # degrees or meters depending on CRS
-    buffered = []
-    for boundary in boundaries[:10]:  # Test first 10
-        try:
-            buf = boundary.buffer(buffer_size)
-            buffered.append(buf)
-        except Exception as e:
-            print(f"   Buffer error: {e}")
-    print(f"   Successfully buffered: {len(buffered)} / 10")
-    
-else:
-    print("   ERROR: Shapefile not found!")
-
-print("\n" + "=" * 70)
-print("TEST COMPLETE")
-print("=" * 70)

test_shapefile_selection.html

@@ -1,177 +0,0 @@
-<!DOCTYPE html>
-<html lang="vi">
-<head>
-    <meta charset="UTF-8">
-    <title>Test Shapefile Selection</title>
-    <link rel="stylesheet" href="https://unpkg.com/leaflet@1.9.4/dist/leaflet.css" />
-    <style>
-        body { font-family: Arial, sans-serif; padding: 20px; }
-        #map { height: 400px; border: 2px solid #ccc; margin: 20px 0; }
-        .info-box { background: #f0f0f0; padding: 15px; margin: 10px 0; border-radius: 5px; }
-    </style>
-</head>
-<body>
-    <h1>🧪 Test Shapefile Auto-Select Bbox</h1>
-    
-    <div class="info-box">
-        <h3>Chọn Shapefile:</h3>
-        <select id="shapefileOverlay" onchange="onShapefileSelected(event)" style="width: 100%; padding: 10px; font-size: 14px;">
-            <option value="">-- Chọn shapefile --</option>
-        </select>
-    </div>
-    
-    <div id="map"></div>
-    
-    <div class="info-box">
-        <h3>Current Bbox:</h3>
-        <pre id="bboxInfo">Chưa chọn shapefile</pre>
-    </div>
-    
-    <div class="info-box">
-        <h3>Console Logs:</h3>
-        <pre id="console" style="max-height: 200px; overflow-y: auto; background: #000; color: #0f0; padding: 10px;"></pre>
-    </div>
-    
-    <script src="https://unpkg.com/leaflet@1.9.4/dist/leaflet.js"></script>
-    <script>
-        // Global variables
-        let map, drawnItems, selectedBbox = null;
-        
-        // Custom console.log to display in page
-        const originalLog = console.log;
-        console.log = function(...args) {
-            originalLog.apply(console, args);
-            const consoleEl = document.getElementById('console');
-            consoleEl.textContent += args.join(' ') + '\n';
-            consoleEl.scrollTop = consoleEl.scrollHeight;
-        };
-        
-        // Initialize map
-        function initMap() {
-            map = L.map('map').setView([10.0, 105.8], 10);
-            L.tileLayer('https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png', {
-                attribution: '© OpenStreetMap contributors'
-            }).addTo(map);
-            
-            drawnItems = new L.FeatureGroup();
-            map.addLayer(drawnItems);
-            
-            console.log('✅ Map initialized');
-        }
-        
-        // Load shapefiles from API
-        async function loadShapefiles() {
-            try {
-                console.log('📡 Fetching shapefiles from API...');
-                const response = await fetch('http://localhost:8000/api/overlay/shapefiles');
-                const data = await response.json();
-                
-                const select = document.getElementById('shapefileOverlay');
-                select.innerHTML = '<option value="">-- Chọn shapefile --</option>';
-                
-                if (data.shapefiles && data.shapefiles.length > 0) {
-                    data.shapefiles.forEach(shp => {
-                        const option = document.createElement('option');
-                        option.value = shp.path;
-                        
-                        let label = `${shp.filename} - ${shp.feature_count} features`;
-                        if (shp.crs) {
-                            const crsCode = shp.crs.split(':').pop();
-                            label += ` | CRS: ${crsCode}`;
-                        }
-                        if (shp.bbox && shp.bbox.length === 4) {
-                            const [minLon, minLat, maxLon, maxLat] = shp.bbox;
-                            label += ` | [${minLon.toFixed(2)}, ${minLat.toFixed(2)}, ${maxLon.toFixed(2)}, ${maxLat.toFixed(2)}]`;
-                        }
-                        
-                        option.textContent = label;
-                        option.dataset.crs = shp.crs || '';
-                        option.dataset.bbox = JSON.stringify(shp.bbox || []);
-                        option.dataset.featureCount = shp.feature_count;
-                        
-                        select.appendChild(option);
-                    });
-                    
-                    console.log(`✅ Loaded ${data.shapefiles.length} shapefiles`);
-                } else {
-                    console.log('⚠️ No shapefiles found');
-                }
-            } catch (error) {
-                console.error('❌ Error loading shapefiles:', error);
-            }
-        }
-        
-        // Handle shapefile selection
-        function onShapefileSelected(event) {
-            console.log('🔔 Shapefile selection changed');
-            
-            const selectedOption = event.target.selectedOptions[0];
-            
-            if (!selectedOption || !selectedOption.value) {
-                console.log('ℹ️ No shapefile selected');
-                document.getElementById('bboxInfo').textContent = 'Chưa chọn shapefile';
-                return;
-            }
-            
-            const bboxData = selectedOption.dataset.bbox;
-            console.log('📦 Bbox data from option:', bboxData);
-            
-            if (!bboxData || bboxData === '[]') {
-                console.log('⚠️ No bbox data in selected option');
-                return;
-            }
-            
-            try {
-                const bbox = JSON.parse(bboxData);
-                console.log('📊 Parsed bbox:', bbox);
-                
-                if (bbox.length !== 4) {
-                    console.log('❌ Invalid bbox length:', bbox.length);
-                    return;
-                }
-                
-                const [minLon, minLat, maxLon, maxLat] = bbox;
-                
-                // Validate bbox
-                if (minLon < -180 || maxLon > 180 || minLat < -90 || maxLat > 90) {
-                    console.log('❌ Bbox out of valid range');
-                    return;
-                }
-                
-                console.log('✅ Valid bbox:', {minLon, minLat, maxLon, maxLat});
-                
-                // Update map
-                const bounds = [[minLat, minLon], [maxLat, maxLon]];
-                const rectangle = L.rectangle(bounds, {
-                    color: '#667eea',
-                    weight: 3,
-                    fillOpacity: 0.2
-                });
-                
-                drawnItems.clearLayers();
-                drawnItems.addLayer(rectangle);
-                map.fitBounds(bounds, { padding: [50, 50] });
-                
-                selectedBbox = {min_lon: minLon, min_lat: minLat, max_lon: maxLon, max_lat: maxLat};
-                
-                console.log('🗺️ Map updated with new bbox');
-                
-                // Update bbox info display
-                document.getElementById('bboxInfo').textContent = JSON.stringify(selectedBbox, null, 2);
-                
-                alert(`✅ Bbox updated!\n\nmin_lon: ${minLon.toFixed(4)}\nmin_lat: ${minLat.toFixed(4)}\nmax_lon: ${maxLon.toFixed(4)}\nmax_lat: ${maxLat.toFixed(4)}`);
-                
-            } catch (e) {
-                console.error('❌ Error:', e);
-            }
-        }
-        
-        // Initialize on load
-        window.onload = function() {
-            console.log('🚀 Page loaded, initializing...');
-            initMap();
-            loadShapefiles();
-        };
-    </script>
-</body>
-</html>