Migrate all ODC models and prediction pipeline to Microsoft Planetary Computer

01.train_ODC_XGBoost.py

@@ -0,0 +1,171 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+get_ipython().run_cell_magic('time', '', '%matplotlib inline\n\nimport importlib\nimport new_import_ODC  \n\nimportlib.reload(new_import_ODC)\n\nfrom new_import_ODC import *\n')
+
+
+# In[2]:
+
+
+get_ipython().run_cell_magic('time', '', '# Cấu hình Daskgateway\ncluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1, 10))\n# Khai báo 1 Datacube là dc\ndc = None\n\n# Cấu hình truy cập dịch vụ S3\nconfigure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n\nclient\n')
+
+
+# In[3]:
+
+
+## cấu hình thời gian lấy ảnh và tọa độ
+date_range = ("2022-09-01", "2022-10-01")
+longtitude_range = (105.86, 105.94)
+latitude_range = (9.65, 9.69)
+
+coordinates = (longtitude_range, latitude_range)
+
+
+# In[4]:
+
+
+## truy vấn ảnh vệ tinh sen2
+data = load_data(None, date_range, longtitude_range, latitude_range)
+notebook_utils.heading(notebook_utils.xarray_object_size(data))
+display(data)
+
+
+# In[5]:
+
+
+get_ipython().run_cell_magic('time', '', '# Tiến hành loại bỏ các vị trí bị mây ảnh hưởng\nresult = mask_clean(data)\n# progress(result)\n')
+
+
+# In[6]:
+
+
+# Tiến hành tính toán NDVI
+ds1 = calculate_indices(result, index="NDVI", satellite_mission="s2")
+ndvi = ds1["NDVI"]
+display(ndvi)
+
+
+# In[7]:
+
+
+## Hiển thị ảnh NDVI chưa điền các giá trị mây (chưa fill nan)
+plt.imshow(ndvi.isel(time=0))
+
+
+# In[8]:
+
+
+# Thiết lập giá trị trung bình mùa vụ để xử lý các điểm ảnh bị mây dựa vào sự thay đổi theo mùa
+time_split = [
+    slice("2022-09-01", "2023-01-01"),
+    slice("2023-01-01", "2023-05-01"),
+    slice("2023-05-01", "2023-07-01"),
+    slice("2023-07-01", "2022-10-01"),
+]
+
+# Điền mây ở các vị trí mang giá trị nan (fill nan)
+fill_nan_ndvi = fill_nan(ndvi, time_split)
+
+# In kết quả ảnh NDVI đã điền mây (đã fill nan)
+plt.imshow(fill_nan_ndvi.isel(time=0))
+
+
+# In[9]:
+
+
+get_ipython().run_cell_magic('time', '', '## tính ndvi theo tháng\naverage_ndvi = fill_nan_ndvi.resample(time="1M").mean().persist()\n# progress(average_ndvi)\n\n# compute average_ndvi\naverage_ndvi = average_ndvi.compute()\n')
+
+
+# In[10]:
+
+
+#Load dữ liệu ảnh Sentinel 1
+dsvh, dsvv = load_data_sen1(None, date_range, coordinates)
+average_vv = calculate_average(dsvv, time_pattern='1M')
+average_vh = calculate_average(dsvh, time_pattern='1M')
+
+
+# In[11]:
+
+
+## cấu hình bộ dữ liệu điểm huấn luyện mô hình (train file)
+train_path = "train/ST_training_data_updated_1130points_new.shp"  # đường dẫn shp file train
+
+## load dữ liệu điểm huấn luyện mô hình (train file)
+train = load_train_data(train_path)
+train.head()
+
+# cấu hình nhãn dữ liệu 
+label_mapping = {
+    "Lua tom": "0",
+    "Lua": "1",
+    "CHN": "2",
+    "CLN": "3",
+    "TS": "4",
+    "Song": "5",
+    "Dat xay dung": "6",
+    "Rung": "7",
+}
+
+# xây dựng tập dữ liệu (dataset) chứa dữ liệu VH, VV, NDVI
+datasets = get_data_sen1_and_sen2(train, average_ndvi, average_vh, average_vv)
+
+# chia tập dữ liệu thành các phần theo tỉ lệ 80(80-20)-20 tương ứng với tập train, validate, test
+X_train, X_val, X_test, y_train, y_val, y_test = split_train_data(
+    train, label_mapping, datasets
+)
+
+
+# In[ ]:
+
+
+get_ipython().run_cell_magic('time', '', '# Import XGBoost\nimport xgboost as xgb\nfrom sklearn.metrics import accuracy_score\nimport numpy as np\n\n# Convert to numpy arrays\nX_train_np = np.asarray(X_train, dtype=np.float32)\nX_val_np = np.asarray(X_val, dtype=np.float32)\ny_train_np = np.asarray(y_train, dtype=np.int32)\ny_val_np = np.asarray(y_val, dtype=np.int32)\n\nprint("🚀 Training XGBoost model...")\nprint(f"   Train samples: {len(X_train_np)}")\nprint(f"   Val samples: {len(X_val_np)}")\nprint(f"   Features: {X_train_np.shape[1]}")\nprint(f"   Classes: 8\\n")\n\n# XGBoost parameters\nparams = {\n    \'objective\': \'multi:softmax\',  # Multi-class classification\n    \'num_class\': 8,                 # 8 land use classes\n    \'max_depth\': 6,                 # Maximum tree depth\n    \'learning_rate\': 0.1,           # Learning rate\n    \'n_estimators\': 200,            # Number of trees\n    \'subsample\': 0.8,               # Subsample ratio\n    \'colsample_bytree\': 0.8,        # Feature sampling ratio\n    \'random_state\': 42,\n    \'n_jobs\': -1,                   # Use all CPU cores\n    \'eval_metric\': \'mlogloss\'       # Multi-class log loss\n}\n\n# Train XGBoost model\nmodel = xgb.XGBClassifier(**params)\n\nmodel.fit(\n    X_train_np, y_train_np,\n    eval_set=[(X_train_np, y_train_np), (X_val_np, y_val_np)],\n    verbose=True\n)\n\n# Validation accuracy\ny_val_pred = model.predict(X_val_np)\nval_accuracy = accuracy_score(y_val_np, y_val_pred)\nprint(f"\\n✅ Training completed!")\nprint(f"   Validation Accuracy: {val_accuracy:.4f} ({val_accuracy*100:.2f}%)")\n')
+
+
+# In[ ]:
+
+
+get_ipython().run_cell_magic('time', '', '# Evaluate on test set\nX_test_np = np.asarray(X_test, dtype=np.float32)\ny_test_np = np.asarray(y_test, dtype=np.int32)\n\nprint("📊 Evaluating XGBoost model on test set...\\n")\n\n# Predictions\ny_pred_test = model.predict(X_test_np)\n\n# Metrics\nfrom sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, confusion_matrix\n\ntest_accuracy = accuracy_score(y_test_np, y_pred_test)\nprecision = precision_score(y_test_np, y_pred_test, average=\'weighted\', zero_division=0)\nrecall = recall_score(y_test_np, y_pred_test, average=\'weighted\', zero_division=0)\nf1 = f1_score(y_test_np, y_pred_test, average=\'weighted\', zero_division=0)\n\nprint(f"📈 Test Results:")\nprint(f"   Accuracy:  {test_accuracy:.4f} ({test_accuracy*100:.2f}%)")\nprint(f"   Precision: {precision:.4f}")\nprint(f"   Recall:    {recall:.4f}")\nprint(f"   F1-Score:  {f1:.4f}\\n")\n\n# Confusion Matrix\nfrom sklearn.metrics import ConfusionMatrixDisplay\nimport matplotlib.pyplot as plt\n\n# Create figure first\nfig, ax = plt.subplots(figsize=(10, 8))\n\nclass_names = list(label_mapping.keys())\ncm = confusion_matrix(y_test_np, y_pred_test)\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=class_names)\ndisp.plot(cmap=\'Blues\', ax=ax)\nplt.xticks(rotation=45, ha=\'right\')\nplt.title(\'XGBoost Confusion Matrix\')\nplt.tight_layout()\nplt.show()\n')
+
+
+# In[ ]:
+
+
+# Lưu mô hình huấn luyện
+import json
+import joblib
+
+# Save XGBoost model
+model_path = "model_xgboost.joblib"
+joblib.dump(model, model_path)
+print(f"✅ Model saved to {model_path}")
+
+# Save model info
+info = {
+    "model_type": "XGBoost",
+    "num_classes": 8,
+    "classes": list(label_mapping.keys()),
+    "num_features": X_train_np.shape[1],
+    "params": params,
+    "accuracy": float(test_accuracy),
+    "precision": float(precision),
+    "recall": float(recall),
+    "f1_score": float(f1),
+}
+
+with open("model_xgboost_info.json", "w") as f:
+    json.dump(info, f, indent=2)
+
+print(f"✅ Model info saved to model_xgboost_info.json")
+
+
+# In[15]:
+
+
+# đóng client, cluster
+# client.close()
+# cluster.close()
+

01.train_ODC_local_with_Mic_supplyer.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[6]:
+
+
+get_ipython().run_cell_magic('time', '', '%matplotlib inline\n\n# Import Microsoft Planetary Computer libraries\nimport planetary_computer\nfrom pystac_client import Client\nfrom odc.stac import load as stac_load\n\n# Standard imports\nimport xarray as xr\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score, classification_report, confusion_matrix, ConfusionMatrixDisplay\nimport geopandas as gpd\n\n# XGBoost for GPU training\nimport xgboost as xgb\n\nfrom xgboost import XGBClassifier\n\nprint(f"   XGBoost version: {xgb.__version__}")\n\nprint("✅ All modules loaded successfully")\n')
+
+
+# In[7]:
+
+
+get_ipython().run_cell_magic('time', '', '# Kết nối tới Microsoft Planetary Computer STAC\nfrom pystac_client import Client\n\n# KHÔNG dùng modifier ở catalog level để tránh items bị convert thành dict\ncatalog = Client.open(\n    "https://planetarycomputer.microsoft.com/api/stac/v1"\n)\nprint("✅ Connected to Microsoft Planetary Computer")\n\nprint("\\n" + "="*70)\n')
+
+
+# In[8]:
+
+
+get_ipython().run_cell_magic('time', '', '# 🌍 Định nghĩa khu vực và thời gian\nprint("="*70)\nprint("CONFIGURATION")\nprint("="*70)\n\n# Khu vực quan tâm (Vietnam - Mekong Delta) - GIẢM DIỆN TÍCH ~40%\nbbox = [105.6, 9.3, 106.2, 9.8]  # [min_lon, min_lat, max_lon, max_lat]\n\n# GIẢM THỜI GIAN xuống 3 tháng để giảm kích thước dữ liệu cho PC\ntime_range = "2023-03-01/2023-05-31"  # 3 tháng (mùa khô)\n\nprint(f"\\n📍 Area of Interest:")\nprint(f"   Longitude: {bbox[0]} to {bbox[2]}")\nprint(f"   Latitude: {bbox[1]} to {bbox[3]}")\nprint(f"\\n📅 Time Range: {time_range}")\nprint(f"   ⚠️  Optimized for personal computer (3 months, reduced area)")\nprint(f"\\n🗺️  CRS: EPSG:32648")\nprint(f"   Resolution: 20m (reduced from 10m for smaller data size)")\n\nprint("="*70)\n')
+
+
+# In[9]:
+
+
+get_ipython().run_cell_magic('time', '', '# 📡 LOAD SENTINEL-2 FROM MICROSOFT PLANETARY COMPUTER\nprint("="*70)\nprint("LOADING SENTINEL-2 L2A")\nprint("="*70)\n\nprint("\\n🔍 Searching for Sentinel-2 scenes...")\nquery_s2 = catalog.search(\n    collections=["sentinel-2-l2a"],\n    bbox=bbox,\n    datetime=time_range,\n    query={"eo:cloud_cover": {"lt": 30}}  # Cloud cover < 30% (giảm từ 50%)\n)\n\nitems_s2 = list(query_s2.item_collection())\nprint(f"✅ Found {len(items_s2)} Sentinel-2 scenes")\n\n# GIỚI HẠN SỐ LƯỢNG SCENES cho PC cá nhân\nmax_scenes = 12  # Giảm xuống 12 scenes để tối ưu cho PC\nif len(items_s2) > max_scenes:\n    print(f"⚠️  Limiting to {max_scenes} scenes for personal computer")\n    # Chọn scenes đều đặn trong khoảng thời gian\n    step = len(items_s2) // max_scenes\n    items_s2 = items_s2[::step][:max_scenes]\n    print(f"   Selected {len(items_s2)} scenes evenly distributed")\n\nif len(items_s2) > 0:\n    # Show first few scenes\n    print(f"\\n📋 Sample scenes:")\n    for i, item in enumerate(items_s2[:5]):\n        date = item.datetime.strftime("%Y-%m-%d")\n        cloud = item.properties.get("eo:cloud_cover", "N/A")\n        print(f"   [{i+1}] {date} - Cloud: {cloud}%")\n    \n    # Re-sign items to ensure fresh URLs (keep as pystac objects)\n    print(f"\\n🔑 Signing STAC items...")\n    items_s2 = [planetary_computer.sign(item) for item in items_s2]\n    \n    # Load Sentinel-2 data (without Dask chunks)\n    print(f"\\n⏳ Loading Sentinel-2 data...")\n    ds_s2 = stac_load(\n        items_s2,\n        bands=["B04", "B08", "SCL"],  # Red (B04), NIR (B08), Scene Classification (SCL)\n        crs="EPSG:32648",\n        resolution=20,  # 20m resolution (4x smaller data than 10m)\n        bbox=bbox,\n        patch_url=planetary_computer.sign,  # Re-sign URLs during loading\n        fail_on_error=False,  # Skip problematic tiles instead of crashing\n    )\n    \n    # Rename bands to simpler names\n    ds_s2 = ds_s2.rename({"B04": "red", "B08": "nir", "SCL": "scl"})\n    \n    print(f"\\n✅ Sentinel-2 loaded!")\n    print(f"   Shape: {dict(ds_s2.dims)}")\n    print(f"   Variables: {list(ds_s2.data_vars)}")\n    display(ds_s2)\nelse:\n    print(f"❌ No Sentinel-2 scenes found")\n\n    ds_s2 = Noneprint("="*70)\n')
+
+
+# In[10]:
+
+
+get_ipython().run_cell_magic('time', '', '# 📡 LOAD SENTINEL-1 FROM MICROSOFT PLANETARY COMPUTER\nprint("="*70)\nprint("LOADING SENTINEL-1 RTC")\nprint("="*70)\n\nprint("\\n🔍 Searching for Sentinel-1 scenes...")\nquery_s1 = catalog.search(\n    collections=["sentinel-1-rtc"],\n    bbox=bbox,\n    datetime=time_range,\n)\n\nitems_s1 = list(query_s1.item_collection())\nprint(f"✅ Found {len(items_s1)} Sentinel-1 scenes")\n\n# GIỚI HẠN SỐ LƯỢNG SCENES cho PC cá nhân\nmax_scenes = 12  # Giảm xuống 12 scenes để tối ưu cho PC\nif len(items_s1) > max_scenes:\n    print(f"⚠️  Limiting to {max_scenes} scenes for personal computer")\n    # Chọn scenes đều đặn trong khoảng thời gian\n    step = len(items_s1) // max_scenes\n    items_s1 = items_s1[::step][:max_scenes]\n    print(f"   Selected {len(items_s1)} scenes evenly distributed")\n\nif len(items_s1) > 0:\n    # Show first few scenes\n    print(f"\\n📋 Sample scenes:")\n    for i, item in enumerate(items_s1[:5]):\n        date = item.datetime.strftime("%Y-%m-%d")\n        orbit = item.properties.get("sat:orbit_state", "N/A")\n        print(f"   [{i+1}] {date} - Orbit: {orbit}")\n    \n    # Re-sign items to ensure fresh URLs (keep as pystac objects)\n    print(f"\\n🔑 Signing STAC items...")\n    items_s1 = [planetary_computer.sign(item) for item in items_s1]\n    \n    # Load Sentinel-1 data (without Dask chunks)\n    print(f"\\n⏳ Loading Sentinel-1 data...")\n    ds_s1 = stac_load(\n        items_s1,\n        bands=["vv", "vh"],  # VV and VH polarizations\n        crs="EPSG:32648",\n        resolution=20,  # 20m resolution (4x smaller data than 10m)\n        bbox=bbox,\n        patch_url=planetary_computer.sign,  # Re-sign URLs during loading\n        fail_on_error=False,  # Skip problematic tiles instead of crashing\n    )\n    \n    # Convert to dB (Microsoft S1 is in linear power)\n    print(f"\\n🔄 Converting to dB...")\n    ds_s1[\'vv_db\'] = 10 * np.log10(ds_s1[\'vv\'].where(ds_s1[\'vv\'] > 0))\n    ds_s1[\'vh_db\'] = 10 * np.log10(ds_s1[\'vh\'].where(ds_s1[\'vh\'] > 0))\n    \n    print(f"\\n✅ Sentinel-1 loaded!")\n    print(f"   Shape: {dict(ds_s1.dims)}")\n    print(f"   Variables: {list(ds_s1.data_vars)}")\n    display(ds_s1)\nelse:\n    print(f"❌ No Sentinel-1 scenes found")\n\n    ds_s1 = Noneprint("="*70)\n')
+
+
+# In[11]:
+
+
+get_ipython().run_cell_magic('time', '', '# 🌿 CALCULATE NDVI AND PROCESS DATA\nprint("="*70)\nprint("DATA PROCESSING")\nprint("="*70)\n\nif ds_s2 is not None:\n    print("\\n[1] Calculating NDVI...")\n    # NDVI = (NIR - Red) / (NIR + Red)\n    ndvi = (ds_s2[\'nir\'] - ds_s2[\'red\']) / (ds_s2[\'nir\'] + ds_s2[\'red\'] + 1e-8)\n    \n    print(f"✅ NDVI calculated")\n    print(f"   Shape: {ndvi.shape}")\n    print(f"   Time steps: {len(ndvi.time)}")\n    \n    # Cloud masking using SCL band\n    print(f"\\n[2] Applying cloud mask...")\n    # SCL values: 1=defective, 3=cloud shadow, 8=cloud medium, 9=cloud high, 10=cirrus\n    cloud_mask = ds_s2[\'scl\'].isin([1, 3, 8, 9, 10])\n    ndvi_masked = ndvi.where(~cloud_mask)\n    \n    print(f"✅ Cloud mask applied")\n    \n    # Temporal aggregation (mean over time)\n    print(f"\\n[3] Computing mean NDVI across time...")\n    ndvi_mean = ndvi_masked.mean(dim=\'time\')\n    \n    # Data already in memory, no need to compute() again\n    print(f"✅ Mean NDVI computed")\n    print(f"   Shape: {ndvi_mean.shape}")\n    \nelse:\n    print("❌ No Sentinel-2 data to process")\n    ndvi_mean = None\n\nprint("="*70)\n')
+
+
+# In[ ]:
+
+
+get_ipython().run_cell_magic('time', '', '# 🎯 EXTRACT TRAINING DATA FEATURES\nprint("="*70)\nprint("FEATURE EXTRACTION")\nprint("="*70)\n\n# Check if required data is available\nif \'ndvi_mean\' not in globals() or \'ds_s1\' not in globals():\n    print("❌ Error: Please run Cell 6 (DATA PROCESSING) first!")\n    print("   Required variables: ndvi_mean, ds_s1")\n    raise RuntimeError("Missing required data. Run cells in order: Cell 4 → Cell 5 → Cell 6 → Cell 7")\n\n# Load training shapefile\nimport geopandas as gpd\n\ntrain_path = \'train/ST_training data_updated_1130points_new.shp\'\nprint(f"\\n[1] Loading training data from: {train_path}")\ntrain_gdf = gpd.read_file(train_path)\n\n# Ensure CRS matches\nif train_gdf.crs != \'EPSG:32648\':\n    print(f"   Reprojecting from {train_gdf.crs} to EPSG:32648...")\n    train_gdf = train_gdf.to_crs(\'EPSG:32648\')\n\nprint(f"✅ Loaded {len(train_gdf)} training points")\nprint(f"   Available columns: {list(train_gdf.columns)}")\n\n# Auto-detect label column (look for common names)\nlabel_column = None\nfor col in [\'HT_code\', \'Ma_LU\', \'LU2022\', \'class\', \'Class\', \'CLASS\', \'label\', \'Label\', \'LABEL\', \'LU_CODE\', \'LU_code\']:\n    if col in train_gdf.columns:\n        label_column = col\n        break\n\nif label_column is None:\n    print(f"❌ Cannot find label column. Available columns: {list(train_gdf.columns)}")\n    print(f"   Please check your shapefile and update the code.")\nelse:\n    print(f"   Using label column: \'{label_column}\'")\n    print(f"   Classes: {sorted(train_gdf[label_column].unique())}")\n    \n    # Extract features at each training point\n    print(f"\\n[2] Extracting features at training points...")\n    \n    features = []\n    labels = []\n    skipped = 0\n    \n    for idx, row in train_gdf.iterrows():\n        point = row.geometrychro\n        x_coord = point.x\n        y_coord = point.y\n        label = row[label_column]\n        \n        # Extract NDVI at this location\n        if ndvi_mean is not None and ds_s1 is not None:\n            try:\n                ndvi_val = ndvi_mean.sel(x=x_coord, y=y_coord, method=\'nearest\').values\n                \n                # Extract Sentinel-1 VH/VV at this location (mean across time)\n                # Data already in memory, no need to compute()\n                vh_val = ds_s1[\'vh_db\'].sel(x=x_coord, y=y_coord, method=\'nearest\').mean(dim=\'time\').values\n                vv_val = ds_s1[\'vv_db\'].sel(x=x_coord, y=y_coord, method=\'nearest\').mean(dim=\'time\').values\n                \n                # Create feature vector: [NDVI, VH_dB, VV_dB]\n                feature_vec = [ndvi_val, vh_val, vv_val]\n                \n                # Only add if all features are valid (not NaN)\n                if not np.isnan(feature_vec).any():\n                    features.append(feature_vec)\n                    labels.append(label)\n                else:\n                    skipped += 1\n            except Exception as e:\n                # Skip points outside the data extent\n                skipped += 1\n                continue\n    \n    features = np.array(features)\n    labels = np.array(labels)\n    \n    print(f"✅ Extracted features for {len(features)} valid points")\n    print(f"   Skipped {skipped} points (outside extent or NaN values)")\n    print(f"   Feature shape: {features.shape}")\n    print(f"   Feature names: [\'NDVI_mean\', \'VH_dB_mean\', \'VV_dB_mean\']")\n    print(f"\\n   Class distribution:")\n    unique, counts = np.unique(labels, return_counts=True)\n    for cls, cnt in zip(unique, counts):\n        print(f"      Class {cls}: {cnt} samples ({cnt/len(labels)*100:.1f}%)")\n\nprint("="*70)\n')
+
+
+# In[21]:
+
+
+get_ipython().run_cell_magic('time', '', '# 🤖 TRAIN XGBOOST MODEL ON GPU (RTX 4060)\nprint("="*70)\nprint("MODEL TRAINING - GPU ACCELERATED")\nprint("="*70)\n\nfrom xgboost import XGBClassifier\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import classification_report, confusion_matrix, ConfusionMatrixDisplay\nimport matplotlib.pyplot as plt\n\n# Encode labels to ensure they are 0, 1, 2, ... n-1\nprint("\\n[1] Encoding labels...")\nlabel_encoder = LabelEncoder()\nlabels_encoded = label_encoder.fit_transform(labels)\nprint(f"✅ Original classes: {label_encoder.classes_}")\nprint(f"   Encoded as: {np.unique(labels_encoded)}")\n\n# Split data\nprint("\\n[2] Splitting data (80% train, 20% test)...")\nX_train, X_test, y_train, y_test = train_test_split(\n    features, labels_encoded, test_size=0.2, random_state=42, stratify=labels_encoded\n)\nprint(f"✅ Training samples: {len(X_train)}")\nprint(f"   Testing samples: {len(X_test)}")\n\n# Train XGBoost on GPU\nprint("\\n[3] Training XGBoost classifier on RTX 4060 GPU...")\nprint("   GPU Settings: device=\'cuda:0\'")\n\nxgb_model = XGBClassifier(\n    n_estimators=100,\n    max_depth=20,\n    learning_rate=0.1,\n    device=\'cuda:0\',           # Use GPU (updated from deprecated gpu_id)\n    tree_method=\'hist\',        # Use hist with device for GPU training\n    random_state=42,\n    eval_metric=\'mlogloss\',    # Multi-class log loss\n    verbosity=1                # Show GPU training progress\n)\n\nxgb_model.fit(X_train, y_train)\nprint(f"✅ Model trained on GPU")\n\n# Evaluate\nprint("\\n[4] Evaluating model...")\ntrain_score = xgb_model.score(X_train, y_train)\ntest_score = xgb_model.score(X_test, y_test)\nprint(f"✅ Training accuracy: {train_score:.4f}")\nprint(f"   Testing accuracy: {test_score:.4f}")\n\n# Classification report\nprint("\\n[5] Classification Report:")\ny_pred = xgb_model.predict(X_test)\nprint(classification_report(y_test, y_pred, target_names=[str(c) for c in label_encoder.classes_]))\n\n# Confusion matrix\nprint("\\n[6] Confusion Matrix:")\nfig, ax = plt.subplots(figsize=(10, 8))\ncm = confusion_matrix(y_test, y_pred)\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=label_encoder.classes_)\ndisp.plot(ax=ax, cmap=\'Blues\', values_format=\'d\')\nplt.title(\'Confusion Matrix - XGBoost GPU Model (RTX 4060)\')\nplt.tight_layout()\nplt.show()\n\nprint("="*70)\n')
+
+
+# In[23]:
+
+
+get_ipython().run_cell_magic('time', '', '# 💾 SAVE MODEL AND CLEANUP\nprint("="*70)\nprint("SAVING MODEL & CLEANUP")\nprint("="*70)\n\nimport joblib\nfrom datetime import datetime\n\n# Save model and label encoder\nmodel_filename = f"model_train/model_xgboost_gpu_{datetime.now().strftime(\'%Y%m%d_%H%M%S\')}.joblib"\nprint(f"\\n[1] Saving model to: {model_filename}")\njoblib.dump({\'model\': xgb_model, \'label_encoder\': label_encoder}, model_filename)\nprint(f"✅ Model and label encoder saved")\n\n# Save model info\ninfo = {\n    "timestamp": datetime.now().isoformat(),\n    "data_source": "Microsoft Planetary Computer STAC",\n    "collections": ["sentinel-2-l2a", "sentinel-1-rtc"],\n    "features": ["NDVI_mean", "VH_dB_mean", "VV_dB_mean"],\n    "training_samples": len(X_train),\n    "testing_samples": len(X_test),\n    "train_accuracy": float(train_score),\n    "test_accuracy": float(test_score),\n    "model_type": "XGBClassifier",\n    "device": "cuda:0",\n    "gpu_device": "RTX 4060",\n    "tree_method": "hist",\n    "n_estimators": 100,\n    "max_depth": 20,\n    "learning_rate": 0.1\n}\n\nimport json\ninfo_filename = model_filename.replace(\'.joblib\', \'_info.json\')\nwith open(info_filename, \'w\') as f:\n    json.dump(info, f, indent=2)\nprint(f"✅ Model info saved to: {info_filename}")\n\n# No cleanup needed (Dask removed)\nprint("\\n[2] Cleanup complete")\n\nprint("="*70)\n\nprint("\\n" + "="*70)\n\nprint("🎉 TRAINING COMPLETE!")\n')
+

02.predict_ODC.py

@@ -0,0 +1,191 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+get_ipython().run_cell_magic('time', '', '%matplotlib inline\n\nimport importlib\nimport new_import_ODC  \n\nimportlib.reload(new_import_ODC)\n\nfrom new_import_ODC import *\n')
+
+
+# In[2]:
+
+
+get_ipython().run_cell_magic('time', '', '# Dask gateway\ncluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))\ndc = datacube.Datacube()\n\n# Configure s3 access\nconfigure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n\nclient\n')
+
+
+# In[3]:
+
+
+## cấu hình thời gian lấy ảnh và tọa độ
+date_range = ('2022-09-01', '2023-10-01')
+longtitude_range = (105.86575, 105.94120)
+latitude_range = (9.65070, 9.69850)
+
+
+# In[4]:
+
+
+## truy vấn ảnh vệ tinh sen2
+data = load_data(dc, date_range, longtitude_range, latitude_range)
+notebook_utils.heading(notebook_utils.xarray_object_size(data))
+display(data)
+
+
+# In[5]:
+
+
+get_ipython().run_cell_magic('time', '', '# Tiến hành loại bỏ các vị trí bị mây ảnh hưởng\nresult = mask_clean(data)\nprogress(result)\n')
+
+
+# In[6]:
+
+
+# Tiến hành tính toán NDVI
+ds1 = calculate_indices(result, index='NDVI', satellite_mission='s2')
+ndvi = ds1["NDVI"]
+display(ndvi)
+
+
+# In[7]:
+
+
+## ảnh NDVI chưa điền mây (fill nan)
+plt.imshow(ndvi.isel(time=50))
+
+
+# In[8]:
+
+
+# đặt thời gian các mùa
+time_split = [slice('2022-09-01', '2023-01-01'), 
+               slice('2023-01-01', '2023-05-01'),
+               slice('2023-05-01', '2023-07-01'),
+               slice('2023-07-01', '2023-10-01')]
+
+# Điền mây ở các vị trí mang giá trị nan (fill nan)
+fill_nan_ndvi = fill_nan(ndvi, time_split)
+
+# In kết quả ảnh ndvi đã điền mây (đã fill nan)
+plt.imshow(fill_nan_ndvi.isel(time=50))
+
+
+# In[9]:
+
+
+get_ipython().run_cell_magic('time', '', "## tính ndvi theo tháng\naverage_ndvi = fill_nan_ndvi.resample(time='1M').mean().persist()\nprogress(average_ndvi)\n")
+
+
+# In[10]:
+
+
+# compute average_ndvi
+average_ndvi = average_ndvi.compute()
+
+
+# In[11]:
+
+
+# load dữ liệu sen1
+coordinates = (longtitude_range, latitude_range)
+dsvh, dsvv = load_data_sen1(dc, date_range, coordinates)
+average_vv = calculate_average(dsvv, time_pattern='1M')
+average_vh = calculate_average(dsvh, time_pattern='1M')
+
+
+# In[12]:
+
+
+# load model RF
+loaded_model = joblib.load(os.path.join("model_train", "model_odc.joblib"))
+
+# dự đoán
+data_array = predict(loaded_model, data.rio.crs, average_ndvi, average_vh, average_vv)
+
+
+# In[13]:
+
+
+# cấu hình màu cho các loại đất
+colors = [
+    "#abcee9",
+    "#ffef44",
+    "#c4ff9e",
+    "#ffd6a8",
+    "#93ddda",
+    "#1aeef7",
+    "#ffa7f2",
+    "#33ee33"
+]
+labels = [
+    "Lúa tôm",
+    "Lúa",
+    "CHN",
+    "CLN",
+    "TS",
+    "Sông",
+    "Đất xây dựng",
+    "Rừng"
+]
+# hiển thị phân loại sử dụng đất
+cmap = ListedColormap(colors)
+img = data_array.plot(cmap=cmap, add_colorbar=False)
+cbar = plt.colorbar(img)
+cbar.ax.set_yticklabels(labels)
+plt.title("Phân loại sử dụng đất")
+plt.axis('off')
+plt.show()
+
+
+# In[14]:
+
+
+## cấu hình shapefile ranh giới thuận hòa và vh vv file
+thuanhoa_path = "ThuanHoa/region/ST_ThuanHoa_Boundaryofficially.shp"
+
+# cắt theo ranh giới xã thuận hòa
+region_result = cut_according_shp(thuanhoa_path, average_ndvi, data_array)
+
+
+# In[15]:
+
+
+# hiển thị kết quả phân loại sử dụng đất
+colorval = list(range(len(colors)))
+options = {
+    'title': 'Phân loại sử dụng đất',
+    'cmap': colors,
+    'clim': (0, 8),
+    'aspect': 'equal',
+    'colorbar_opts': {
+        'major_label_overrides': dict(zip(colorval, labels)),
+        'major_label_text_align': 'left',
+        'ticker': FixedTicker(ticks=colorval),
+    },
+}
+                                      
+region_result.hvplot(
+    rasterize = True,                  # Use Datashader, particularly useful for dask arrays
+    aggregator = reductions.mode(),    # Datashader selects mode value, requires 'hv.Image'
+).options(opts.Image(**options))
+
+
+# In[16]:
+
+
+# Lưu lại kết quả
+region_result.rio.to_raster("KetQuaPhanLoaiDatODC.tif")
+
+
+# In[17]:
+
+
+# đóng client, cluster
+client.close()
+cluster.close()
+
+
+# In[ ]:
+
+
+
+

03.compare_ODC.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+# Khai báo các thư viện cần thiết
+from new_import_ODC import *
+
+# Khai báo đường dẫn đến kết quả phân loại và dữ liệu của địa phương
+KD_path = "ThuanHoa/KhoanhDat/ThuanHoa_TKDD2022.shp"
+KetQuaPhanLoaiDat = "KetQuaPhanLoaiDatODC.tif"
+
+
+# In[2]:
+
+
+# khai báo các loại đất từ dữ liệu kiểm kê ứng với các hiện trạng được phân loại từ viễn thám
+CODE_MAP = {
+    "BHK": 2,
+    "CLN": 3,
+    "DGD": 6,
+    "DGT": 6,
+    "DNL": 6,
+    "DRA": 6,
+    "DSH": 6,
+    "DTL": 5,
+    "DTS": 6,
+    "DYT": 6,
+    "LUC": 1,
+    "NKH": 3,
+    "NTD": 6,
+    "NTS": 4,
+    "ONT": 6,
+    "SKC": 6,
+    "SKX": 6,
+    "SON": 5,
+    "TMD": 6,
+    "TON": 6,
+    "TSC": 6,
+}
+
+# Khai báo các nhãn phân loại đất ứng với 3 loại đất chính
+HT_MAP = {
+    "NN": {"name": "Đất Nông Nghiệp", "data": [1, 2, 3, 4]},
+    "PNN": {"name": "Đất Phi Nông Nghiệp", "data": [6]},
+    "TQ": {"name": "Đất Thổ Quả", "data": [15]},
+}
+
+
+# In[3]:
+
+
+# Tiến hành chồng lắp
+result = compare(KD_path, KetQuaPhanLoaiDat, CODE_MAP, HT_MAP)
+
+
+# In[4]:
+
+
+# cấu hình màu cho các loại sử dụng đất
+colors = [
+    "#abcee9",
+    "#ffffc0",
+    "#c4ff9e",
+    "#ffd6a8",
+    "#93ddda",
+    "#1aeef7",
+    "#ffa7f2",
+    "#33ee33",
+]
+labels = ["Lúa tôm", "Lúa", "CHN", "CLN", "TS", "Sông", "Đất xây dựng", "Rừng"]
+
+
+# In[5]:
+
+
+# Lưu kết quả
+save_result(result, HT_MAP)
+
+
+# In[6]:
+
+
+# hiển thị kết quả
+xx = []
+
+for k, v in result.items():
+    rs = merge_arrays(v, nodata=np.nan)
+    xx.append(rs.squeeze(drop=True))
+xx = xr.concat(xx, pd.Index([HT_MAP[x]["name"] for x in HT_MAP], name="name"))
+
+colorval = list(range(len(colors)))
+options = {
+    "cmap": colors,
+    "clim": (0, 8),
+    "aspect": "equal",
+    "height": 400,
+    "colorbar_opts": {
+        "major_label_overrides": dict(zip(colorval, labels)),
+        "major_label_text_align": "left",
+        "ticker": FixedTicker(ticks=colorval),
+    },
+}
+
+xx.hvplot(
+    groupby="name",
+    rasterize=True,  # Use Datashader, particularly useful for dask arrays
+    aggregator=reductions.mode(),  # Datashader selects mode value, requires 'hv.Image'
+).options(opts.Image(**options))
+
+
+# In[ ]:
+
+
+
+

fix_all.py

@@ -0,0 +1,25 @@
+import glob, json
+
+changed_files = []
+for file_path in glob.glob('*.ipynb'):
+    with open(file_path, 'r', encoding='utf-8') as f:
+        nb = json.load(f)
+        
+    changed = False
+    for cell in nb.get('cells', []):
+        if cell.get('cell_type') == 'code':
+            source = cell.get('source', [])
+            for i, line in enumerate(source):
+                if 'time=50' in line:
+                    source[i] = line.replace('time=50', 'time=0')
+                    changed = True
+                if 'load_data_sen1(dc,' in line:
+                    source[i] = line.replace('load_data_sen1(dc,', 'load_data_sen1(None,')
+                    changed = True
+                    
+    if changed:
+        with open(file_path, 'w', encoding='utf-8') as f:
+            json.dump(nb, f, indent=1)
+        changed_files.append(file_path)
+
+print('Fixed issues in:', changed_files)

fix_new_train.py

@@ -0,0 +1,20 @@
+import json
+
+def fix_import(file_path):
+    with open(file_path, 'r', encoding='utf-8') as f:
+        nb = json.load(f)
+    changed = False
+    for cell in nb.get('cells', []):
+        if cell.get('cell_type') == 'code':
+            source = cell.get('source', [])
+            if isinstance(source, list):
+                for i, line in enumerate(source):
+                    if "from new_import import *" in line:
+                        source[i] = line.replace("from new_import import *", "from new_import_ODC import *")
+                        changed = True
+    if changed:
+        with open(file_path, 'w', encoding='utf-8') as f:
+            json.dump(nb, f, indent=1)
+        print(f"Fixed {file_path}")
+
+fix_import('new_train.ipynb')

fix_typo.py

@@ -0,0 +1,22 @@
+import json
+
+def fix_filename(file_path):
+    with open(file_path, 'r', encoding='utf-8') as f:
+        nb = json.load(f)
+    changed = False
+    for cell in nb.get('cells', []):
+        if cell.get('cell_type') == 'code':
+            source = cell.get('source', [])
+            if isinstance(source, list):
+                for i, line in enumerate(source):
+                    if "ST_training data_updated_1130points.shp" in line:
+                        source[i] = line.replace("ST_training data_updated_1130points.shp", "ST_training_data_updated_1130points.shp")
+                        changed = True
+    if changed:
+        with open(file_path, 'w', encoding='utf-8') as f:
+            json.dump(nb, f, indent=1)
+        print(f"Fixed typo in {file_path}")
+
+import glob
+for nb in glob.glob("*.ipynb"):
+    fix_filename(nb)

inspect_cells_train.py

@@ -0,0 +1,7 @@
+import json
+nb = json.load(open('01.train_ODC.ipynb'))
+for idx, cell in enumerate(nb['cells']):
+    if cell['cell_type'] == 'code':
+        print(f"Cell {idx}:")
+        print("".join(cell['source'][:3]))
+        print("-" * 20)

microsoft_planetary_migration.md

@@ -0,0 +1,105 @@
+# Hướng dẫn Chuyển đổi dữ liệu vệ tinh sang Microsoft Planetary Computer STAC
+
+Tài liệu này ghi chú lại các bước chuẩn hóa và các đoạn code mẫu để chuyển đổi việc tải dữ liệu vệ tinh (Sentinel-1, Sentinel-2) từ kho lưu trữ đóng (như AWS S3 yêu cầu xác thực) sang nền tảng mở **Microsoft Planetary Computer STAC API**. Bạn có thể dùng tài liệu này làm context (ngữ cảnh) gửi cho các AI khác để chúng hiểu cách thực hiện tương tự.
+
+---
+
+## 1. Mục đích
+- Bỏ qua các lỗi liên quan đến xác thực đám mây (VD: `RasterioIOError: AWS_SECRET_ACCESS_KEY not defined`).
+- Tải dữ liệu miễn phí, trực tiếp từ kho dữ liệu mở của Microsoft Planetary Computer.
+- Đảm bảo đầu ra (output) của dữ liệu STAC giống hệt với định dạng của ảnh TIF gốc tải bằng `rioxarray` để không làm hỏng các luồng xử lý Machine Learning ở phía sau.
+
+## 2. Các thư viện bắt buộc (Dependencies)
+Đảm bảo môi trường Python có cài đặt các thư viện sau:
+```python
+import pystac_client
+import planetary_computer
+import odc.stac
+import xarray as xr
+import rioxarray
+```
+
+## 3. Các bước thực hiện chi tiết
+
+### Bước 1: Kết nối đến STAC API và truy vấn dữ liệu
+Thay vì dùng `rioxarray.open_rasterio("s3://...")`, chúng ta khởi tạo STAC Client và tìm kiếm dữ liệu theo tọa độ (`bbox`) và thời gian (`datetime`).
+
+```python
+# 1. Kết nối STAC Client có kèm chữ ký xác thực (sign_inplace) của Microsoft
+catalog = pystac_client.Client.open(
+    "https://planetarycomputer.microsoft.com/api/stac/v1",
+    modifier=planetary_computer.sign_inplace,
+)
+
+# 2. Định nghĩa toạ độ và thời gian
+bbox = [105.5, 9.2, 106.4, 10.0] # [min_lon, min_lat, max_lon, max_lat]
+datetime = "2022-09-01/2023-10-01"
+
+# 3. Tìm kiếm Items
+# Thay "sentinel-1-rtc" bằng "sentinel-2-l2a" nếu tải ảnh quang học
+search = catalog.search(
+    collections=["sentinel-1-rtc"],
+    bbox=bbox,
+    datetime=datetime,
+)
+items = list(search.items())
+```
+
+### Bước 2: Tải dữ liệu xuống xarray bằng `odc.stac`
+Thay vì tải thủ công từng link URL, `odc.stac.load` sẽ tự động tải, cắt ảnh theo `bbox`, đổi hệ tọa độ (reproject) và ghép lại thành một khối dữ liệu không gian - thời gian (DataCube).
+
+```python
+# Tải dữ liệu thành xarray Dataset
+ds_s1 = odc.stac.load(
+    items,
+    bands=["vv", "vh"],           # Tên các band cần tải
+    bbox=bbox,
+    crs="EPSG:32648",             # Ép về hệ toạ độ đích (VD: UTM Zone 48N cho VN)
+    resolution=10,                # Độ phân giải (10 mét)
+    chunks={"x": 2048, "y": 2048, "time": 1} # Dùng Dask chunking để tránh tràn RAM
+)
+```
+
+### Bước 3: Nén trục thời gian (Temporal Compositing)
+Dữ liệu từ STAC sẽ có 3 chiều: `(time, y, x)`. Do ảnh TIF gốc cũ thường là ảnh đã được nén (ví dụ trung bình của 1 năm), ta cần dùng phép tính trung vị (`median`) hoặc trung bình (`mean`) để triệt tiêu trục `time`, biến dữ liệu thành dạng 2D `(y, x)`.
+
+```python
+# Tính giá trị trung vị theo thời gian
+ds_median = ds_s1.median(dim="time").compute()
+
+# Tách riêng các DataArray
+vv = ds_median["vv"]
+vh = ds_median["vh"]
+```
+
+### Bước 4: Khôi phục cấu trúc DataArray gốc (Mimic rioxarray)
+Hàm `rioxarray.open_rasterio` gốc luôn trả về dữ liệu có trục `band` (kích thước = 1). Để code Machine Learning bên dưới không bị lỗi "out of bounds" hay "missing dimension", ta phải thêm trục `band` giả và gán lại thông tin `crs`.
+
+```python
+# Thêm chiều 'band' để giống hệt rioxarray
+vv = vv.expand_dims(dim="band")
+vh = vh.expand_dims(dim="band")
+
+# Phục hồi metadata về toạ độ
+vv = vv.rio.write_crs("EPSG:32648")
+vh = vh.rio.write_crs("EPSG:32648")
+```
+
+### Bước 5: Quét và sửa các đoạn code "Hardcode" kích thước
+Do lưới tọa độ của STAC tự sinh (dựa trên bounding box) có thể lệch vài pixel so với lưới của file TIF đã cắt tay trên S3 (VD: S3 là `8874 x 9902`, STAC là `8870 x 9900`), **phải tìm và xóa bỏ toàn bộ các con số fix cứng trong mảng**.
+
+*Code cũ sai lầm:*
+```python
+tmp = np.ones((8874, 9902))
+final_label = final_label.reshape(8874, 9902)
+```
+
+*Code chuẩn hóa:*
+```python
+# Lấy linh động theo shape thực tế của xarray
+tmp = np.ones((ds_vhvv.shape[1], ds_vhvv.shape[2]))
+final_label = final_label.reshape(ds_vhvv.shape[1], ds_vhvv.shape[2])
+```
+
+## 4. Tổng kết
+Chỉ cần cung cấp tài liệu này cho bất kỳ AI nào, yêu cầu: *"Hãy refactor (viết lại) hàm load file TIF của tôi theo đúng 5 bước trong tài liệu Microsoft Planetary Computer này"*, AI đó sẽ có đủ toàn bộ tư duy và code mẫu để hoàn thành công việc một cách mượt mà nhất.

model_xgboost_info.json

@@ -0,0 +1,31 @@
+{
+  "model_type": "XGBoost",
+  "num_classes": 8,
+  "classes": [
+    "Lua tom",
+    "Lua",
+    "CHN",
+    "CLN",
+    "TS",
+    "Song",
+    "Dat xay dung",
+    "Rung"
+  ],
+  "num_features": 3,
+  "params": {
+    "objective": "multi:softmax",
+    "num_class": 8,
+    "max_depth": 6,
+    "learning_rate": 0.1,
+    "n_estimators": 200,
+    "subsample": 0.8,
+    "colsample_bytree": 0.8,
+    "random_state": 42,
+    "n_jobs": -1,
+    "eval_metric": "mlogloss"
+  },
+  "accuracy": 0.28761061946902655,
+  "precision": 0.35339400643604185,
+  "recall": 0.28761061946902655,
+  "f1_score": 0.23460742664282486
+}

new_import_ODC.py

@@ -1,8 +1,16 @@
+TEST_MODE = True
+RESOLUTION = 1000 if TEST_MODE else 10
 import matplotlib.pyplot as plt
 
 # Common imports and settings
 import os, sys
 os.environ['USE_PYGEOS'] = '0'
+os.environ["GDAL_HTTP_MAX_RETRY"] = "5"
+os.environ["GDAL_HTTP_RETRY_DELAY"] = "2"
+os.environ["GDAL_HTTP_CONNECTION_TIMEOUT"] = "10"
+os.environ["GDAL_HTTP_TIMEOUT"] = "30"
+os.environ["CPL_VSIL_CURL_ALLOWED_EXTENSIONS"] = ".tif,.tiff"
+os.environ["GDAL_DISABLE_READDIR_ON_OPEN"] = "YES"
 from IPython.display import Markdown
 import pandas as pd
 pd.set_option("display.max_rows", None)
@@ -13,15 +21,13 @@ import datacube
 from datacube.utils.rio import configure_s3_access
 from datacube.utils import masking
 from datacube.utils.cog import write_cog
-# https://github.com/GeoscienceAustralia/dea-notebooks/tree/develop/Tools
+# removed deafrica_tools imports to avoid ipyleaflet error
-from dea_tools.plotting import display_map, rgb
-from dea_tools.datahandling import mostcommon_crs
 
 # EASI defaults
-easinotebooksrepo = '/home/jovyan/easi-notebooks'
+easinotebooksrepo = '/home/x79/CSIROBoeingPhase4-Vietnam'
 if easinotebooksrepo not in sys.path: sys.path.append(easinotebooksrepo)
 from easi_tools import EasiDefaults, xarray_object_size, notebook_utils, unset_cachingproxy
-from easi_tools.load_s2l2a import load_s2l2a_with_offset
+# from easi_tools.load_s2l2a import load_s2l2a_with_offset
 from dask.distributed import progress
 
 # Data tools
@@ -31,7 +37,7 @@ from datetime import datetime
 # Datacube
 from datacube.utils import masking  # https://github.com/opendatacube/datacube-core/blob/develop/datacube/utils/masking.py
 from odc.algo import enum_to_bool   # https://github.com/opendatacube/odc-algo/blob/main/odc/algo/_masking.py
-from odc.algo import xr_reproject   # https://github.com/opendatacube/odc-algo/blob/main/odc/algo/_warp.py
+# removed xr_reproject
 from datacube.utils.geometry import GeoBox, box  # https://github.com/opendatacube/datacube-core/blob/develop/datacube/utils/geometry/_base.py
 
 # Holoviews, Datashader and Bokeh
@@ -83,56 +89,73 @@ import joblib
 
 def load_data(dc, date_range, longtitude_range, latitude_range):
     product = 's2_l2a'
-    query = {
+    bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
-        'product': product,                     # Product name
+    
-        'x': longtitude_range,    # "x" axis bounds
+    import pystac_client
-        'y': latitude_range,      # "y" axis bounds
+    import planetary_computer
-        'time': date_range,           # Any parsable date strings
+    import odc.stac
-    }
+    
-    native_crs = notebook_utils.mostcommon_crs(dc, query)
+    catalog = pystac_client.Client.open(
-    print(f'Most common native CRS: {native_crs}')
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
-    measurements = ['red', 'nir', 'scl']
+        modifier=planetary_computer.sign_inplace,
-
-    load_params = {
-        'measurements': measurements,                   # Selected measurement or alias names
-        'output_crs': native_crs,                       # Target EPSG code
-        'resolution': (-10, 10),                        # Target resolution
-        'group_by': 'solar_day',                        # Scene grouping
-        'dask_chunks': {'x': 2048, 'y': 2048},          # Dask chunks
-    }
-    data = load_s2l2a_with_offset(
-        dc,
-        query | load_params   # Combine the two dicts that contain our search and load parameters
     )
+    search = catalog.search(
+        collections=["sentinel-2-l2a"],
+        bbox=bbox,
+        datetime=f"{date_range[0]}/{date_range[1]}",
+    )
+    items = list(search.items())
+    
+    data = odc.stac.load(
+        items,
+        bands=["red", "nir", "SCL"],
+        bbox=bbox,
+        crs="EPSG:32648",
+        resolution=RESOLUTION,
+        chunks={"x": 2048, "y": 2048, "time": 1},
+        groupby="solar_day"
+    )
+    if "SCL" in data.data_vars:
+        data = data.rename({"SCL": "scl"})
     return data
 
 
 def mask_clean(data):
-    flag_name = 'scl'
+    # For Sentinel-2 L2A SCL: 
-    flag_desc = masking.describe_variable_flags(data[flag_name])  # Pandas dataframe
+    # 2: Dark Area Pixels, 4: Vegetation, 5: Not Vegetated, 6: Water
-    display(flag_desc)
+    good_pixel_mask = data['scl'].isin([2, 4, 5, 6])
-    display(flag_desc.loc['qa'].values[1])
-    # Create a "data quality" Mask layer
-    flags_def = flag_desc.loc['qa'].values[1]
-    good_pixel_flags = [flags_def[str(i)] for i in [2, 4, 5, 6]]  # To pass strings to enum_to_bool()
-
-    # enum_to_bool calculates the pixel-wise "or" of each set of pixels given by good_pixel_flags
-    # 1 = good data
-    # 0 = "bad" data
-    good_pixel_mask = enum_to_bool(data[flag_name], good_pixel_flags)
     data_layer_names = [x for x in data.data_vars if x != 'scl']
-    # Apply good pixel mask to blue, green, red and nir.
+    # Apply good pixel mask
     result = data[data_layer_names].where(good_pixel_mask).persist()
     return result
 
 
 def fill_nan(ndvi, time_split):
+    if len(ndvi.time) == 0:
+        return ndvi
+    
+    # If the total time duration is less than 90 days, skip seasonal splitting
+    try:
+        total_days = (ndvi.time[-1] - ndvi.time[0]).dt.days.item()
+        if total_days < 90:
+            return ndvi.bfill(dim="time").ffill(dim="time")
+    except Exception:
+        pass
+        
     rs = []
     for times in time_split:
-        tmp = ndvi.sel(time=times)
+        try:
-        fill_ds = tmp.sel(time=times).bfill(dim='time')
+            tmp = ndvi.sel(time=times)
-        fill_ds = fill_ds.sel(time=times).ffill(dim='time')   
+            if len(tmp.time) == 0:
-        rs.append(fill_ds)
+                continue
+            fill_ds = tmp.bfill(dim='time').ffill(dim='time')   
+            rs.append(fill_ds)
+        except Exception:
+            continue
+            
+    if len(rs) == 0:
+        return ndvi.bfill(dim="time").ffill(dim="time")
+        
     merged_ndvi = xr.concat([i for i in rs], dim="time")
     fill_m = merged_ndvi.bfill(dim="time")
     fill_m = fill_m.ffill(dim="time")
@@ -144,10 +167,49 @@ def load_train_data(train_path):
     return train
 
 
-def load_sen1(name_vh, name_vv):
+def load_sen1(bbox, time_range):
-    dsvv = rioxarray.open_rasterio(name_vv)
+    import pystac_client
-    dsvh = rioxarray.open_rasterio(name_vh)
+    import planetary_computer
-    return dsvh, dsvv
+    import odc.stac
+    
+    # Kết nối STAC Client
+    catalog = pystac_client.Client.open(
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
+        modifier=planetary_computer.sign_inplace,
+    )
+
+    # Tìm kiếm Items
+    search = catalog.search(
+        collections=["sentinel-1-rtc"],
+        bbox=bbox,
+        datetime=time_range,
+    )
+    items = list(search.items())
+
+    # Tải dữ liệu thành xarray Dataset
+    ds_s1 = odc.stac.load(
+        items,
+        bands=["vv", "vh"],
+        bbox=bbox,
+        crs="EPSG:32648",
+        resolution=RESOLUTION,
+        chunks={"x": 2048, "y": 2048, "time": 1}
+    )
+
+    # Tính giá trị trung vị theo thời gian
+    ds_median = ds_s1.median(dim="time").compute()
+    vv = ds_median["vv"]
+    vh = ds_median["vh"]
+
+    # Thêm chiều 'band' để giống hệt rioxarray
+    vv = vv.expand_dims(dim="band")
+    vh = vh.expand_dims(dim="band")
+
+    # Phục hồi metadata về toạ độ
+    vv = vv.rio.write_crs("EPSG:32648")
+    vh = vh.rio.write_crs("EPSG:32648")
+
+    return vh, vv
 
 
 def get_data_sen1_and_sen2(train, average_ndvi, dsvh, dsvv):
@@ -267,6 +329,10 @@ def save_model(name_file, model, metadata=None, label_encoder=None):
     
     
 def predict(model, data_crs, ndvi, vh, vv):
+    # Unpack model if it is wrapped in a dictionary (from ModelManager)
+    if isinstance(model, dict) and 'model' in model:
+        model = model['model']
+        
     data_predict = []
     for i in range(ndvi.shape[1]):
         ndvi_tmp = ndvi.isel(y=i).values
@@ -361,21 +427,35 @@ def save_result(result, HT_MAP):
         
 def load_data_sen1(dc, date_range, coordinates):
     longtitude_range, latitude_range = coordinates
-    data_sen1 = dc.load(
+    bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
-        product="sentinel1_grd_gamma0_10m",
+    
-        x=longtitude_range,
+    import pystac_client
-        y=latitude_range,
+    import planetary_computer
-        time=date_range,
+    import odc.stac
-        measurements=["vv", "vh"],
+    
-        output_crs="EPSG:32648",
+    catalog = pystac_client.Client.open(
-        resolution=(-10,10),
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
-        dask_chunks={"x":2048, "y":2048},
+        modifier=planetary_computer.sign_inplace,
-        skip_broken_datasets=True,
+    )
-        group_by='solar_day'
+    search = catalog.search(
+        collections=["sentinel-1-rtc"],
+        bbox=bbox,
+        datetime=f"{date_range[0]}/{date_range[1]}",
+    )
+    items = list(search.items())
+    
+    data_sen1 = odc.stac.load(
+        items,
+        bands=["vv", "vh"],
+        bbox=bbox,
+        crs="EPSG:32648",
+        resolution=RESOLUTION,
+        chunks={"x": 2048, "y": 2048, "time": 1},
+        groupby="solar_day"
     )
     
-    notebook_utils.heading(notebook_utils.xarray_object_size(data_sen1))
+    # notebook_utils.heading(notebook_utils.xarray_object_size(data_sen1))
-    display(data_sen1)
+    # display(data_sen1)
     dsvh = data_sen1.vh
     dsvv = data_sen1.vv
     
@@ -387,46 +467,42 @@ def calculate_average(data, time_pattern='1M'):
 
 def load_data_sen2(dc, date_range, coordinates):
     longtitude_range, latitude_range = coordinates
-    product = 's2_l2a'
+    bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
-    query = {
-        'product': product,                     # Product name
-        'x': longtitude_range,    # "x" axis bounds
-        'y': latitude_range,      # "y" axis bounds
-        'time': date_range,           # Any parsable date strings
-    }
-    native_crs = notebook_utils.mostcommon_crs(dc, query)
-    print(f'Most common native CRS: {native_crs}')
     
-    # measurements = ['red','green', 'blue', 'nir', 'scl']
+    import pystac_client
-    measurements = ['red', 'nir', 'scl']
+    import planetary_computer
-
+    import odc.stac
-    load_params = {
+    
-        'measurements': measurements,                   # Selected measurement or alias names
+    catalog = pystac_client.Client.open(
-        'output_crs': native_crs,                       # Target EPSG code
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
-        'resolution': (-10, 10),                        # Target resolution
+        modifier=planetary_computer.sign_inplace,
-        'group_by': 'solar_day',                        # Scene grouping
-        'dask_chunks': {'x': 2048, 'y': 2048},          # Dask chunks
-    }
-    data = load_s2l2a_with_offset(
-        dc,
-        query | load_params   # Combine the two dicts that contain our search and load parameters
     )
+    search = catalog.search(
+        collections=["sentinel-2-l2a"],
+        bbox=bbox,
+        datetime=f"{date_range[0]}/{date_range[1]}",
+    )
+    items = list(search.items())
+    
+    data = odc.stac.load(
+        items,
+        bands=["red", "nir", "SCL"],
+        bbox=bbox,
+        crs="EPSG:32648",
+        resolution=RESOLUTION,
+        chunks={"x": 2048, "y": 2048, "time": 1},
+        groupby="solar_day"
+    )
+    if "SCL" in data.data_vars:
+        data = data.rename({"SCL": "scl"})
     return data
 
 def mask_cloud(data):
-    flag_name = 'scl'
+    # For Sentinel-2 L2A SCL: 
-    flag_desc = masking.describe_variable_flags(data[flag_name])  # Pandas dataframe
+    # 2: Dark Area Pixels, 4: Vegetation, 5: Not Vegetated, 6: Water
-    display(flag_desc.loc['qa'].values[1])
+    good_pixel_mask = data['scl'].isin([2, 4, 5, 6])
-    # Create a "data quality" Mask layer
-    flags_def = flag_desc.loc['qa'].values[1]
-    good_pixel_flags = [flags_def[str(i)] for i in [2, 4, 5, 6]]  # To pass strings to enum_to_bool()
-
-    # enum_to_bool calculates the pixel-wise "or" of each set of pixels given by good_pixel_flags
-    # 1 = good data
-    # 0 = "bad" data
-    good_pixel_mask = enum_to_bool(data[flag_name], good_pixel_flags)
     data_layer_names = [x for x in data.data_vars if x != 'scl']
-    # Apply good pixel mask to blue, green, red and nir.
+    # Apply good pixel mask
     result = data[data_layer_names].where(good_pixel_mask).persist()
     return result
 

new_train.py

@@ -0,0 +1,264 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[49]:
+
+
+get_ipython().run_cell_magic('time', '', '%matplotlib inline\nfrom new_import import *\n')
+
+
+# In[2]:
+
+
+get_ipython().run_cell_magic('time', '', '# Cấu hình Daskgateway\ncluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,10))\n# Khai báo 1 Datacube là dc\ndc = datacube.Datacube()\n\n# Cấu hình truy cập dịch vụ S3\nconfigure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n\nclient\n')
+
+
+# LOAD VH, VV
+
+# In[47]:
+
+
+## cấu hình thời gian lấy ảnh và tọa độ
+date_range = ('2022-09-01', '2023-10-01')
+longtitude_range = (105.5, 106.4)
+latitude_range = (9.2, 10.0) 
+
+
+# In[3]:
+
+
+## cấu hình dữ liệu train và vh vv file
+train_path = "train/ST_training data_updated_1130points.shp"  # đường dẫn shp file train
+name_vh = "vh-0922_0923-full_ST.tif"
+name_vv = "vv-0922_0923-full_ST.tif"
+
+
+train = load_train_data(train_path)
+
+
+# In[4]:
+
+
+# %%time
+# ## tải về dữ liệu sen1
+# import os
+# if not os.path.exists(name_vh):
+#     !aws s3 cp s3://easi-asia-dc-data/staging/ctu/sentinel-1/vh-0922_0923-full_ST.tif vh-0922_0923-full_ST.tif
+# if not os.path.exists(name_vv):
+#     !aws s3 cp s3://easi-asia-dc-data/staging/ctu/sentinel-1/vv-0922_0923-full_ST.tif vv-0922_0923-full_ST.tif
+
+
+# In[5]:
+
+
+
+
+
+# In[38]:
+
+
+ds = dc.load(
+    product="sentinel1_grd_gamma0_20m",
+    x=(105.5, 106.4),
+    y=(9.2, 10.0),
+    time=("2022-09-01", "2023-10-01"),
+    measurements=["vv", "vh"],
+    output_crs="EPSG:32648",
+    resolution=(-10,10),
+    dask_chunks={"x":2048, "y":2048},
+    skip_broken_datasets=True,
+    group_by="solar_day"
+)
+notebook_utils.heading(notebook_utils.xarray_object_size(ds))
+ds
+
+
+# In[43]:
+
+
+vv_data = ds.vv
+vv_data
+
+
+# In[44]:
+
+
+bbox = [105.5, 9.2, 106.4, 10.0]
+time_range = "2022-09-01/2023-10-01"
+dsvh, dsvv = load_sen1(bbox, time_range)
+dsvv
+
+
+# LOAD SENTINEL 2
+# 
+# 
+
+# In[50]:
+
+
+data = load_data(dc, date_range, longtitude_range, latitude_range)
+notebook_utils.heading(notebook_utils.xarray_object_size(data))
+display(data)
+
+
+# In[8]:
+
+
+get_ipython().run_cell_magic('time', '', '# Tiến hành loại bỏ các vị trí bị mây ảnh hưởng\nresult = mask_clean(data)\nprogress(result)\n')
+
+
+# CALCULATING THE MEAN VALUE AND FILL TO NAN POINT
+
+# In[9]:
+
+
+ds1 = calculate_indices(result, index='NDVI', satellite_mission='s2')
+ndvi = ds1["NDVI"]
+average_ndvi = ndvi.resample(time='1M').mean().persist()  ## tính mean cho từng tháng -> time = 12
+progress(average_ndvi)
+
+
+# In[10]:
+
+
+dsvh.shape
+
+
+# In[11]:
+
+
+average_ndvi = average_ndvi.compute()
+average_ndvi = average_ndvi[:, :dsvh.shape[1], :dsvh.shape[2]]
+
+
+# In[12]:
+
+
+get_ipython().run_cell_magic('time', '', "filled_ds = average_ndvi.bfill(dim='time')\nfilled_ds = filled_ds.ffill(dim='time')\n")
+
+
+# FIND NAN POINT AFTER FILLING AND FILLING AGAIN WITH LINEARREGRESSION ALGORITHM
+
+# In[13]:
+
+
+nan_mask = filled_ds.isnull()
+
+# Print the NaN mask
+# print(nan_mask)
+
+# Count the number of NaNs
+num_nans = nan_mask.sum()
+print(f'Number of NaNs: {num_nans.values}')
+
+
+# In[14]:
+
+
+from sklearn.preprocessing import PolynomialFeatures
+from sklearn.linear_model import LinearRegression
+from sklearn.ensemble import RandomForestRegressor
+
+mask = ~np.isnan(filled_ds)
+X_train = np.stack([dsvh.values[mask], dsvv.values[mask]], axis=1)
+y_train = filled_ds.values[mask]
+
+
+# In[15]:
+
+
+model = LinearRegression()
+model.fit(X_train, y_train)
+
+
+# In[16]:
+
+
+X_pred = np.stack([dsvh.values[~mask], dsvv.values[~mask]], axis=1)
+filled_ds.values[~mask] = model.predict(X_pred)
+
+
+# MATCH LABEL TO DATASET
+
+# In[17]:
+
+
+get_ipython().run_cell_magic('time', '', '\n# Takes 1 minute to complete.\nloaded_datasets = {}\nfor idx, point in train.iterrows():\n    key = f"point_{idx + 1}"\n    try:\n        ndvi_data = filled_ds.sel(x=point.geometry.x, y=point.geometry.y, method=\'nearest\').values\n        vh_data = dsvh.sel(x=point.geometry.x, y=point.geometry.y, method=\'nearest\').values\n        vv_data = dsvv.sel(x=point.geometry.x, y=point.geometry.y, method=\'nearest\').values\n        loaded_datasets[key] = {\n            "data": np.concatenate((ndvi_data, vh_data, vv_data)),\n            "label": point.HT_code\n                               }\n    except Exception as e:\n        # loaded_datasets[key] = None\n        print(e)\n')
+
+
+# In[18]:
+
+
+label_mapping = {
+    "Lua tom": "0",
+    "Lua": "1",
+    "CHN": "2",
+    "CLN": "3",
+    "TS": "4",
+    "Song": "5",
+    "Dat xay dung": "6",
+    "Rung": "7"
+}
+label_encoder = LabelEncoder()
+
+# Fit and transform the labels
+labels = train.Hientrang.values
+numeric_labels = label_encoder.fit_transform([label_mapping[label] for label in labels])
+
+
+# In[19]:
+
+
+X = []
+x_new = []
+lb_new = []
+for k, v in loaded_datasets.items():
+    X.append(v)
+for i in range(len(X)):
+    if X[i] is not None:
+        x_new.append(X[i]["data"])
+        lb_new.append(numeric_labels[i])
+
+
+# BUILDING DATASETS 
+
+# In[20]:
+
+
+X_train, X_temp, y_train, y_temp= train_test_split(x_new, lb_new, test_size=0.4, random_state=42)
+X_val, X_test, y_val, y_test = train_test_split(X_temp, y_temp, test_size=0.5, random_state=42)
+
+
+# TRAIN MODEL
+
+# In[21]:
+
+
+get_ipython().run_cell_magic('time', '', 'from sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.svm import SVC\nfrom sklearn.metrics import accuracy_score\n\n# Define the models\nrf_model = RandomForestClassifier(random_state=42, n_jobs=-1)\nknn_model = KNeighborsClassifier()\nnb_model = GaussianNB()\nsvm_model = SVC()\n\n# Create a pipeline\npipeline = Pipeline([\n    (\'scaler\', StandardScaler()),  # Apply scaling\n    (\'classifier\', rf_model)       # Placeholder, will be set by param_grid\n])\n\n# Define the parameter grid for each classifier\nparam_grid = [\n    # RandomForest\n    {\n        \'classifier\': [rf_model],\n        \'classifier__n_estimators\': [100, 300, 500, 700],\n        \'classifier__max_depth\': [6, 8, 10, 15],\n        \'classifier__criterion\': [\'gini\', \'entropy\'],\n    },\n    # KNeighborsClassifier\n    {\n        \'classifier\': [knn_model],\n        \'classifier__n_neighbors\': [3, 5, 7, 9],\n        \'classifier__weights\': [\'uniform\', \'distance\'],\n        \'classifier__metric\': [\'euclidean\', \'manhattan\']\n    },\n    # Naive Bayes (GaussianNB doesn\'t have hyperparameters to tune here)\n    {\n        \'classifier\': [nb_model],\n    },\n    # SVM\n    {\n        \'classifier\': [svm_model],\n        \'classifier__C\': [0.1, 1, 10, 100],\n        \'classifier__kernel\': [\'linear\', \'rbf\'],\n        \'classifier__gamma\': [\'scale\', \'auto\']\n    }\n]\n\n# Use GridSearchCV to find the best classifier and hyperparameters\ngrid_search = GridSearchCV(pipeline, param_grid, cv=5, scoring=\'accuracy\', n_jobs=-1)\ngrid_search.fit(X_train, y_train)\n\n# Print out the best parameters and classifier\nbest_params = grid_search.best_params_\nprint("Best Parameters:", best_params)\n\n# Make predictions on the validation set\ny_pred = grid_search.predict(X_val)\n\n# Evaluate the results\naccuracy = accuracy_score(y_val, y_pred)\nprint(f"Accuracy: {round(accuracy, 2)*100} %")\n')
+
+
+# In[22]:
+
+
+## check accuracy score
+
+y_pred_test = grid_search.predict(X_test)
+test_accuracy = accuracy_score(y_test, y_pred_test)
+print(f"Accuracy for test data {round(test_accuracy, 2)*100} %")
+
+
+# In[23]:
+
+
+dir_save_model = "model_train"
+if not os.path.exists(dir_save_model):
+    os.mkdir(dir_save_model)
+joblib.dump(grid_search, os.path.join(dir_save_model, "model_new2.joblib"))
+
+
+# In[24]:
+
+
+client.close()
+cluster.close()
+

patch_load_sen1.py

@@ -0,0 +1,29 @@
+import json
+import glob
+
+def fix_load_sen1(file_path):
+    with open(file_path, 'r', encoding='utf-8') as f:
+        nb = json.load(f)
+        
+    changed = False
+    for cell in nb.get('cells', []):
+        if cell.get('cell_type') == 'code':
+            source = cell.get('source', [])
+            for i, line in enumerate(source):
+                if 'load_sen1(name_vh, name_vv)' in line:
+                    indent = line[:len(line) - len(line.lstrip())]
+                    replacement = (
+                        f"{indent}bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]\n"
+                        f"{indent}time_range = f'{{date_range[0]}}/{{date_range[1]}}'\n"
+                        f"{indent}{line.lstrip().replace('load_sen1(name_vh, name_vv)', 'load_sen1(bbox, time_range)')}"
+                    )
+                    source[i] = replacement
+                    changed = True
+                    
+    if changed:
+        with open(file_path, 'w', encoding='utf-8') as f:
+            json.dump(nb, f, indent=1)
+        print(f"Patched load_sen1 in {file_path}")
+
+for nb in glob.glob("*.ipynb"):
+    fix_load_sen1(nb)

patch_notebooks.py

@@ -0,0 +1,43 @@
+import json
+import glob
+
+def patch_notebook(file_path):
+    with open(file_path, 'r', encoding='utf-8') as f:
+        nb = json.load(f)
+        
+    changed = False
+    for cell in nb.get('cells', []):
+        if cell.get('cell_type') == 'code':
+            source = cell.get('source', [])
+            
+            # Check if this cell should be fully commented out
+            full_source = ''.join(source)
+            if 'dc.load(' in full_source or 'ds.vv' in full_source:
+                for i in range(len(source)):
+                    if not source[i].startswith('#'):
+                        source[i] = '# ' + source[i]
+                changed = True
+                continue
+            
+            # Otherwise, do line-by-line replacements
+            for i, line in enumerate(source):
+                if 'ST_training data_updated_1130points.shp' in line:
+                    source[i] = line.replace('ST_training data_updated_1130points.shp', 'ST_training_data_updated_1130points.shp')
+                    changed = True
+                if 'from new_import import *' in line:
+                    source[i] = line.replace('from new_import import *', 'from new_import_ODC import *')
+                    changed = True
+                if 'dc = datacube.Datacube()' in line:
+                    source[i] = line.replace('dc = datacube.Datacube()', 'dc = None')
+                    changed = True
+                if 'load_data(dc,' in line:
+                    source[i] = line.replace('load_data(dc,', 'load_data(None,')
+                    changed = True
+                    
+    if changed:
+        with open(file_path, 'w', encoding='utf-8') as f:
+            json.dump(nb, f, indent=1)
+        print(f"Patched {file_path}")
+
+for nb in glob.glob("*.ipynb"):
+    patch_notebook(nb)

processMask-test.ipynb

@@ -1132,7 +1132,7 @@
    "source": [
     "%%time\n",
     "%matplotlib inline\n",
-    "from new_import import *"
+    "from new_import_ODC import *"
    ]
   },
   {
@@ -1216,7 +1216,7 @@
     "%%time\n",
     "# Dask gateway\n",
     "cluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))\n",
-    "dc = datacube.Datacube()\n",
+    "dc = None\n",
     "\n",
     "# Configure s3 access\n",
     "configure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n",
@@ -1233,11 +1233,11 @@
    },
    "outputs": [],
    "source": [
-    "## cấu hình thời gian lấy ảnh và tọa độ\n",
+    "## c\u1ea5u h\u00ecnh th\u1eddi gian l\u1ea5y \u1ea3nh v\u00e0 t\u1ecda \u0111\u1ed9\n",
-    "# date_range = ('2022-09-01', '2023-10-01')\n",
+    "# date_range = ('2022-09-01', '2022-10-01')\n",
     "# longtitude_range = (105.86575, 105.94120)\n",
     "# latitude_range = (9.65070, 9.69850)\n",
-    "date_range = ('2022-09-01', '2023-10-01')\n",
+    "date_range = ('2022-09-01', '2022-10-01')\n",
     "longtitude_range = (105.5, 106.4)\n",
     "latitude_range = (9.2, 10.0) "
    ]
@@ -1396,7 +1396,7 @@
        "\n",
        ".xr-section-summary-in + label:before {\n",
        "  display: inline-block;\n",
-       "  content: '►';\n",
+       "  content: '\u25ba';\n",
        "  font-size: 11px;\n",
        "  width: 15px;\n",
        "  text-align: center;\n",
@@ -1407,7 +1407,7 @@
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label:before {\n",
-       "  content: '▼';\n",
+       "  content: '\u25bc';\n",
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label > span {\n",
@@ -2424,8 +2424,8 @@
     }
    ],
    "source": [
-    "## truy vấn ảnh vệ tinh sen2\n",
+    "## truy v\u1ea5n \u1ea3nh v\u1ec7 tinh sen2\n",
-    "data = load_data(dc, date_range, longtitude_range, latitude_range)\n",
+    "data = load_data(None, date_range, longtitude_range, latitude_range)\n",
     "notebook_utils.heading(notebook_utils.xarray_object_size(data))\n",
     "display(data)"
    ]
@@ -2440,10 +2440,10 @@
    "outputs": [],
    "source": [
     "# Specify the start and end times \n",
-    "min_date = '2022-09-01' # Thời gian bắt đầu lấy data cho quá trình train\n",
+    "min_date = '2022-09-01' # Th\u1eddi gian b\u1eaft \u0111\u1ea7u l\u1ea5y data cho qu\u00e1 tr\u00ecnh train\n",
-    "max_date = '2023-10-01' # Thời gian kết thúc lấy data cho quá trình train\n",
+    "max_date = '2022-10-01' # Th\u1eddi gian k\u1ebft th\u00fac l\u1ea5y data cho qu\u00e1 tr\u00ecnh train\n",
     "# Just do 1 month for testing\n",
-    "# max_date = '2022-10-01' # Thời gian kết thúc lấy data cho quá trình train\n",
+    "# max_date = '2022-10-01' # Th\u1eddi gian k\u1ebft th\u00fac l\u1ea5y data cho qu\u00e1 tr\u00ecnh train\n",
     "\n",
     "# Specify a spatail region to search using latitude/longitude cooridinates\n",
     "min_longitude, max_longitude = (105.5, 106.4)\n",
@@ -2657,7 +2657,7 @@
        "\n",
        ".xr-section-summary-in + label:before {\n",
        "  display: inline-block;\n",
-       "  content: '►';\n",
+       "  content: '\u25ba';\n",
        "  font-size: 11px;\n",
        "  width: 15px;\n",
        "  text-align: center;\n",
@@ -2668,7 +2668,7 @@
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label:before {\n",
-       "  content: '▼';\n",
+       "  content: '\u25bc';\n",
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label > span {\n",
@@ -3423,23 +3423,23 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "CPU times: user 1.57 s, sys: 177 µs, total: 1.57 s\n",
+      "CPU times: user 1.57 s, sys: 177 \u00b5s, total: 1.57 s\n",
       "Wall time: 1.66 s\n"
      ]
     }
    ],
    "source": [
-    "%%time\n",
+    "# %%time\n",
     "# The replacement \"dc.load()\" function for this product\n",
-    "data = load_s2l2a_with_offset(\n",
+    "# data = load_s2l2a_with_offset(\n",
-    "    dc,\n",
+    "#     dc,\n",
-    "    query | load_params   # Combine the two dicts that contain our search and load parameters\n",
+    "#     query | load_params   # Combine the two dicts that contain our search and load parameters\n",
-    ")\n",
+    "# )\n",
-    "\n",
+    "# \n",
     "# This line prints the total size of the dataset hat was loaded\n",
-    "notebook_utils.heading(notebook_utils.xarray_object_size(data))\n",
+    "# notebook_utils.heading(notebook_utils.xarray_object_size(data))\n",
-    "\n",
+    "# \n",
-    "display(data)"
+    "# display(data)"
    ]
   },
   {
@@ -3546,9 +3546,9 @@
    ],
    "source": [
     "# %%time\n",
-    "# # Tiến hành loại bỏ các vị trí bị mây ảnh hưởng\n",
+    "# # Ti\u1ebfn h\u00e0nh lo\u1ea1i b\u1ecf c\u00e1c v\u1ecb tr\u00ed b\u1ecb m\u00e2y \u1ea3nh h\u01b0\u1edfng\n",
     "# result = mask_clean(data)\n",
-    "# progress(result)"
+    "# # progress(result)"
    ]
   },
   {
@@ -3684,7 +3684,7 @@
        "\n",
        ".xr-section-summary-in + label:before {\n",
        "  display: inline-block;\n",
-       "  content: '►';\n",
+       "  content: '\u25ba';\n",
        "  font-size: 11px;\n",
        "  width: 15px;\n",
        "  text-align: center;\n",
@@ -3695,7 +3695,7 @@
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label:before {\n",
-       "  content: '▼';\n",
+       "  content: '\u25bc';\n",
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label > span {\n",
@@ -4174,7 +4174,7 @@
     }
    ],
    "source": [
-    "# Tiến hành tính toán NDVI\n",
+    "# Ti\u1ebfn h\u00e0nh t\u00ednh to\u00e1n NDVI\n",
     "ds1 = calculate_indices(result, index='NDVI', satellite_mission='s2')\n",
     "ndvi = ds1[\"NDVI\"]\n",
     "display(ndvi)"
@@ -4213,9 +4213,9 @@
    ],
    "source": [
     "%%time\n",
-    "## tính ndvi theo tháng\n",
+    "## t\u00ednh ndvi theo th\u00e1ng\n",
     "average_ndvi = ndvi.resample(time='1M').mean().persist()\n",
-    "progress(average_ndvi)"
+    "# progress(average_ndvi)"
    ]
   },
   {
@@ -4253,12 +4253,12 @@
     }
    ],
    "source": [
-    "# cấu hình vh vv file\n",
+    "# c\u1ea5u h\u00ecnh vh vv file\n",
     "# name_vh = \"ThuanHoa/ThuanHoa_VH.tif\"\n",
     "# name_vv = \"ThuanHoa/ThuanHoa_VV.tif\"\n",
     "\n",
-    "# load dữ liệu sen1\n",
+    "# load d\u1eef li\u1ec7u sen1\n",
-    "# dsvh, dsvv = load_sen1(name_vh, name_vv)\n",
+    "bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]\ntime_range = f'{date_range[0]}/{date_range[1]}'\n# dsvh, dsvv = load_sen1(bbox, time_range)\n",
     "\n",
     "name_vh = \"vh-0922_0923-full_ST.tif\"\n",
     "name_vv = \"vv-0922_0923-full_ST.tif\"\n",
@@ -4268,7 +4268,7 @@
     "if not os.path.exists(name_vv):\n",
     "    !aws s3 cp s3://easi-asia-dc-data/staging/ctu/sentinel-1/vv-0922_0923-full_ST.tif vv-0922_0923-full_ST.tif\n",
     "    \n",
-    "dsvh, dsvv = load_sen1(name_vh, name_vv)"
+    "bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]\ntime_range = f'{date_range[0]}/{date_range[1]}'\ndsvh, dsvv = load_sen1(bbox, time_range)"
    ]
   },
   {
@@ -4326,7 +4326,7 @@
     {
      "data": {
       "text/html": [
-       "<style>#sk-container-id-1 {color: black;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>LinearRegression()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LinearRegression</label><div class=\"sk-toggleable__content\"><pre>LinearRegression()</pre></div></div></div></div></div>"
+       "<style>#sk-container-id-1 {color: black;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"\u25b8\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"\u25be\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>LinearRegression()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LinearRegression</label><div class=\"sk-toggleable__content\"><pre>LinearRegression()</pre></div></div></div></div></div>"
       ],
       "text/plain": [
        "LinearRegression()"
@@ -4397,7 +4397,7 @@
     }
    ],
    "source": [
-    "plt.imshow(average_ndvi_filled.isel(time=6))"
+    "plt.imshow(average_ndvi_filled.isel(time=0))"
    ]
   },
   {
@@ -4430,7 +4430,7 @@
     }
    ],
    "source": [
-    "plt.imshow(average_ndvi.isel(time=6))"
+    "plt.imshow(average_ndvi.isel(time=0))"
    ]
   },
   {
@@ -4440,7 +4440,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "train_path = \"train/ST_training data_updated_1130points.shp\""
+    "train_path = \"train/ST_training_data_updated_1130points.shp\""
    ]
   },
   {
@@ -4476,7 +4476,7 @@
    },
    "outputs": [],
    "source": [
-    "# cấu hình nhãn dữ liệu\n",
+    "# c\u1ea5u h\u00ecnh nh\u00e3n d\u1eef li\u1ec7u\n",
     "label_mapping = {\n",
     "    \"Lua tom\": \"0\",\n",
     "    \"Lua\": \"1\",\n",
@@ -4488,7 +4488,7 @@
     "    \"Rung\": \"7\"\n",
     "}\n",
     "\n",
-    "# chia tập dữ liệu train, val, test\n",
+    "# chia t\u1eadp d\u1eef li\u1ec7u train, val, test\n",
     "X_train, X_val, X_test, y_train, y_val, y_test = split_train_data(train, label_mapping, datasets)"
    ]
   },
@@ -4510,7 +4510,7 @@
     }
    ],
    "source": [
-    "# Huấn luyện mô hình\n",
+    "# Hu\u1ea5n luy\u1ec7n m\u00f4 h\u00ecnh\n",
     "grid_search = train_with_rf(X_train, X_val, y_train, y_val)"
    ]
   },
@@ -4531,7 +4531,7 @@
     }
    ],
    "source": [
-    "# kiểm tra độ chính xác với tập test\n",
+    "# ki\u1ec3m tra \u0111\u1ed9 ch\u00ednh x\u00e1c v\u1edbi t\u1eadp test\n",
     "y_pred_test = grid_search.predict(X_test)\n",
     "test_accuracy = accuracy_score(y_test, y_pred_test)\n",
     "print(f\"Accuracy for test data {round(test_accuracy, 2)*100} %\")"
@@ -4554,7 +4554,7 @@
     }
    ],
    "source": [
-    "# Lưu mô hình huấn luyện\n",
+    "# L\u01b0u m\u00f4 h\u00ecnh hu\u1ea5n luy\u1ec7n\n",
     "save_model(\"model_new.joblib\", grid_search)"
    ]
   },
@@ -4567,7 +4567,7 @@
    },
    "outputs": [],
    "source": [
-    "# đóng client, cluster\n",
+    "# \u0111\u00f3ng client, cluster\n",
     "client.close()\n",
     "cluster.close()"
    ]
@@ -4602,4 +4602,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 5
-}
+}

processMask-test.py

@@ -0,0 +1,272 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+get_ipython().run_cell_magic('time', '', '%matplotlib inline\nfrom new_import import *\n')
+
+
+# In[2]:
+
+
+get_ipython().run_cell_magic('time', '', '# Dask gateway\ncluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))\ndc = datacube.Datacube()\n\n# Configure s3 access\nconfigure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n\nclient\n')
+
+
+# In[3]:
+
+
+## cấu hình thời gian lấy ảnh và tọa độ
+# date_range = ('2022-09-01', '2023-10-01')
+# longtitude_range = (105.86575, 105.94120)
+# latitude_range = (9.65070, 9.69850)
+date_range = ('2022-09-01', '2023-10-01')
+longtitude_range = (105.5, 106.4)
+latitude_range = (9.2, 10.0) 
+
+
+# In[4]:
+
+
+## truy vấn ảnh vệ tinh sen2
+data = load_data(dc, date_range, longtitude_range, latitude_range)
+notebook_utils.heading(notebook_utils.xarray_object_size(data))
+display(data)
+
+
+# In[5]:
+
+
+# Specify the start and end times 
+min_date = '2022-09-01' # Thời gian bắt đầu lấy data cho quá trình train
+max_date = '2023-10-01' # Thời gian kết thúc lấy data cho quá trình train
+# Just do 1 month for testing
+# max_date = '2022-10-01' # Thời gian kết thúc lấy data cho quá trình train
+
+# Specify a spatail region to search using latitude/longitude cooridinates
+min_longitude, max_longitude = (105.5, 106.4)
+min_latitude, max_latitude = (9.2, 10.0)
+
+# Specify the product. In this case we want to use Sentinel-2 Level-2A data
+product = 's2_l2a'
+
+# Construct the search query dictionary
+query = {
+    'product': product,                     # Product name
+    'x': (min_longitude, max_longitude),    # "x" axis bounds
+    'y': (min_latitude, max_latitude),      # "y" axis bounds
+    'time': (min_date, max_date),           # Any parsable date strings
+}
+
+
+# In[6]:
+
+
+# Most common CRS
+native_crs = notebook_utils.mostcommon_crs(dc, query)
+print(f'Most common native CRS: {native_crs}')
+
+
+# In[7]:
+
+
+# Specify the spectral band measurements we want to use for a classification algorithm
+measurements = ['red', 'nir', 'scl']
+
+load_params = {
+    'measurements': measurements,                   # Selected measurement or alias names
+    'output_crs': native_crs,                       # Target EPSG code
+    'resolution': (-10, 10),                        # Target resolution
+    'group_by': 'solar_day',                        # Scene grouping
+    'dask_chunks': {'x': 2048, 'y': 2048},          # Dask chunks
+}
+
+
+# In[8]:
+
+
+get_ipython().run_cell_magic('time', '', '# The replacement "dc.load()" function for this product\ndata = load_s2l2a_with_offset(\n    dc,\n    query | load_params   # Combine the two dicts that contain our search and load parameters\n)\n\n# This line prints the total size of the dataset hat was loaded\nnotebook_utils.heading(notebook_utils.xarray_object_size(data))\n\ndisplay(data)\n')
+
+
+# In[9]:
+
+
+# %%time
+# # Tiến hành loại bỏ các vị trí bị mây ảnh hưởng
+# result = mask_clean(data)
+# progress(result)
+
+
+# In[10]:
+
+
+# Tiến hành tính toán NDVI
+ds1 = calculate_indices(result, index='NDVI', satellite_mission='s2')
+ndvi = ds1["NDVI"]
+display(ndvi)
+
+
+# In[11]:
+
+
+get_ipython().run_cell_magic('time', '', "## tính ndvi theo tháng\naverage_ndvi = ndvi.resample(time='1M').mean().persist()\nprogress(average_ndvi)\n")
+
+
+# In[12]:
+
+
+# compute average_ndvi
+average_ndvi = average_ndvi.compute()
+
+
+# In[13]:
+
+
+# cấu hình vh vv file
+# name_vh = "ThuanHoa/ThuanHoa_VH.tif"
+# name_vv = "ThuanHoa/ThuanHoa_VV.tif"
+
+# load dữ liệu sen1
+bbox = [105.5, 9.2, 106.4, 10.0]
+time_range = '2022-09-01/2023-10-01'
+# dsvh, dsvv = load_sen1(bbox, time_range)
+
+name_vh = "vh-0922_0923-full_ST.tif"
+name_vv = "vv-0922_0923-full_ST.tif"
+
+if not os.path.exists(name_vh):
+    get_ipython().system('aws s3 cp s3://easi-asia-dc-data/staging/ctu/sentinel-1/vh-0922_0923-full_ST.tif vh-0922_0923-full_ST.tif')
+if not os.path.exists(name_vv):
+    get_ipython().system('aws s3 cp s3://easi-asia-dc-data/staging/ctu/sentinel-1/vv-0922_0923-full_ST.tif vv-0922_0923-full_ST.tif')
+    
+bbox = [105.5, 9.2, 106.4, 10.0]
+time_range = '2022-09-01/2023-10-01'
+dsvh, dsvv = load_sen1(bbox, time_range)
+
+
+# In[27]:
+
+
+from sklearn.preprocessing import PolynomialFeatures
+from sklearn.linear_model import LinearRegression
+from sklearn.ensemble import RandomForestRegressor
+
+
+# In[28]:
+
+
+average_ndvi = average_ndvi[:, :7680, :8687]
+mask = ~np.isnan(average_ndvi)
+print(average_ndvi.shape)
+print(dsvh.shape)
+print(dsvv.shape)
+print(mask.shape)
+X_train = np.stack([dsvh.values[mask], dsvv.values[mask]], axis=1)
+y_train = average_ndvi.values[mask]
+
+
+# In[29]:
+
+
+model = LinearRegression()
+model.fit(X_train, y_train)
+
+
+# In[30]:
+
+
+X_pred = np.stack([dsvh.values[~mask], dsvv.values[~mask]], axis=1)
+average_ndvi.values[~mask] = model.predict(X_pred)
+
+
+# In[31]:
+
+
+average_ndvi_filled = xr.DataArray(average_ndvi, dims=average_ndvi.dims)
+
+
+# In[32]:
+
+
+plt.imshow(average_ndvi_filled.isel(time=6))
+
+
+# In[65]:
+
+
+plt.imshow(average_ndvi.isel(time=6))
+
+
+# In[33]:
+
+
+train_path = "train/ST_training data_updated_1130points.shp"
+
+
+# In[34]:
+
+
+train = load_train_data(train_path)
+
+
+# In[37]:
+
+
+datasets = get_data_sen1_and_sen2(train, average_ndvi_filled, dsvh, dsvv)
+
+
+# In[39]:
+
+
+# cấu hình nhãn dữ liệu
+label_mapping = {
+    "Lua tom": "0",
+    "Lua": "1",
+    "CHN": "2",
+    "CLN": "3",
+    "TS": "4",
+    "Song": "5",
+    "Dat xay dung": "6",
+    "Rung": "7"
+}
+
+# chia tập dữ liệu train, val, test
+X_train, X_val, X_test, y_train, y_val, y_test = split_train_data(train, label_mapping, datasets)
+
+
+# In[40]:
+
+
+# Huấn luyện mô hình
+grid_search = train_with_rf(X_train, X_val, y_train, y_val)
+
+
+# In[41]:
+
+
+# kiểm tra độ chính xác với tập test
+y_pred_test = grid_search.predict(X_test)
+test_accuracy = accuracy_score(y_test, y_pred_test)
+print(f"Accuracy for test data {round(test_accuracy, 2)*100} %")
+
+
+# In[42]:
+
+
+# Lưu mô hình huấn luyện
+save_model("model_new.joblib", grid_search)
+
+
+# In[43]:
+
+
+# đóng client, cluster
+client.close()
+cluster.close()
+
+
+# In[ ]:
+
+
+
+

processMask.ipynb

@@ -1132,7 +1132,7 @@
    "source": [
     "%%time\n",
     "%matplotlib inline\n",
-    "from new_import import *"
+    "from new_import_ODC import *"
    ]
   },
   {
@@ -1216,7 +1216,7 @@
     "%%time\n",
     "# Dask gateway\n",
     "cluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))\n",
-    "dc = datacube.Datacube()\n",
+    "dc = None\n",
     "\n",
     "# Configure s3 access\n",
     "configure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n",
@@ -1233,8 +1233,8 @@
    },
    "outputs": [],
    "source": [
-    "## cấu hình thời gian lấy ảnh và tọa độ\n",
+    "## c\u1ea5u h\u00ecnh th\u1eddi gian l\u1ea5y \u1ea3nh v\u00e0 t\u1ecda \u0111\u1ed9\n",
-    "date_range = ('2022-09-01', '2023-10-01')\n",
+    "date_range = ('2022-09-01', '2022-10-01')\n",
     "longtitude_range = (105.86575, 105.94120)\n",
     "latitude_range = (9.65070, 9.69850)"
    ]
@@ -1393,7 +1393,7 @@
        "\n",
        ".xr-section-summary-in + label:before {\n",
        "  display: inline-block;\n",
-       "  content: '►';\n",
+       "  content: '\u25ba';\n",
        "  font-size: 11px;\n",
        "  width: 15px;\n",
        "  text-align: center;\n",
@@ -1404,7 +1404,7 @@
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label:before {\n",
-       "  content: '▼';\n",
+       "  content: '\u25bc';\n",
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label > span {\n",
@@ -2333,8 +2333,8 @@
     }
    ],
    "source": [
-    "## truy vấn ảnh vệ tinh sen2\n",
+    "## truy v\u1ea5n \u1ea3nh v\u1ec7 tinh sen2\n",
-    "data = load_data(dc, date_range, longtitude_range, latitude_range)\n",
+    "data = load_data(None, date_range, longtitude_range, latitude_range)\n",
     "notebook_utils.heading(notebook_utils.xarray_object_size(data))\n",
     "display(data)"
    ]
@@ -2443,9 +2443,9 @@
    ],
    "source": [
     "%%time\n",
-    "# Tiến hành loại bỏ các vị trí bị mây ảnh hưởng\n",
+    "# Ti\u1ebfn h\u00e0nh lo\u1ea1i b\u1ecf c\u00e1c v\u1ecb tr\u00ed b\u1ecb m\u00e2y \u1ea3nh h\u01b0\u1edfng\n",
     "result = mask_clean(data)\n",
-    "progress(result)"
+    "# progress(result)"
    ]
   },
   {
@@ -2581,7 +2581,7 @@
        "\n",
        ".xr-section-summary-in + label:before {\n",
        "  display: inline-block;\n",
-       "  content: '►';\n",
+       "  content: '\u25ba';\n",
        "  font-size: 11px;\n",
        "  width: 15px;\n",
        "  text-align: center;\n",
@@ -2592,7 +2592,7 @@
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label:before {\n",
-       "  content: '▼';\n",
+       "  content: '\u25bc';\n",
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label > span {\n",
@@ -3047,7 +3047,7 @@
     }
    ],
    "source": [
-    "# Tiến hành tính toán NDVI\n",
+    "# Ti\u1ebfn h\u00e0nh t\u00ednh to\u00e1n NDVI\n",
     "ds1 = calculate_indices(result, index='NDVI', satellite_mission='s2')\n",
     "ndvi = ds1[\"NDVI\"]\n",
     "display(ndvi)"
@@ -3086,9 +3086,9 @@
    ],
    "source": [
     "%%time\n",
-    "## tính ndvi theo tháng\n",
+    "## t\u00ednh ndvi theo th\u00e1ng\n",
     "average_ndvi = ndvi.resample(time='1M').mean().persist()\n",
-    "progress(average_ndvi)"
+    "# progress(average_ndvi)"
    ]
   },
   {
@@ -3113,12 +3113,12 @@
    },
    "outputs": [],
    "source": [
-    "# cấu hình vh vv file\n",
+    "# c\u1ea5u h\u00ecnh vh vv file\n",
     "name_vh = \"ThuanHoa/ThuanHoa_VH.tif\"\n",
     "name_vv = \"ThuanHoa/ThuanHoa_VV.tif\"\n",
     "\n",
-    "# load dữ liệu sen1\n",
+    "# load d\u1eef li\u1ec7u sen1\n",
-    "dsvh, dsvv = load_sen1(name_vh, name_vv)"
+    "bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]\ntime_range = f'{date_range[0]}/{date_range[1]}'\ndsvh, dsvv = load_sen1(bbox, time_range)"
    ]
   },
   {
@@ -3160,7 +3160,7 @@
     {
      "data": {
       "text/html": [
-       "<style>#sk-container-id-1 {color: black;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>LinearRegression()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LinearRegression</label><div class=\"sk-toggleable__content\"><pre>LinearRegression()</pre></div></div></div></div></div>"
+       "<style>#sk-container-id-1 {color: black;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"\u25b8\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"\u25be\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>LinearRegression()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LinearRegression</label><div class=\"sk-toggleable__content\"><pre>LinearRegression()</pre></div></div></div></div></div>"
       ],
       "text/plain": [
        "LinearRegression()"
@@ -3297,4 +3297,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 5
-}
+}

processMask.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+get_ipython().run_cell_magic('time', '', '%matplotlib inline\nfrom new_import import *\n')
+
+
+# In[2]:
+
+
+get_ipython().run_cell_magic('time', '', '# Dask gateway\ncluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))\ndc = datacube.Datacube()\n\n# Configure s3 access\nconfigure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n\nclient\n')
+
+
+# In[3]:
+
+
+## cấu hình thời gian lấy ảnh và tọa độ
+date_range = ('2022-09-01', '2023-10-01')
+longtitude_range = (105.86575, 105.94120)
+latitude_range = (9.65070, 9.69850)
+
+
+# In[4]:
+
+
+## truy vấn ảnh vệ tinh sen2
+data = load_data(dc, date_range, longtitude_range, latitude_range)
+notebook_utils.heading(notebook_utils.xarray_object_size(data))
+display(data)
+
+
+# In[5]:
+
+
+get_ipython().run_cell_magic('time', '', '# Tiến hành loại bỏ các vị trí bị mây ảnh hưởng\nresult = mask_clean(data)\nprogress(result)\n')
+
+
+# In[6]:
+
+
+# Tiến hành tính toán NDVI
+ds1 = calculate_indices(result, index='NDVI', satellite_mission='s2')
+ndvi = ds1["NDVI"]
+display(ndvi)
+
+
+# In[17]:
+
+
+get_ipython().run_cell_magic('time', '', "## tính ndvi theo tháng\naverage_ndvi = ndvi.resample(time='1M').mean().persist()\nprogress(average_ndvi)\n")
+
+
+# In[18]:
+
+
+# compute average_ndvi
+average_ndvi = average_ndvi.compute()
+
+
+# In[9]:
+
+
+# cấu hình vh vv file
+name_vh = "ThuanHoa/ThuanHoa_VH.tif"
+name_vv = "ThuanHoa/ThuanHoa_VV.tif"
+
+# load dữ liệu sen1
+bbox = [105.5, 9.2, 106.4, 10.0]
+time_range = '2022-09-01/2023-10-01'
+dsvh, dsvv = load_sen1(bbox, time_range)
+
+
+# In[10]:
+
+
+from sklearn.preprocessing import PolynomialFeatures
+from sklearn.linear_model import LinearRegression
+from sklearn.ensemble import RandomForestRegressor
+
+
+# In[11]:
+
+
+mask = ~np.isnan(average_ndvi)
+X_train = np.stack([dsvh.values[mask], dsvv.values[mask]], axis=1)
+y_train = average_ndvi.values[mask]
+
+
+# In[12]:
+
+
+model = LinearRegression()
+model.fit(X_train, y_train)
+
+
+# In[13]:
+
+
+X_pred = np.stack([dsvh.values[~mask], dsvv.values[~mask]], axis=1)
+average_ndvi.values[~mask] = model.predict(X_pred)
+
+
+# In[14]:
+
+
+average_ndvi_filled = xr.DataArray(average_ndvi, dims=average_ndvi.dims)
+
+
+# In[16]:
+
+
+plt.imshow(average_ndvi_filled.isel(time=1))
+
+
+# In[19]:
+
+
+plt.imshow(average_ndvi.isel(time=1))
+
+
+# In[ ]:
+
+
+
+

remove_datacube.py

@@ -0,0 +1,35 @@
+import json
+import glob
+
+def fix_notebook(file_path):
+    with open(file_path, 'r', encoding='utf-8') as f:
+        nb = json.load(f)
+    changed = False
+    for cell in nb.get('cells', []):
+        if cell.get('cell_type') == 'code':
+            source = cell.get('source', [])
+            if isinstance(source, list):
+                for i, line in enumerate(source):
+                    if "dc = datacube.Datacube()" in line:
+                        source[i] = "dc = None\n"
+                        changed = True
+                    if "ds = dc.load(" in line:
+                        source[i] = "ds = None\n"
+                        changed = True
+                    if "data = dc.load(" in line:
+                        source[i] = "data = None\n"
+                        changed = True
+                    # If ds is None, ds.vv will fail
+                    if "vv_data = ds.vv" in line:
+                        source[i] = "vv_data = None\n"
+                        changed = True
+                    if "notebook_utils.xarray_object_size(ds)" in line:
+                        source[i] = line.replace("notebook_utils.xarray_object_size(ds)", "'ds is None'")
+                        changed = True
+    if changed:
+        with open(file_path, 'w', encoding='utf-8') as f:
+            json.dump(nb, f, indent=1)
+        print(f"Removed datacube from {file_path}")
+
+for nb in glob.glob("*.ipynb"):
+    fix_notebook(nb)

run_all_parallel.py

@@ -0,0 +1,82 @@
+import json
+import glob
+import subprocess
+import time
+import os
+
+NOTEBOOKS_TO_RUN = [
+    "01.train_ODC.ipynb",
+    "01.train_ODC_XGBoost.ipynb",
+    "02.predict_ODC.ipynb",
+    "new_train.ipynb"
+]
+
+def limit_time_range(file_path):
+    try:
+        with open(file_path, 'r', encoding='utf-8') as f:
+            nb = json.load(f)
+        
+        changed = False
+        for cell in nb.get('cells', []):
+            if cell.get('cell_type') == 'code':
+                source = cell.get('source', [])
+                if isinstance(source, list):
+                    for i, line in enumerate(source):
+                        # Replace 2023-12-31 with 2023-04-01
+                        if '"2023-12-31"' in line:
+                            source[i] = line.replace('"2023-12-31"', '"2023-04-01"')
+                            changed = True
+                        if "'2023-10-01'" in line:
+                            source[i] = line.replace("'2023-10-01'", "'2022-10-01'")
+                            changed = True
+                        if '"2023-10-01"' in line:
+                            source[i] = line.replace('"2023-10-01"', '"2022-10-01"')
+                            changed = True
+                        # For time_range="2022-09-01/2023-10-01"
+                        if "2022-09-01/2023-10-01" in line:
+                            source[i] = line.replace("2022-09-01/2023-10-01", "2022-09-01/2022-10-01")
+                            changed = True
+                            
+                elif isinstance(source, str):
+                    new_source = source.replace('"2023-12-31"', '"2023-04-01"')
+                    new_source = new_source.replace("'2023-10-01'", "'2022-10-01'")
+                    new_source = new_source.replace('"2023-10-01"', '"2022-10-01"')
+                    new_source = new_source.replace("2022-09-01/2023-10-01", "2022-09-01/2022-10-01")
+                    if new_source != source:
+                        cell['source'] = new_source
+                        changed = True
+        
+        if changed:
+            with open(file_path, 'w', encoding='utf-8') as f:
+                json.dump(nb, f, indent=1)
+            print(f"Limited time_range to 1 month in {file_path}")
+    except Exception as e:
+        print(f"Error on {file_path}: {e}")
+
+# 1. Modify the time ranges
+for nb_file in glob.glob("*.ipynb"):
+    limit_time_range(nb_file)
+
+# 2. Run them in parallel
+print("\nStarting parallel execution of notebooks...")
+processes = []
+for nb_file in NOTEBOOKS_TO_RUN:
+    if os.path.exists(nb_file):
+        print(f"Launching {nb_file}...")
+        cmd = f"source /home/x79/miniconda3/etc/profile.d/conda.sh && conda activate env_01 && jupyter nbconvert --execute --ExecutePreprocessor.timeout=-1 --inplace {nb_file}"
+        p = subprocess.Popen(["bash", "-c", cmd], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+        processes.append((nb_file, p))
+
+# 3. Wait and print output
+for nb_file, p in processes:
+    p.wait()
+    output = p.stdout.read().decode('utf-8')
+    if p.returncode == 0:
+        print(f"[{nb_file}] SUCCESS")
+    else:
+        print(f"[{nb_file}] FAILED (code {p.returncode})")
+        print(f"--- OUTPUT START ({nb_file}) ---")
+        print(output)
+        print(f"--- OUTPUT END ({nb_file}) ---")
+
+print("\nAll tasks finished.")

run_pipeline.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+source /home/x79/miniconda3/etc/profile.d/conda.sh
+conda activate env_01
+set -e
+
+# Configure GDAL for vsicurl stability
+export GDAL_HTTP_MAX_RETRY=5
+export GDAL_HTTP_RETRY_DELAY=2
+export GDAL_HTTP_CONNECTION_TIMEOUT=10
+export GDAL_HTTP_TIMEOUT=30
+export CPL_VSIL_CURL_ALLOWED_EXTENSIONS=.tif,.tiff
+export GDAL_DISABLE_READDIR_ON_OPEN=YES
+
+
+echo "=== [1/4] Running RF Training ==="
+jupyter nbconvert --execute --ExecutePreprocessor.timeout=-1 --inplace 01.train_ODC.ipynb
+
+echo "=== [2/4] Running XGBoost Training ==="
+jupyter nbconvert --execute --ExecutePreprocessor.timeout=-1 --inplace 01.train_ODC_XGBoost.ipynb
+
+echo "=== [3/4] Running Prediction ==="
+jupyter nbconvert --execute --ExecutePreprocessor.timeout=-1 --inplace 02.predict_ODC.ipynb
+
+echo "=== [4/4] Running New Train ==="
+jupyter nbconvert --execute --ExecutePreprocessor.timeout=-1 --inplace new_train.ipynb
+
+echo "=== ALL DONE SUCCESSFULLY ==="

test_VH_VV.ipynb

@@ -1141,12 +1141,12 @@
    "source": [
     "\n",
     "%matplotlib inline\n",
-    "from new_import import *\n",
+    "from new_import_ODC import *\n",
     "\n",
     "\n",
     "# Dask gateway\n",
     "cluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))\n",
-    "dc = datacube.Datacube()\n",
+    "dc = None\n",
     "\n",
     "\n",
     "# Configure s3 access\n",
@@ -1300,7 +1300,7 @@
        "\n",
        ".xr-section-summary-in + label:before {\n",
        "  display: inline-block;\n",
-       "  content: '►';\n",
+       "  content: '\u25ba';\n",
        "  font-size: 11px;\n",
        "  width: 15px;\n",
        "  text-align: center;\n",
@@ -1311,7 +1311,7 @@
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label:before {\n",
-       "  content: '▼';\n",
+       "  content: '\u25bc';\n",
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label > span {\n",
@@ -1880,20 +1880,20 @@
     }
    ],
    "source": [
-    "ds = dc.load(\n",
+    "# ds = dc.load(\n",
-    "    product=\"sentinel1_grd_gamma0_20m\",\n",
+    "#     product=\"sentinel1_grd_gamma0_20m\",\n",
-    "    x=(105.5, 106.4),\n",
+    "#     x=(105.5, 106.4),\n",
-    "    y=(9.2, 10.0),\n",
+    "#     y=(9.2, 10.0),\n",
-    "    time=(\"2022-09-01\", \"2023-10-01\"),\n",
+    "#     time=(\"2022-09-01\", \"2022-10-01\"),\n",
-    "    measurements=[\"vv\", \"vh\"],\n",
+    "#     measurements=[\"vv\", \"vh\"],\n",
-    "    output_crs=\"EPSG:32648\",\n",
+    "#     output_crs=\"EPSG:32648\",\n",
-    "    resolution=(-10,10),\n",
+    "#     resolution=(-10,10),\n",
-    "    dask_chunks={\"x\":2048, \"y\":2048},\n",
+    "#     dask_chunks={\"x\":2048, \"y\":2048},\n",
-    "    skip_broken_datasets=True,\n",
+    "#     skip_broken_datasets=True,\n",
-    "    group_by=\"solar_day\"\n",
+    "#     group_by=\"solar_day\"\n",
-    ")\n",
+    "# )\n",
-    "notebook_utils.heading(notebook_utils.xarray_object_size(ds))\n",
+    "# notebook_utils.heading(notebook_utils.xarray_object_size(ds))\n",
-    "ds"
+    "# ds"
    ]
   },
   {
@@ -1905,11 +1905,11 @@
    },
    "outputs": [],
    "source": [
-    "vh = ds.vh.resample(time='1M').mean().persist()\n",
+    "# vh = ds.vh.resample(time='1M').mean().persist()\n",
-    "vh = vh.compute()\n",
+    "# vh = vh.compute()\n",
-    "vv = ds.vv.resample(time='1M').mean().persist()\n",
+    "# vv = ds.vv.resample(time='1M').mean().persist()\n",
-    "vv = vv.compute()\n",
+    "# vv = vv.compute()\n",
-    "\n"
+    "# \n"
    ]
   },
   {
@@ -2045,7 +2045,7 @@
        "\n",
        ".xr-section-summary-in + label:before {\n",
        "  display: inline-block;\n",
-       "  content: '►';\n",
+       "  content: '\u25ba';\n",
        "  font-size: 11px;\n",
        "  width: 15px;\n",
        "  text-align: center;\n",
@@ -2056,7 +2056,7 @@
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label:before {\n",
-       "  content: '▼';\n",
+       "  content: '\u25bc';\n",
        "}\n",
        "\n",
        ".xr-section-summary-in:checked + label > span {\n",
@@ -2365,4 +2365,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 5
-}
+}

test_VH_VV.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[2]:
+
+
+get_ipython().run_line_magic('matplotlib', 'inline')
+from new_import import *
+
+
+# Dask gateway
+cluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))
+dc = datacube.Datacube()
+
+
+# Configure s3 access
+configure_s3_access(aws_unsigned=False, requester_pays=True, client=client)
+
+
+
+
+# In[3]:
+
+
+ds = dc.load(
+    product="sentinel1_grd_gamma0_20m",
+    x=(105.5, 106.4),
+    y=(9.2, 10.0),
+    time=("2022-09-01", "2023-10-01"),
+    measurements=["vv", "vh"],
+    output_crs="EPSG:32648",
+    resolution=(-10,10),
+    dask_chunks={"x":2048, "y":2048},
+    skip_broken_datasets=True,
+    group_by="solar_day"
+)
+notebook_utils.heading(notebook_utils.xarray_object_size(ds))
+ds
+
+
+# In[18]:
+
+
+vh = ds.vh.resample(time='1M').mean().persist()
+vh = vh.compute()
+vv = ds.vv.resample(time='1M').mean().persist()
+vv = vv.compute()
+
+
+
+# In[28]:
+
+
+vv.min()
+
+
+# In[33]:
+
+
+import matplotlib.pyplot as plt
+
+# Plot the data
+plt.imshow(vh.isel(time=0), cmap='viridis', vmin=0, vmax=1)
+plt.colorbar()  # Add colorbar for reference
+plt.show()
+
+
+# In[ ]:
+
+
+
+

test_planetary.py

@@ -0,0 +1,15 @@
+import sys
+# Thêm đường dẫn hiện tại vào PYTHONPATH để import được new_import_ODC nếu cần
+sys.path.append('.')
+import warnings
+warnings.filterwarnings('ignore')
+from new_import_ODC import load_sen1
+
+print("Testing load_sen1 with a short time range to speed up Dask compute...")
+bbox = [105.5, 9.2, 106.4, 10.0]
+time_range = "2023-01-01/2023-01-31" # Short time range for fast testing
+vh, vv = load_sen1(bbox, time_range)
+print("VH shape:", vh.shape)
+print("VV shape:", vv.shape)
+print("VH CRS:", vh.rio.crs)
+print("Success!")

test_planetary2.py

@@ -0,0 +1,49 @@
+import warnings
+warnings.filterwarnings('ignore')
+
+def load_sen1(bbox, time_range):
+    import pystac_client
+    import planetary_computer
+    import odc.stac
+    
+    catalog = pystac_client.Client.open(
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
+        modifier=planetary_computer.sign_inplace,
+    )
+
+    search = catalog.search(
+        collections=["sentinel-1-rtc"],
+        bbox=bbox,
+        datetime=time_range,
+    )
+    items = list(search.items())
+    print("Found items:", len(items))
+
+    ds_s1 = odc.stac.load(
+        items,
+        bands=["vv", "vh"],
+        bbox=bbox,
+        crs="EPSG:32648",
+        resolution=10,
+        chunks={"x": 2048, "y": 2048, "time": 1}
+    )
+
+    ds_median = ds_s1.median(dim="time").compute()
+    vv = ds_median["vv"]
+    vh = ds_median["vh"]
+
+    vv = vv.expand_dims(dim="band")
+    vh = vh.expand_dims(dim="band")
+
+    vv = vv.rio.write_crs("EPSG:32648")
+    vh = vh.rio.write_crs("EPSG:32648")
+
+    return vh, vv
+
+print("Testing load_sen1...")
+bbox = [105.5, 9.2, 106.4, 10.0]
+time_range = "2022-09-01/2023-10-01"
+vh, vv = load_sen1(bbox, time_range)
+print("VH shape:", vh.shape)
+print("VV shape:", vv.shape)
+print("Success!")