Cập nhật mã nguồn và file Colab Cache

new_import_ODC.py

@@ -57,13 +57,12 @@ import rioxarray
 hv.extension('bokeh', logo=False)
 
 from deafrica_tools.bandindices import calculate_indices
-from sklearn.ensemble import RandomForestClassifier
+from xgboost import XGBClassifier
 from sklearn.model_selection import train_test_split
 from sklearn.metrics import accuracy_score, classification_report
 from sklearn.preprocessing import LabelEncoder
 
 from sklearn.pipeline import Pipeline
-from sklearn.ensemble import RandomForestClassifier
 from sklearn.impute import SimpleImputer
 from sklearn.preprocessing import StandardScaler
 from sklearn.model_selection import GridSearchCV
@@ -88,6 +87,17 @@ import joblib
 
 
 def load_data(dc, date_range, longtitude_range, latitude_range):
+    import os, hashlib
+    cache_dir = "dataset_cache"
+    os.makedirs(cache_dir, exist_ok=True)
+    key_str = f"s2_{date_range}_{longtitude_range}_{latitude_range}"
+    cache_key = hashlib.md5(key_str.encode()).hexdigest() + ".nc"
+    cache_path = os.path.join(cache_dir, cache_key)
+    
+    if os.path.exists(cache_path):
+        print(f"✅ Loading cached S2 data from {cache_path}")
+        return xr.open_dataset(cache_path, engine='netcdf4')
+
     product = 's2_l2a'
     bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
     
@@ -117,6 +127,11 @@ def load_data(dc, date_range, longtitude_range, latitude_range):
     )
     if "SCL" in data.data_vars:
         data = data.rename({"SCL": "scl"})
+        
+    print(f"💾 Caching S2 data to {cache_path}")
+    data = data.compute()
+    data.to_netcdf(cache_path, engine='netcdf4')
+    
     return data
 
 
@@ -168,6 +183,21 @@ def load_train_data(train_path):
 
 
 def load_sen1(bbox, time_range):
+    import os, hashlib
+    cache_dir = "dataset_cache"
+    os.makedirs(cache_dir, exist_ok=True)
+    key_str = f"s1_vh_vv_{bbox}_{time_range}"
+    cache_key_vh = hashlib.md5((key_str + "vh").encode()).hexdigest() + ".nc"
+    cache_key_vv = hashlib.md5((key_str + "vv").encode()).hexdigest() + ".nc"
+    cache_path_vh = os.path.join(cache_dir, cache_key_vh)
+    cache_path_vv = os.path.join(cache_dir, cache_key_vv)
+    
+    if os.path.exists(cache_path_vh) and os.path.exists(cache_path_vv):
+        print(f"✅ Loading cached S1 data from {cache_path_vh} and {cache_path_vv}")
+        ds_vh = xr.open_dataset(cache_path_vh, engine='netcdf4')
+        ds_vv = xr.open_dataset(cache_path_vv, engine='netcdf4')
+        return ds_vh[list(ds_vh.data_vars)[0]], ds_vv[list(ds_vv.data_vars)[0]]
+
     import pystac_client
     import planetary_computer
     import odc.stac
@@ -209,6 +239,10 @@ def load_sen1(bbox, time_range):
     vv = vv.rio.write_crs("EPSG:32648")
     vh = vh.rio.write_crs("EPSG:32648")
 
+    print(f"💾 Caching S1 data to {cache_dir}")
+    vh.to_netcdf(cache_path_vh, engine='netcdf4')
+    vv.to_netcdf(cache_path_vv, engine='netcdf4')
+
     return vh, vv
 
 
@@ -254,7 +288,7 @@ def train_with_rf(X_train, X_val, y_train, y_val):
     # Takes 1-2 minutes to complete
 
     # Tạo RandomForestClassifier mặc định để sử dụng làm mô hình ban đầu trong pipeline
-    base_model = RandomForestClassifier(random_state=42, n_jobs=-1)
+    base_model = XGBClassifier(tree_method="hist", device="cuda", random_state=42, n_jobs=-1)
 
     # Tạo pipeline
     pipeline = Pipeline([
@@ -266,7 +300,7 @@ def train_with_rf(X_train, X_val, y_train, y_val):
     param_grid = {
         'classifier__n_estimators': [100, 300, 500, 700, 1000],
         'classifier__max_depth': [6, 8, 10, 15, 20],
-        'classifier__criterion': ['gini', 'entropy'],
+        'classifier__learning_rate': [0.01, 0.1, 0.2],
     }
 
     # Sử dụng GridSearchCV để tìm bộ tham số tốt nhất
@@ -426,9 +460,26 @@ def save_result(result, HT_MAP):
         # plt.show()
         
 def load_data_sen1(dc, date_range, coordinates):
+    import os, hashlib
     longtitude_range, latitude_range = coordinates
     bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
     
+    cache_dir = "dataset_cache"
+    os.makedirs(cache_dir, exist_ok=True)
+    key_str = f"data_sen1_{date_range}_{bbox}"
+    cache_key_vh = hashlib.md5((key_str + "vh").encode()).hexdigest() + ".nc"
+    cache_key_vv = hashlib.md5((key_str + "vv").encode()).hexdigest() + ".nc"
+    cache_path_vh = os.path.join(cache_dir, cache_key_vh)
+    cache_path_vv = os.path.join(cache_dir, cache_key_vv)
+    
+    if os.path.exists(cache_path_vh) and os.path.exists(cache_path_vv):
+        print(f"✅ Loading cached S1 (coord) data")
+        ds_vh = xr.open_dataset(cache_path_vh, engine='netcdf4')
+        ds_vv = xr.open_dataset(cache_path_vv, engine='netcdf4')
+        var_vh = [v for v in ds_vh.data_vars if v != 'spatial_ref'][0]
+        var_vv = [v for v in ds_vv.data_vars if v != 'spatial_ref'][0]
+        return ds_vh[var_vh], ds_vv[var_vv]
+    
     import pystac_client
     import planetary_computer
     import odc.stac
@@ -454,11 +505,14 @@ def load_data_sen1(dc, date_range, coordinates):
         groupby="solar_day"
     )
     
-    # notebook_utils.heading(notebook_utils.xarray_object_size(data_sen1))
-    # display(data_sen1)
+    data_sen1 = data_sen1.compute()
     dsvh = data_sen1.vh
     dsvv = data_sen1.vv
     
+    print(f"💾 Caching S1 (coord) data")
+    dsvh.to_netcdf(cache_path_vh, engine='netcdf4')
+    dsvv.to_netcdf(cache_path_vv, engine='netcdf4')
+    
     return dsvh, dsvv
     
 def calculate_average(data, time_pattern='1M'):
@@ -466,9 +520,20 @@ def calculate_average(data, time_pattern='1M'):
 
 
 def load_data_sen2(dc, date_range, coordinates):
+    import os, hashlib
     longtitude_range, latitude_range = coordinates
     bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
     
+    cache_dir = "dataset_cache"
+    os.makedirs(cache_dir, exist_ok=True)
+    key_str = f"data_sen2_{date_range}_{bbox}"
+    cache_key = hashlib.md5(key_str.encode()).hexdigest() + ".nc"
+    cache_path = os.path.join(cache_dir, cache_key)
+    
+    if os.path.exists(cache_path):
+        print(f"✅ Loading cached S2 (coord) data from {cache_path}")
+        return xr.open_dataset(cache_path, engine='netcdf4')
+    
     import pystac_client
     import planetary_computer
     import odc.stac
@@ -495,6 +560,11 @@ def load_data_sen2(dc, date_range, coordinates):
     )
     if "SCL" in data.data_vars:
         data = data.rename({"SCL": "scl"})
+        
+    data = data.compute()
+    print(f"💾 Caching S2 (coord) data to {cache_path}")
+    data.to_netcdf(cache_path, engine='netcdf4')
+    
     return data
 
 def mask_cloud(data):
@@ -509,7 +579,7 @@ def mask_cloud(data):
 def find_best_model(dataset):
     X_train, X_val, y_train, y_val = dataset
     # Tạo RandomForestClassifier mặc định để sử dụng làm mô hình ban đầu trong pipeline
-    base_model = RandomForestClassifier(random_state=42, n_jobs=-1)
+    base_model = XGBClassifier(tree_method="hist", device="cuda", random_state=42, n_jobs=-1)
 
     # Tạo pipeline
     pipeline = Pipeline([
@@ -521,7 +591,7 @@ def find_best_model(dataset):
     param_grid = {
         'classifier__n_estimators': [100, 300, 500, 700, 1000],
         'classifier__max_depth': [6, 8, 10, 15, 20],
-        'classifier__criterion': ['gini', 'entropy'],
+        'classifier__learning_rate': [0.01, 0.1, 0.2],
     }
 
     # Sử dụng GridSearchCV để tìm bộ tham số tốt nhất