Migrate all ODC models and prediction pipeline to Microsoft Planetary Computer
This commit is contained in:
+4100
-805
File diff suppressed because one or more lines are too long
+233
File diff suppressed because one or more lines are too long
+4195
-1541
File diff suppressed because one or more lines are too long
@@ -0,0 +1,171 @@
|
||||
#!/usr/bin/env python
|
||||
# coding: utf-8
|
||||
|
||||
# In[1]:
|
||||
|
||||
|
||||
get_ipython().run_cell_magic('time', '', '%matplotlib inline\n\nimport importlib\nimport new_import_ODC \n\nimportlib.reload(new_import_ODC)\n\nfrom new_import_ODC import *\n')
|
||||
|
||||
|
||||
# In[2]:
|
||||
|
||||
|
||||
get_ipython().run_cell_magic('time', '', '# Cấu hình Daskgateway\ncluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1, 10))\n# Khai báo 1 Datacube là dc\ndc = None\n\n# Cấu hình truy cập dịch vụ S3\nconfigure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n\nclient\n')
|
||||
|
||||
|
||||
# In[3]:
|
||||
|
||||
|
||||
## cấu hình thời gian lấy ảnh và tọa độ
|
||||
date_range = ("2022-09-01", "2022-10-01")
|
||||
longtitude_range = (105.86, 105.94)
|
||||
latitude_range = (9.65, 9.69)
|
||||
|
||||
coordinates = (longtitude_range, latitude_range)
|
||||
|
||||
|
||||
# In[4]:
|
||||
|
||||
|
||||
## truy vấn ảnh vệ tinh sen2
|
||||
data = load_data(None, date_range, longtitude_range, latitude_range)
|
||||
notebook_utils.heading(notebook_utils.xarray_object_size(data))
|
||||
display(data)
|
||||
|
||||
|
||||
# In[5]:
|
||||
|
||||
|
||||
get_ipython().run_cell_magic('time', '', '# Tiến hành loại bỏ các vị trí bị mây ảnh hưởng\nresult = mask_clean(data)\n# progress(result)\n')
|
||||
|
||||
|
||||
# In[6]:
|
||||
|
||||
|
||||
# Tiến hành tính toán NDVI
|
||||
ds1 = calculate_indices(result, index="NDVI", satellite_mission="s2")
|
||||
ndvi = ds1["NDVI"]
|
||||
display(ndvi)
|
||||
|
||||
|
||||
# In[7]:
|
||||
|
||||
|
||||
## Hiển thị ảnh NDVI chưa điền các giá trị mây (chưa fill nan)
|
||||
plt.imshow(ndvi.isel(time=0))
|
||||
|
||||
|
||||
# In[8]:
|
||||
|
||||
|
||||
# Thiết lập giá trị trung bình mùa vụ để xử lý các điểm ảnh bị mây dựa vào sự thay đổi theo mùa
|
||||
time_split = [
|
||||
slice("2022-09-01", "2023-01-01"),
|
||||
slice("2023-01-01", "2023-05-01"),
|
||||
slice("2023-05-01", "2023-07-01"),
|
||||
slice("2023-07-01", "2022-10-01"),
|
||||
]
|
||||
|
||||
# Điền mây ở các vị trí mang giá trị nan (fill nan)
|
||||
fill_nan_ndvi = fill_nan(ndvi, time_split)
|
||||
|
||||
# In kết quả ảnh NDVI đã điền mây (đã fill nan)
|
||||
plt.imshow(fill_nan_ndvi.isel(time=0))
|
||||
|
||||
|
||||
# In[9]:
|
||||
|
||||
|
||||
get_ipython().run_cell_magic('time', '', '## tính ndvi theo tháng\naverage_ndvi = fill_nan_ndvi.resample(time="1M").mean().persist()\n# progress(average_ndvi)\n\n# compute average_ndvi\naverage_ndvi = average_ndvi.compute()\n')
|
||||
|
||||
|
||||
# In[10]:
|
||||
|
||||
|
||||
#Load dữ liệu ảnh Sentinel 1
|
||||
dsvh, dsvv = load_data_sen1(None, date_range, coordinates)
|
||||
average_vv = calculate_average(dsvv, time_pattern='1M')
|
||||
average_vh = calculate_average(dsvh, time_pattern='1M')
|
||||
|
||||
|
||||
# In[11]:
|
||||
|
||||
|
||||
## cấu hình bộ dữ liệu điểm huấn luyện mô hình (train file)
|
||||
train_path = "train/ST_training_data_updated_1130points_new.shp" # đường dẫn shp file train
|
||||
|
||||
## load dữ liệu điểm huấn luyện mô hình (train file)
|
||||
train = load_train_data(train_path)
|
||||
train.head()
|
||||
|
||||
# cấu hình nhãn dữ liệu
|
||||
label_mapping = {
|
||||
"Lua tom": "0",
|
||||
"Lua": "1",
|
||||
"CHN": "2",
|
||||
"CLN": "3",
|
||||
"TS": "4",
|
||||
"Song": "5",
|
||||
"Dat xay dung": "6",
|
||||
"Rung": "7",
|
||||
}
|
||||
|
||||
# xây dựng tập dữ liệu (dataset) chứa dữ liệu VH, VV, NDVI
|
||||
datasets = get_data_sen1_and_sen2(train, average_ndvi, average_vh, average_vv)
|
||||
|
||||
# chia tập dữ liệu thành các phần theo tỉ lệ 80(80-20)-20 tương ứng với tập train, validate, test
|
||||
X_train, X_val, X_test, y_train, y_val, y_test = split_train_data(
|
||||
train, label_mapping, datasets
|
||||
)
|
||||
|
||||
|
||||
# In[ ]:
|
||||
|
||||
|
||||
get_ipython().run_cell_magic('time', '', '# Import XGBoost\nimport xgboost as xgb\nfrom sklearn.metrics import accuracy_score\nimport numpy as np\n\n# Convert to numpy arrays\nX_train_np = np.asarray(X_train, dtype=np.float32)\nX_val_np = np.asarray(X_val, dtype=np.float32)\ny_train_np = np.asarray(y_train, dtype=np.int32)\ny_val_np = np.asarray(y_val, dtype=np.int32)\n\nprint("🚀 Training XGBoost model...")\nprint(f" Train samples: {len(X_train_np)}")\nprint(f" Val samples: {len(X_val_np)}")\nprint(f" Features: {X_train_np.shape[1]}")\nprint(f" Classes: 8\\n")\n\n# XGBoost parameters\nparams = {\n \'objective\': \'multi:softmax\', # Multi-class classification\n \'num_class\': 8, # 8 land use classes\n \'max_depth\': 6, # Maximum tree depth\n \'learning_rate\': 0.1, # Learning rate\n \'n_estimators\': 200, # Number of trees\n \'subsample\': 0.8, # Subsample ratio\n \'colsample_bytree\': 0.8, # Feature sampling ratio\n \'random_state\': 42,\n \'n_jobs\': -1, # Use all CPU cores\n \'eval_metric\': \'mlogloss\' # Multi-class log loss\n}\n\n# Train XGBoost model\nmodel = xgb.XGBClassifier(**params)\n\nmodel.fit(\n X_train_np, y_train_np,\n eval_set=[(X_train_np, y_train_np), (X_val_np, y_val_np)],\n verbose=True\n)\n\n# Validation accuracy\ny_val_pred = model.predict(X_val_np)\nval_accuracy = accuracy_score(y_val_np, y_val_pred)\nprint(f"\\n✅ Training completed!")\nprint(f" Validation Accuracy: {val_accuracy:.4f} ({val_accuracy*100:.2f}%)")\n')
|
||||
|
||||
|
||||
# In[ ]:
|
||||
|
||||
|
||||
get_ipython().run_cell_magic('time', '', '# Evaluate on test set\nX_test_np = np.asarray(X_test, dtype=np.float32)\ny_test_np = np.asarray(y_test, dtype=np.int32)\n\nprint("📊 Evaluating XGBoost model on test set...\\n")\n\n# Predictions\ny_pred_test = model.predict(X_test_np)\n\n# Metrics\nfrom sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, confusion_matrix\n\ntest_accuracy = accuracy_score(y_test_np, y_pred_test)\nprecision = precision_score(y_test_np, y_pred_test, average=\'weighted\', zero_division=0)\nrecall = recall_score(y_test_np, y_pred_test, average=\'weighted\', zero_division=0)\nf1 = f1_score(y_test_np, y_pred_test, average=\'weighted\', zero_division=0)\n\nprint(f"📈 Test Results:")\nprint(f" Accuracy: {test_accuracy:.4f} ({test_accuracy*100:.2f}%)")\nprint(f" Precision: {precision:.4f}")\nprint(f" Recall: {recall:.4f}")\nprint(f" F1-Score: {f1:.4f}\\n")\n\n# Confusion Matrix\nfrom sklearn.metrics import ConfusionMatrixDisplay\nimport matplotlib.pyplot as plt\n\n# Create figure first\nfig, ax = plt.subplots(figsize=(10, 8))\n\nclass_names = list(label_mapping.keys())\ncm = confusion_matrix(y_test_np, y_pred_test)\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=class_names)\ndisp.plot(cmap=\'Blues\', ax=ax)\nplt.xticks(rotation=45, ha=\'right\')\nplt.title(\'XGBoost Confusion Matrix\')\nplt.tight_layout()\nplt.show()\n')
|
||||
|
||||
|
||||
# In[ ]:
|
||||
|
||||
|
||||
# Lưu mô hình huấn luyện
|
||||
import json
|
||||
import joblib
|
||||
|
||||
# Save XGBoost model
|
||||
model_path = "model_xgboost.joblib"
|
||||
joblib.dump(model, model_path)
|
||||
print(f"✅ Model saved to {model_path}")
|
||||
|
||||
# Save model info
|
||||
info = {
|
||||
"model_type": "XGBoost",
|
||||
"num_classes": 8,
|
||||
"classes": list(label_mapping.keys()),
|
||||
"num_features": X_train_np.shape[1],
|
||||
"params": params,
|
||||
"accuracy": float(test_accuracy),
|
||||
"precision": float(precision),
|
||||
"recall": float(recall),
|
||||
"f1_score": float(f1),
|
||||
}
|
||||
|
||||
with open("model_xgboost_info.json", "w") as f:
|
||||
json.dump(info, f, indent=2)
|
||||
|
||||
print(f"✅ Model info saved to model_xgboost_info.json")
|
||||
|
||||
|
||||
# In[15]:
|
||||
|
||||
|
||||
# đóng client, cluster
|
||||
# client.close()
|
||||
# cluster.close()
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,56 @@
|
||||
#!/usr/bin/env python
|
||||
# coding: utf-8
|
||||
|
||||
# In[6]:
|
||||
|
||||
|
||||
get_ipython().run_cell_magic('time', '', '%matplotlib inline\n\n# Import Microsoft Planetary Computer libraries\nimport planetary_computer\nfrom pystac_client import Client\nfrom odc.stac import load as stac_load\n\n# Standard imports\nimport xarray as xr\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score, classification_report, confusion_matrix, ConfusionMatrixDisplay\nimport geopandas as gpd\n\n# XGBoost for GPU training\nimport xgboost as xgb\n\nfrom xgboost import XGBClassifier\n\nprint(f" XGBoost version: {xgb.__version__}")\n\nprint("✅ All modules loaded successfully")\n')
|
||||
|
||||
|
||||
# In[7]:
|
||||
|
||||
|
||||
get_ipython().run_cell_magic('time', '', '# Kết nối tới Microsoft Planetary Computer STAC\nfrom pystac_client import Client\n\n# KHÔNG dùng modifier ở catalog level để tránh items bị convert thành dict\ncatalog = Client.open(\n "https://planetarycomputer.microsoft.com/api/stac/v1"\n)\nprint("✅ Connected to Microsoft Planetary Computer")\n\nprint("\\n" + "="*70)\n')
|
||||
|
||||
|
||||
# In[8]:
|
||||
|
||||
|
||||
get_ipython().run_cell_magic('time', '', '# 🌍 Định nghĩa khu vực và thời gian\nprint("="*70)\nprint("CONFIGURATION")\nprint("="*70)\n\n# Khu vực quan tâm (Vietnam - Mekong Delta) - GIẢM DIỆN TÍCH ~40%\nbbox = [105.6, 9.3, 106.2, 9.8] # [min_lon, min_lat, max_lon, max_lat]\n\n# GIẢM THỜI GIAN xuống 3 tháng để giảm kích thước dữ liệu cho PC\ntime_range = "2023-03-01/2023-05-31" # 3 tháng (mùa khô)\n\nprint(f"\\n📍 Area of Interest:")\nprint(f" Longitude: {bbox[0]} to {bbox[2]}")\nprint(f" Latitude: {bbox[1]} to {bbox[3]}")\nprint(f"\\n📅 Time Range: {time_range}")\nprint(f" ⚠️ Optimized for personal computer (3 months, reduced area)")\nprint(f"\\n🗺️ CRS: EPSG:32648")\nprint(f" Resolution: 20m (reduced from 10m for smaller data size)")\n\nprint("="*70)\n')
|
||||
|
||||
|
||||
# In[9]:
|
||||
|
||||
|
||||
get_ipython().run_cell_magic('time', '', '# 📡 LOAD SENTINEL-2 FROM MICROSOFT PLANETARY COMPUTER\nprint("="*70)\nprint("LOADING SENTINEL-2 L2A")\nprint("="*70)\n\nprint("\\n🔍 Searching for Sentinel-2 scenes...")\nquery_s2 = catalog.search(\n collections=["sentinel-2-l2a"],\n bbox=bbox,\n datetime=time_range,\n query={"eo:cloud_cover": {"lt": 30}} # Cloud cover < 30% (giảm từ 50%)\n)\n\nitems_s2 = list(query_s2.item_collection())\nprint(f"✅ Found {len(items_s2)} Sentinel-2 scenes")\n\n# GIỚI HẠN SỐ LƯỢNG SCENES cho PC cá nhân\nmax_scenes = 12 # Giảm xuống 12 scenes để tối ưu cho PC\nif len(items_s2) > max_scenes:\n print(f"⚠️ Limiting to {max_scenes} scenes for personal computer")\n # Chọn scenes đều đặn trong khoảng thời gian\n step = len(items_s2) // max_scenes\n items_s2 = items_s2[::step][:max_scenes]\n print(f" Selected {len(items_s2)} scenes evenly distributed")\n\nif len(items_s2) > 0:\n # Show first few scenes\n print(f"\\n📋 Sample scenes:")\n for i, item in enumerate(items_s2[:5]):\n date = item.datetime.strftime("%Y-%m-%d")\n cloud = item.properties.get("eo:cloud_cover", "N/A")\n print(f" [{i+1}] {date} - Cloud: {cloud}%")\n \n # Re-sign items to ensure fresh URLs (keep as pystac objects)\n print(f"\\n🔑 Signing STAC items...")\n items_s2 = [planetary_computer.sign(item) for item in items_s2]\n \n # Load Sentinel-2 data (without Dask chunks)\n print(f"\\n⏳ Loading Sentinel-2 data...")\n ds_s2 = stac_load(\n items_s2,\n bands=["B04", "B08", "SCL"], # Red (B04), NIR (B08), Scene Classification (SCL)\n crs="EPSG:32648",\n resolution=20, # 20m resolution (4x smaller data than 10m)\n bbox=bbox,\n patch_url=planetary_computer.sign, # Re-sign URLs during loading\n fail_on_error=False, # Skip problematic tiles instead of crashing\n )\n \n # Rename bands to simpler names\n ds_s2 = ds_s2.rename({"B04": "red", "B08": "nir", "SCL": "scl"})\n \n print(f"\\n✅ Sentinel-2 loaded!")\n print(f" Shape: {dict(ds_s2.dims)}")\n print(f" Variables: {list(ds_s2.data_vars)}")\n display(ds_s2)\nelse:\n print(f"❌ No Sentinel-2 scenes found")\n\n ds_s2 = Noneprint("="*70)\n')
|
||||
|
||||
|
||||
# In[10]:
|
||||
|
||||
|
||||
get_ipython().run_cell_magic('time', '', '# 📡 LOAD SENTINEL-1 FROM MICROSOFT PLANETARY COMPUTER\nprint("="*70)\nprint("LOADING SENTINEL-1 RTC")\nprint("="*70)\n\nprint("\\n🔍 Searching for Sentinel-1 scenes...")\nquery_s1 = catalog.search(\n collections=["sentinel-1-rtc"],\n bbox=bbox,\n datetime=time_range,\n)\n\nitems_s1 = list(query_s1.item_collection())\nprint(f"✅ Found {len(items_s1)} Sentinel-1 scenes")\n\n# GIỚI HẠN SỐ LƯỢNG SCENES cho PC cá nhân\nmax_scenes = 12 # Giảm xuống 12 scenes để tối ưu cho PC\nif len(items_s1) > max_scenes:\n print(f"⚠️ Limiting to {max_scenes} scenes for personal computer")\n # Chọn scenes đều đặn trong khoảng thời gian\n step = len(items_s1) // max_scenes\n items_s1 = items_s1[::step][:max_scenes]\n print(f" Selected {len(items_s1)} scenes evenly distributed")\n\nif len(items_s1) > 0:\n # Show first few scenes\n print(f"\\n📋 Sample scenes:")\n for i, item in enumerate(items_s1[:5]):\n date = item.datetime.strftime("%Y-%m-%d")\n orbit = item.properties.get("sat:orbit_state", "N/A")\n print(f" [{i+1}] {date} - Orbit: {orbit}")\n \n # Re-sign items to ensure fresh URLs (keep as pystac objects)\n print(f"\\n🔑 Signing STAC items...")\n items_s1 = [planetary_computer.sign(item) for item in items_s1]\n \n # Load Sentinel-1 data (without Dask chunks)\n print(f"\\n⏳ Loading Sentinel-1 data...")\n ds_s1 = stac_load(\n items_s1,\n bands=["vv", "vh"], # VV and VH polarizations\n crs="EPSG:32648",\n resolution=20, # 20m resolution (4x smaller data than 10m)\n bbox=bbox,\n patch_url=planetary_computer.sign, # Re-sign URLs during loading\n fail_on_error=False, # Skip problematic tiles instead of crashing\n )\n \n # Convert to dB (Microsoft S1 is in linear power)\n print(f"\\n🔄 Converting to dB...")\n ds_s1[\'vv_db\'] = 10 * np.log10(ds_s1[\'vv\'].where(ds_s1[\'vv\'] > 0))\n ds_s1[\'vh_db\'] = 10 * np.log10(ds_s1[\'vh\'].where(ds_s1[\'vh\'] > 0))\n \n print(f"\\n✅ Sentinel-1 loaded!")\n print(f" Shape: {dict(ds_s1.dims)}")\n print(f" Variables: {list(ds_s1.data_vars)}")\n display(ds_s1)\nelse:\n print(f"❌ No Sentinel-1 scenes found")\n\n ds_s1 = Noneprint("="*70)\n')
|
||||
|
||||
|
||||
# In[11]:
|
||||
|
||||
|
||||
get_ipython().run_cell_magic('time', '', '# 🌿 CALCULATE NDVI AND PROCESS DATA\nprint("="*70)\nprint("DATA PROCESSING")\nprint("="*70)\n\nif ds_s2 is not None:\n print("\\n[1] Calculating NDVI...")\n # NDVI = (NIR - Red) / (NIR + Red)\n ndvi = (ds_s2[\'nir\'] - ds_s2[\'red\']) / (ds_s2[\'nir\'] + ds_s2[\'red\'] + 1e-8)\n \n print(f"✅ NDVI calculated")\n print(f" Shape: {ndvi.shape}")\n print(f" Time steps: {len(ndvi.time)}")\n \n # Cloud masking using SCL band\n print(f"\\n[2] Applying cloud mask...")\n # SCL values: 1=defective, 3=cloud shadow, 8=cloud medium, 9=cloud high, 10=cirrus\n cloud_mask = ds_s2[\'scl\'].isin([1, 3, 8, 9, 10])\n ndvi_masked = ndvi.where(~cloud_mask)\n \n print(f"✅ Cloud mask applied")\n \n # Temporal aggregation (mean over time)\n print(f"\\n[3] Computing mean NDVI across time...")\n ndvi_mean = ndvi_masked.mean(dim=\'time\')\n \n # Data already in memory, no need to compute() again\n print(f"✅ Mean NDVI computed")\n print(f" Shape: {ndvi_mean.shape}")\n \nelse:\n print("❌ No Sentinel-2 data to process")\n ndvi_mean = None\n\nprint("="*70)\n')
|
||||
|
||||
|
||||
# In[ ]:
|
||||
|
||||
|
||||
get_ipython().run_cell_magic('time', '', '# 🎯 EXTRACT TRAINING DATA FEATURES\nprint("="*70)\nprint("FEATURE EXTRACTION")\nprint("="*70)\n\n# Check if required data is available\nif \'ndvi_mean\' not in globals() or \'ds_s1\' not in globals():\n print("❌ Error: Please run Cell 6 (DATA PROCESSING) first!")\n print(" Required variables: ndvi_mean, ds_s1")\n raise RuntimeError("Missing required data. Run cells in order: Cell 4 → Cell 5 → Cell 6 → Cell 7")\n\n# Load training shapefile\nimport geopandas as gpd\n\ntrain_path = \'train/ST_training data_updated_1130points_new.shp\'\nprint(f"\\n[1] Loading training data from: {train_path}")\ntrain_gdf = gpd.read_file(train_path)\n\n# Ensure CRS matches\nif train_gdf.crs != \'EPSG:32648\':\n print(f" Reprojecting from {train_gdf.crs} to EPSG:32648...")\n train_gdf = train_gdf.to_crs(\'EPSG:32648\')\n\nprint(f"✅ Loaded {len(train_gdf)} training points")\nprint(f" Available columns: {list(train_gdf.columns)}")\n\n# Auto-detect label column (look for common names)\nlabel_column = None\nfor col in [\'HT_code\', \'Ma_LU\', \'LU2022\', \'class\', \'Class\', \'CLASS\', \'label\', \'Label\', \'LABEL\', \'LU_CODE\', \'LU_code\']:\n if col in train_gdf.columns:\n label_column = col\n break\n\nif label_column is None:\n print(f"❌ Cannot find label column. Available columns: {list(train_gdf.columns)}")\n print(f" Please check your shapefile and update the code.")\nelse:\n print(f" Using label column: \'{label_column}\'")\n print(f" Classes: {sorted(train_gdf[label_column].unique())}")\n \n # Extract features at each training point\n print(f"\\n[2] Extracting features at training points...")\n \n features = []\n labels = []\n skipped = 0\n \n for idx, row in train_gdf.iterrows():\n point = row.geometrychro\n x_coord = point.x\n y_coord = point.y\n label = row[label_column]\n \n # Extract NDVI at this location\n if ndvi_mean is not None and ds_s1 is not None:\n try:\n ndvi_val = ndvi_mean.sel(x=x_coord, y=y_coord, method=\'nearest\').values\n \n # Extract Sentinel-1 VH/VV at this location (mean across time)\n # Data already in memory, no need to compute()\n vh_val = ds_s1[\'vh_db\'].sel(x=x_coord, y=y_coord, method=\'nearest\').mean(dim=\'time\').values\n vv_val = ds_s1[\'vv_db\'].sel(x=x_coord, y=y_coord, method=\'nearest\').mean(dim=\'time\').values\n \n # Create feature vector: [NDVI, VH_dB, VV_dB]\n feature_vec = [ndvi_val, vh_val, vv_val]\n \n # Only add if all features are valid (not NaN)\n if not np.isnan(feature_vec).any():\n features.append(feature_vec)\n labels.append(label)\n else:\n skipped += 1\n except Exception as e:\n # Skip points outside the data extent\n skipped += 1\n continue\n \n features = np.array(features)\n labels = np.array(labels)\n \n print(f"✅ Extracted features for {len(features)} valid points")\n print(f" Skipped {skipped} points (outside extent or NaN values)")\n print(f" Feature shape: {features.shape}")\n print(f" Feature names: [\'NDVI_mean\', \'VH_dB_mean\', \'VV_dB_mean\']")\n print(f"\\n Class distribution:")\n unique, counts = np.unique(labels, return_counts=True)\n for cls, cnt in zip(unique, counts):\n print(f" Class {cls}: {cnt} samples ({cnt/len(labels)*100:.1f}%)")\n\nprint("="*70)\n')
|
||||
|
||||
|
||||
# In[21]:
|
||||
|
||||
|
||||
get_ipython().run_cell_magic('time', '', '# 🤖 TRAIN XGBOOST MODEL ON GPU (RTX 4060)\nprint("="*70)\nprint("MODEL TRAINING - GPU ACCELERATED")\nprint("="*70)\n\nfrom xgboost import XGBClassifier\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import classification_report, confusion_matrix, ConfusionMatrixDisplay\nimport matplotlib.pyplot as plt\n\n# Encode labels to ensure they are 0, 1, 2, ... n-1\nprint("\\n[1] Encoding labels...")\nlabel_encoder = LabelEncoder()\nlabels_encoded = label_encoder.fit_transform(labels)\nprint(f"✅ Original classes: {label_encoder.classes_}")\nprint(f" Encoded as: {np.unique(labels_encoded)}")\n\n# Split data\nprint("\\n[2] Splitting data (80% train, 20% test)...")\nX_train, X_test, y_train, y_test = train_test_split(\n features, labels_encoded, test_size=0.2, random_state=42, stratify=labels_encoded\n)\nprint(f"✅ Training samples: {len(X_train)}")\nprint(f" Testing samples: {len(X_test)}")\n\n# Train XGBoost on GPU\nprint("\\n[3] Training XGBoost classifier on RTX 4060 GPU...")\nprint(" GPU Settings: device=\'cuda:0\'")\n\nxgb_model = XGBClassifier(\n n_estimators=100,\n max_depth=20,\n learning_rate=0.1,\n device=\'cuda:0\', # Use GPU (updated from deprecated gpu_id)\n tree_method=\'hist\', # Use hist with device for GPU training\n random_state=42,\n eval_metric=\'mlogloss\', # Multi-class log loss\n verbosity=1 # Show GPU training progress\n)\n\nxgb_model.fit(X_train, y_train)\nprint(f"✅ Model trained on GPU")\n\n# Evaluate\nprint("\\n[4] Evaluating model...")\ntrain_score = xgb_model.score(X_train, y_train)\ntest_score = xgb_model.score(X_test, y_test)\nprint(f"✅ Training accuracy: {train_score:.4f}")\nprint(f" Testing accuracy: {test_score:.4f}")\n\n# Classification report\nprint("\\n[5] Classification Report:")\ny_pred = xgb_model.predict(X_test)\nprint(classification_report(y_test, y_pred, target_names=[str(c) for c in label_encoder.classes_]))\n\n# Confusion matrix\nprint("\\n[6] Confusion Matrix:")\nfig, ax = plt.subplots(figsize=(10, 8))\ncm = confusion_matrix(y_test, y_pred)\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=label_encoder.classes_)\ndisp.plot(ax=ax, cmap=\'Blues\', values_format=\'d\')\nplt.title(\'Confusion Matrix - XGBoost GPU Model (RTX 4060)\')\nplt.tight_layout()\nplt.show()\n\nprint("="*70)\n')
|
||||
|
||||
|
||||
# In[23]:
|
||||
|
||||
|
||||
get_ipython().run_cell_magic('time', '', '# 💾 SAVE MODEL AND CLEANUP\nprint("="*70)\nprint("SAVING MODEL & CLEANUP")\nprint("="*70)\n\nimport joblib\nfrom datetime import datetime\n\n# Save model and label encoder\nmodel_filename = f"model_train/model_xgboost_gpu_{datetime.now().strftime(\'%Y%m%d_%H%M%S\')}.joblib"\nprint(f"\\n[1] Saving model to: {model_filename}")\njoblib.dump({\'model\': xgb_model, \'label_encoder\': label_encoder}, model_filename)\nprint(f"✅ Model and label encoder saved")\n\n# Save model info\ninfo = {\n "timestamp": datetime.now().isoformat(),\n "data_source": "Microsoft Planetary Computer STAC",\n "collections": ["sentinel-2-l2a", "sentinel-1-rtc"],\n "features": ["NDVI_mean", "VH_dB_mean", "VV_dB_mean"],\n "training_samples": len(X_train),\n "testing_samples": len(X_test),\n "train_accuracy": float(train_score),\n "test_accuracy": float(test_score),\n "model_type": "XGBClassifier",\n "device": "cuda:0",\n "gpu_device": "RTX 4060",\n "tree_method": "hist",\n "n_estimators": 100,\n "max_depth": 20,\n "learning_rate": 0.1\n}\n\nimport json\ninfo_filename = model_filename.replace(\'.joblib\', \'_info.json\')\nwith open(info_filename, \'w\') as f:\n json.dump(info, f, indent=2)\nprint(f"✅ Model info saved to: {info_filename}")\n\n# No cleanup needed (Dask removed)\nprint("\\n[2] Cleanup complete")\n\nprint("="*70)\n\nprint("\\n" + "="*70)\n\nprint("🎉 TRAINING COMPLETE!")\n')
|
||||
|
||||
+1426
-1667
File diff suppressed because one or more lines are too long
@@ -0,0 +1,191 @@
|
||||
#!/usr/bin/env python
|
||||
# coding: utf-8
|
||||
|
||||
# In[1]:
|
||||
|
||||
|
||||
get_ipython().run_cell_magic('time', '', '%matplotlib inline\n\nimport importlib\nimport new_import_ODC \n\nimportlib.reload(new_import_ODC)\n\nfrom new_import_ODC import *\n')
|
||||
|
||||
|
||||
# In[2]:
|
||||
|
||||
|
||||
get_ipython().run_cell_magic('time', '', '# Dask gateway\ncluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))\ndc = datacube.Datacube()\n\n# Configure s3 access\nconfigure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n\nclient\n')
|
||||
|
||||
|
||||
# In[3]:
|
||||
|
||||
|
||||
## cấu hình thời gian lấy ảnh và tọa độ
|
||||
date_range = ('2022-09-01', '2023-10-01')
|
||||
longtitude_range = (105.86575, 105.94120)
|
||||
latitude_range = (9.65070, 9.69850)
|
||||
|
||||
|
||||
# In[4]:
|
||||
|
||||
|
||||
## truy vấn ảnh vệ tinh sen2
|
||||
data = load_data(dc, date_range, longtitude_range, latitude_range)
|
||||
notebook_utils.heading(notebook_utils.xarray_object_size(data))
|
||||
display(data)
|
||||
|
||||
|
||||
# In[5]:
|
||||
|
||||
|
||||
get_ipython().run_cell_magic('time', '', '# Tiến hành loại bỏ các vị trí bị mây ảnh hưởng\nresult = mask_clean(data)\nprogress(result)\n')
|
||||
|
||||
|
||||
# In[6]:
|
||||
|
||||
|
||||
# Tiến hành tính toán NDVI
|
||||
ds1 = calculate_indices(result, index='NDVI', satellite_mission='s2')
|
||||
ndvi = ds1["NDVI"]
|
||||
display(ndvi)
|
||||
|
||||
|
||||
# In[7]:
|
||||
|
||||
|
||||
## ảnh NDVI chưa điền mây (fill nan)
|
||||
plt.imshow(ndvi.isel(time=50))
|
||||
|
||||
|
||||
# In[8]:
|
||||
|
||||
|
||||
# đặt thời gian các mùa
|
||||
time_split = [slice('2022-09-01', '2023-01-01'),
|
||||
slice('2023-01-01', '2023-05-01'),
|
||||
slice('2023-05-01', '2023-07-01'),
|
||||
slice('2023-07-01', '2023-10-01')]
|
||||
|
||||
# Điền mây ở các vị trí mang giá trị nan (fill nan)
|
||||
fill_nan_ndvi = fill_nan(ndvi, time_split)
|
||||
|
||||
# In kết quả ảnh ndvi đã điền mây (đã fill nan)
|
||||
plt.imshow(fill_nan_ndvi.isel(time=50))
|
||||
|
||||
|
||||
# In[9]:
|
||||
|
||||
|
||||
get_ipython().run_cell_magic('time', '', "## tính ndvi theo tháng\naverage_ndvi = fill_nan_ndvi.resample(time='1M').mean().persist()\nprogress(average_ndvi)\n")
|
||||
|
||||
|
||||
# In[10]:
|
||||
|
||||
|
||||
# compute average_ndvi
|
||||
average_ndvi = average_ndvi.compute()
|
||||
|
||||
|
||||
# In[11]:
|
||||
|
||||
|
||||
# load dữ liệu sen1
|
||||
coordinates = (longtitude_range, latitude_range)
|
||||
dsvh, dsvv = load_data_sen1(dc, date_range, coordinates)
|
||||
average_vv = calculate_average(dsvv, time_pattern='1M')
|
||||
average_vh = calculate_average(dsvh, time_pattern='1M')
|
||||
|
||||
|
||||
# In[12]:
|
||||
|
||||
|
||||
# load model RF
|
||||
loaded_model = joblib.load(os.path.join("model_train", "model_odc.joblib"))
|
||||
|
||||
# dự đoán
|
||||
data_array = predict(loaded_model, data.rio.crs, average_ndvi, average_vh, average_vv)
|
||||
|
||||
|
||||
# In[13]:
|
||||
|
||||
|
||||
# cấu hình màu cho các loại đất
|
||||
colors = [
|
||||
"#abcee9",
|
||||
"#ffef44",
|
||||
"#c4ff9e",
|
||||
"#ffd6a8",
|
||||
"#93ddda",
|
||||
"#1aeef7",
|
||||
"#ffa7f2",
|
||||
"#33ee33"
|
||||
]
|
||||
labels = [
|
||||
"Lúa tôm",
|
||||
"Lúa",
|
||||
"CHN",
|
||||
"CLN",
|
||||
"TS",
|
||||
"Sông",
|
||||
"Đất xây dựng",
|
||||
"Rừng"
|
||||
]
|
||||
# hiển thị phân loại sử dụng đất
|
||||
cmap = ListedColormap(colors)
|
||||
img = data_array.plot(cmap=cmap, add_colorbar=False)
|
||||
cbar = plt.colorbar(img)
|
||||
cbar.ax.set_yticklabels(labels)
|
||||
plt.title("Phân loại sử dụng đất")
|
||||
plt.axis('off')
|
||||
plt.show()
|
||||
|
||||
|
||||
# In[14]:
|
||||
|
||||
|
||||
## cấu hình shapefile ranh giới thuận hòa và vh vv file
|
||||
thuanhoa_path = "ThuanHoa/region/ST_ThuanHoa_Boundaryofficially.shp"
|
||||
|
||||
# cắt theo ranh giới xã thuận hòa
|
||||
region_result = cut_according_shp(thuanhoa_path, average_ndvi, data_array)
|
||||
|
||||
|
||||
# In[15]:
|
||||
|
||||
|
||||
# hiển thị kết quả phân loại sử dụng đất
|
||||
colorval = list(range(len(colors)))
|
||||
options = {
|
||||
'title': 'Phân loại sử dụng đất',
|
||||
'cmap': colors,
|
||||
'clim': (0, 8),
|
||||
'aspect': 'equal',
|
||||
'colorbar_opts': {
|
||||
'major_label_overrides': dict(zip(colorval, labels)),
|
||||
'major_label_text_align': 'left',
|
||||
'ticker': FixedTicker(ticks=colorval),
|
||||
},
|
||||
}
|
||||
|
||||
region_result.hvplot(
|
||||
rasterize = True, # Use Datashader, particularly useful for dask arrays
|
||||
aggregator = reductions.mode(), # Datashader selects mode value, requires 'hv.Image'
|
||||
).options(opts.Image(**options))
|
||||
|
||||
|
||||
# In[16]:
|
||||
|
||||
|
||||
# Lưu lại kết quả
|
||||
region_result.rio.to_raster("KetQuaPhanLoaiDatODC.tif")
|
||||
|
||||
|
||||
# In[17]:
|
||||
|
||||
|
||||
# đóng client, cluster
|
||||
client.close()
|
||||
cluster.close()
|
||||
|
||||
|
||||
# In[ ]:
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
#!/usr/bin/env python
|
||||
# coding: utf-8
|
||||
|
||||
# In[1]:
|
||||
|
||||
|
||||
# Khai báo các thư viện cần thiết
|
||||
from new_import_ODC import *
|
||||
|
||||
# Khai báo đường dẫn đến kết quả phân loại và dữ liệu của địa phương
|
||||
KD_path = "ThuanHoa/KhoanhDat/ThuanHoa_TKDD2022.shp"
|
||||
KetQuaPhanLoaiDat = "KetQuaPhanLoaiDatODC.tif"
|
||||
|
||||
|
||||
# In[2]:
|
||||
|
||||
|
||||
# khai báo các loại đất từ dữ liệu kiểm kê ứng với các hiện trạng được phân loại từ viễn thám
|
||||
CODE_MAP = {
|
||||
"BHK": 2,
|
||||
"CLN": 3,
|
||||
"DGD": 6,
|
||||
"DGT": 6,
|
||||
"DNL": 6,
|
||||
"DRA": 6,
|
||||
"DSH": 6,
|
||||
"DTL": 5,
|
||||
"DTS": 6,
|
||||
"DYT": 6,
|
||||
"LUC": 1,
|
||||
"NKH": 3,
|
||||
"NTD": 6,
|
||||
"NTS": 4,
|
||||
"ONT": 6,
|
||||
"SKC": 6,
|
||||
"SKX": 6,
|
||||
"SON": 5,
|
||||
"TMD": 6,
|
||||
"TON": 6,
|
||||
"TSC": 6,
|
||||
}
|
||||
|
||||
# Khai báo các nhãn phân loại đất ứng với 3 loại đất chính
|
||||
HT_MAP = {
|
||||
"NN": {"name": "Đất Nông Nghiệp", "data": [1, 2, 3, 4]},
|
||||
"PNN": {"name": "Đất Phi Nông Nghiệp", "data": [6]},
|
||||
"TQ": {"name": "Đất Thổ Quả", "data": [15]},
|
||||
}
|
||||
|
||||
|
||||
# In[3]:
|
||||
|
||||
|
||||
# Tiến hành chồng lắp
|
||||
result = compare(KD_path, KetQuaPhanLoaiDat, CODE_MAP, HT_MAP)
|
||||
|
||||
|
||||
# In[4]:
|
||||
|
||||
|
||||
# cấu hình màu cho các loại sử dụng đất
|
||||
colors = [
|
||||
"#abcee9",
|
||||
"#ffffc0",
|
||||
"#c4ff9e",
|
||||
"#ffd6a8",
|
||||
"#93ddda",
|
||||
"#1aeef7",
|
||||
"#ffa7f2",
|
||||
"#33ee33",
|
||||
]
|
||||
labels = ["Lúa tôm", "Lúa", "CHN", "CLN", "TS", "Sông", "Đất xây dựng", "Rừng"]
|
||||
|
||||
|
||||
# In[5]:
|
||||
|
||||
|
||||
# Lưu kết quả
|
||||
save_result(result, HT_MAP)
|
||||
|
||||
|
||||
# In[6]:
|
||||
|
||||
|
||||
# hiển thị kết quả
|
||||
xx = []
|
||||
|
||||
for k, v in result.items():
|
||||
rs = merge_arrays(v, nodata=np.nan)
|
||||
xx.append(rs.squeeze(drop=True))
|
||||
xx = xr.concat(xx, pd.Index([HT_MAP[x]["name"] for x in HT_MAP], name="name"))
|
||||
|
||||
colorval = list(range(len(colors)))
|
||||
options = {
|
||||
"cmap": colors,
|
||||
"clim": (0, 8),
|
||||
"aspect": "equal",
|
||||
"height": 400,
|
||||
"colorbar_opts": {
|
||||
"major_label_overrides": dict(zip(colorval, labels)),
|
||||
"major_label_text_align": "left",
|
||||
"ticker": FixedTicker(ticks=colorval),
|
||||
},
|
||||
}
|
||||
|
||||
xx.hvplot(
|
||||
groupby="name",
|
||||
rasterize=True, # Use Datashader, particularly useful for dask arrays
|
||||
aggregator=reductions.mode(), # Datashader selects mode value, requires 'hv.Image'
|
||||
).options(opts.Image(**options))
|
||||
|
||||
|
||||
# In[ ]:
|
||||
|
||||
|
||||
|
||||
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
import glob, json
|
||||
|
||||
changed_files = []
|
||||
for file_path in glob.glob('*.ipynb'):
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
nb = json.load(f)
|
||||
|
||||
changed = False
|
||||
for cell in nb.get('cells', []):
|
||||
if cell.get('cell_type') == 'code':
|
||||
source = cell.get('source', [])
|
||||
for i, line in enumerate(source):
|
||||
if 'time=50' in line:
|
||||
source[i] = line.replace('time=50', 'time=0')
|
||||
changed = True
|
||||
if 'load_data_sen1(dc,' in line:
|
||||
source[i] = line.replace('load_data_sen1(dc,', 'load_data_sen1(None,')
|
||||
changed = True
|
||||
|
||||
if changed:
|
||||
with open(file_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(nb, f, indent=1)
|
||||
changed_files.append(file_path)
|
||||
|
||||
print('Fixed issues in:', changed_files)
|
||||
@@ -0,0 +1,20 @@
|
||||
import json
|
||||
|
||||
def fix_import(file_path):
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
nb = json.load(f)
|
||||
changed = False
|
||||
for cell in nb.get('cells', []):
|
||||
if cell.get('cell_type') == 'code':
|
||||
source = cell.get('source', [])
|
||||
if isinstance(source, list):
|
||||
for i, line in enumerate(source):
|
||||
if "from new_import import *" in line:
|
||||
source[i] = line.replace("from new_import import *", "from new_import_ODC import *")
|
||||
changed = True
|
||||
if changed:
|
||||
with open(file_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(nb, f, indent=1)
|
||||
print(f"Fixed {file_path}")
|
||||
|
||||
fix_import('new_train.ipynb')
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
import json
|
||||
|
||||
def fix_filename(file_path):
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
nb = json.load(f)
|
||||
changed = False
|
||||
for cell in nb.get('cells', []):
|
||||
if cell.get('cell_type') == 'code':
|
||||
source = cell.get('source', [])
|
||||
if isinstance(source, list):
|
||||
for i, line in enumerate(source):
|
||||
if "ST_training data_updated_1130points.shp" in line:
|
||||
source[i] = line.replace("ST_training data_updated_1130points.shp", "ST_training_data_updated_1130points.shp")
|
||||
changed = True
|
||||
if changed:
|
||||
with open(file_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(nb, f, indent=1)
|
||||
print(f"Fixed typo in {file_path}")
|
||||
|
||||
import glob
|
||||
for nb in glob.glob("*.ipynb"):
|
||||
fix_filename(nb)
|
||||
@@ -0,0 +1,7 @@
|
||||
import json
|
||||
nb = json.load(open('01.train_ODC.ipynb'))
|
||||
for idx, cell in enumerate(nb['cells']):
|
||||
if cell['cell_type'] == 'code':
|
||||
print(f"Cell {idx}:")
|
||||
print("".join(cell['source'][:3]))
|
||||
print("-" * 20)
|
||||
@@ -0,0 +1,105 @@
|
||||
# Hướng dẫn Chuyển đổi dữ liệu vệ tinh sang Microsoft Planetary Computer STAC
|
||||
|
||||
Tài liệu này ghi chú lại các bước chuẩn hóa và các đoạn code mẫu để chuyển đổi việc tải dữ liệu vệ tinh (Sentinel-1, Sentinel-2) từ kho lưu trữ đóng (như AWS S3 yêu cầu xác thực) sang nền tảng mở **Microsoft Planetary Computer STAC API**. Bạn có thể dùng tài liệu này làm context (ngữ cảnh) gửi cho các AI khác để chúng hiểu cách thực hiện tương tự.
|
||||
|
||||
---
|
||||
|
||||
## 1. Mục đích
|
||||
- Bỏ qua các lỗi liên quan đến xác thực đám mây (VD: `RasterioIOError: AWS_SECRET_ACCESS_KEY not defined`).
|
||||
- Tải dữ liệu miễn phí, trực tiếp từ kho dữ liệu mở của Microsoft Planetary Computer.
|
||||
- Đảm bảo đầu ra (output) của dữ liệu STAC giống hệt với định dạng của ảnh TIF gốc tải bằng `rioxarray` để không làm hỏng các luồng xử lý Machine Learning ở phía sau.
|
||||
|
||||
## 2. Các thư viện bắt buộc (Dependencies)
|
||||
Đảm bảo môi trường Python có cài đặt các thư viện sau:
|
||||
```python
|
||||
import pystac_client
|
||||
import planetary_computer
|
||||
import odc.stac
|
||||
import xarray as xr
|
||||
import rioxarray
|
||||
```
|
||||
|
||||
## 3. Các bước thực hiện chi tiết
|
||||
|
||||
### Bước 1: Kết nối đến STAC API và truy vấn dữ liệu
|
||||
Thay vì dùng `rioxarray.open_rasterio("s3://...")`, chúng ta khởi tạo STAC Client và tìm kiếm dữ liệu theo tọa độ (`bbox`) và thời gian (`datetime`).
|
||||
|
||||
```python
|
||||
# 1. Kết nối STAC Client có kèm chữ ký xác thực (sign_inplace) của Microsoft
|
||||
catalog = pystac_client.Client.open(
|
||||
"https://planetarycomputer.microsoft.com/api/stac/v1",
|
||||
modifier=planetary_computer.sign_inplace,
|
||||
)
|
||||
|
||||
# 2. Định nghĩa toạ độ và thời gian
|
||||
bbox = [105.5, 9.2, 106.4, 10.0] # [min_lon, min_lat, max_lon, max_lat]
|
||||
datetime = "2022-09-01/2023-10-01"
|
||||
|
||||
# 3. Tìm kiếm Items
|
||||
# Thay "sentinel-1-rtc" bằng "sentinel-2-l2a" nếu tải ảnh quang học
|
||||
search = catalog.search(
|
||||
collections=["sentinel-1-rtc"],
|
||||
bbox=bbox,
|
||||
datetime=datetime,
|
||||
)
|
||||
items = list(search.items())
|
||||
```
|
||||
|
||||
### Bước 2: Tải dữ liệu xuống xarray bằng `odc.stac`
|
||||
Thay vì tải thủ công từng link URL, `odc.stac.load` sẽ tự động tải, cắt ảnh theo `bbox`, đổi hệ tọa độ (reproject) và ghép lại thành một khối dữ liệu không gian - thời gian (DataCube).
|
||||
|
||||
```python
|
||||
# Tải dữ liệu thành xarray Dataset
|
||||
ds_s1 = odc.stac.load(
|
||||
items,
|
||||
bands=["vv", "vh"], # Tên các band cần tải
|
||||
bbox=bbox,
|
||||
crs="EPSG:32648", # Ép về hệ toạ độ đích (VD: UTM Zone 48N cho VN)
|
||||
resolution=10, # Độ phân giải (10 mét)
|
||||
chunks={"x": 2048, "y": 2048, "time": 1} # Dùng Dask chunking để tránh tràn RAM
|
||||
)
|
||||
```
|
||||
|
||||
### Bước 3: Nén trục thời gian (Temporal Compositing)
|
||||
Dữ liệu từ STAC sẽ có 3 chiều: `(time, y, x)`. Do ảnh TIF gốc cũ thường là ảnh đã được nén (ví dụ trung bình của 1 năm), ta cần dùng phép tính trung vị (`median`) hoặc trung bình (`mean`) để triệt tiêu trục `time`, biến dữ liệu thành dạng 2D `(y, x)`.
|
||||
|
||||
```python
|
||||
# Tính giá trị trung vị theo thời gian
|
||||
ds_median = ds_s1.median(dim="time").compute()
|
||||
|
||||
# Tách riêng các DataArray
|
||||
vv = ds_median["vv"]
|
||||
vh = ds_median["vh"]
|
||||
```
|
||||
|
||||
### Bước 4: Khôi phục cấu trúc DataArray gốc (Mimic rioxarray)
|
||||
Hàm `rioxarray.open_rasterio` gốc luôn trả về dữ liệu có trục `band` (kích thước = 1). Để code Machine Learning bên dưới không bị lỗi "out of bounds" hay "missing dimension", ta phải thêm trục `band` giả và gán lại thông tin `crs`.
|
||||
|
||||
```python
|
||||
# Thêm chiều 'band' để giống hệt rioxarray
|
||||
vv = vv.expand_dims(dim="band")
|
||||
vh = vh.expand_dims(dim="band")
|
||||
|
||||
# Phục hồi metadata về toạ độ
|
||||
vv = vv.rio.write_crs("EPSG:32648")
|
||||
vh = vh.rio.write_crs("EPSG:32648")
|
||||
```
|
||||
|
||||
### Bước 5: Quét và sửa các đoạn code "Hardcode" kích thước
|
||||
Do lưới tọa độ của STAC tự sinh (dựa trên bounding box) có thể lệch vài pixel so với lưới của file TIF đã cắt tay trên S3 (VD: S3 là `8874 x 9902`, STAC là `8870 x 9900`), **phải tìm và xóa bỏ toàn bộ các con số fix cứng trong mảng**.
|
||||
|
||||
*Code cũ sai lầm:*
|
||||
```python
|
||||
tmp = np.ones((8874, 9902))
|
||||
final_label = final_label.reshape(8874, 9902)
|
||||
```
|
||||
|
||||
*Code chuẩn hóa:*
|
||||
```python
|
||||
# Lấy linh động theo shape thực tế của xarray
|
||||
tmp = np.ones((ds_vhvv.shape[1], ds_vhvv.shape[2]))
|
||||
final_label = final_label.reshape(ds_vhvv.shape[1], ds_vhvv.shape[2])
|
||||
```
|
||||
|
||||
## 4. Tổng kết
|
||||
Chỉ cần cung cấp tài liệu này cho bất kỳ AI nào, yêu cầu: *"Hãy refactor (viết lại) hàm load file TIF của tôi theo đúng 5 bước trong tài liệu Microsoft Planetary Computer này"*, AI đó sẽ có đủ toàn bộ tư duy và code mẫu để hoàn thành công việc một cách mượt mà nhất.
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"model_type": "XGBoost",
|
||||
"num_classes": 8,
|
||||
"classes": [
|
||||
"Lua tom",
|
||||
"Lua",
|
||||
"CHN",
|
||||
"CLN",
|
||||
"TS",
|
||||
"Song",
|
||||
"Dat xay dung",
|
||||
"Rung"
|
||||
],
|
||||
"num_features": 3,
|
||||
"params": {
|
||||
"objective": "multi:softmax",
|
||||
"num_class": 8,
|
||||
"max_depth": 6,
|
||||
"learning_rate": 0.1,
|
||||
"n_estimators": 200,
|
||||
"subsample": 0.8,
|
||||
"colsample_bytree": 0.8,
|
||||
"random_state": 42,
|
||||
"n_jobs": -1,
|
||||
"eval_metric": "mlogloss"
|
||||
},
|
||||
"accuracy": 0.28761061946902655,
|
||||
"precision": 0.35339400643604185,
|
||||
"recall": 0.28761061946902655,
|
||||
"f1_score": 0.23460742664282486
|
||||
}
|
||||
+166
-90
@@ -1,8 +1,16 @@
|
||||
TEST_MODE = True
|
||||
RESOLUTION = 1000 if TEST_MODE else 10
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
# Common imports and settings
|
||||
import os, sys
|
||||
os.environ['USE_PYGEOS'] = '0'
|
||||
os.environ["GDAL_HTTP_MAX_RETRY"] = "5"
|
||||
os.environ["GDAL_HTTP_RETRY_DELAY"] = "2"
|
||||
os.environ["GDAL_HTTP_CONNECTION_TIMEOUT"] = "10"
|
||||
os.environ["GDAL_HTTP_TIMEOUT"] = "30"
|
||||
os.environ["CPL_VSIL_CURL_ALLOWED_EXTENSIONS"] = ".tif,.tiff"
|
||||
os.environ["GDAL_DISABLE_READDIR_ON_OPEN"] = "YES"
|
||||
from IPython.display import Markdown
|
||||
import pandas as pd
|
||||
pd.set_option("display.max_rows", None)
|
||||
@@ -13,15 +21,13 @@ import datacube
|
||||
from datacube.utils.rio import configure_s3_access
|
||||
from datacube.utils import masking
|
||||
from datacube.utils.cog import write_cog
|
||||
# https://github.com/GeoscienceAustralia/dea-notebooks/tree/develop/Tools
|
||||
from dea_tools.plotting import display_map, rgb
|
||||
from dea_tools.datahandling import mostcommon_crs
|
||||
# removed deafrica_tools imports to avoid ipyleaflet error
|
||||
|
||||
# EASI defaults
|
||||
easinotebooksrepo = '/home/jovyan/easi-notebooks'
|
||||
easinotebooksrepo = '/home/x79/CSIROBoeingPhase4-Vietnam'
|
||||
if easinotebooksrepo not in sys.path: sys.path.append(easinotebooksrepo)
|
||||
from easi_tools import EasiDefaults, xarray_object_size, notebook_utils, unset_cachingproxy
|
||||
from easi_tools.load_s2l2a import load_s2l2a_with_offset
|
||||
# from easi_tools.load_s2l2a import load_s2l2a_with_offset
|
||||
from dask.distributed import progress
|
||||
|
||||
# Data tools
|
||||
@@ -31,7 +37,7 @@ from datetime import datetime
|
||||
# Datacube
|
||||
from datacube.utils import masking # https://github.com/opendatacube/datacube-core/blob/develop/datacube/utils/masking.py
|
||||
from odc.algo import enum_to_bool # https://github.com/opendatacube/odc-algo/blob/main/odc/algo/_masking.py
|
||||
from odc.algo import xr_reproject # https://github.com/opendatacube/odc-algo/blob/main/odc/algo/_warp.py
|
||||
# removed xr_reproject
|
||||
from datacube.utils.geometry import GeoBox, box # https://github.com/opendatacube/datacube-core/blob/develop/datacube/utils/geometry/_base.py
|
||||
|
||||
# Holoviews, Datashader and Bokeh
|
||||
@@ -83,56 +89,73 @@ import joblib
|
||||
|
||||
def load_data(dc, date_range, longtitude_range, latitude_range):
|
||||
product = 's2_l2a'
|
||||
query = {
|
||||
'product': product, # Product name
|
||||
'x': longtitude_range, # "x" axis bounds
|
||||
'y': latitude_range, # "y" axis bounds
|
||||
'time': date_range, # Any parsable date strings
|
||||
}
|
||||
native_crs = notebook_utils.mostcommon_crs(dc, query)
|
||||
print(f'Most common native CRS: {native_crs}')
|
||||
measurements = ['red', 'nir', 'scl']
|
||||
bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
|
||||
|
||||
load_params = {
|
||||
'measurements': measurements, # Selected measurement or alias names
|
||||
'output_crs': native_crs, # Target EPSG code
|
||||
'resolution': (-10, 10), # Target resolution
|
||||
'group_by': 'solar_day', # Scene grouping
|
||||
'dask_chunks': {'x': 2048, 'y': 2048}, # Dask chunks
|
||||
}
|
||||
data = load_s2l2a_with_offset(
|
||||
dc,
|
||||
query | load_params # Combine the two dicts that contain our search and load parameters
|
||||
import pystac_client
|
||||
import planetary_computer
|
||||
import odc.stac
|
||||
|
||||
catalog = pystac_client.Client.open(
|
||||
"https://planetarycomputer.microsoft.com/api/stac/v1",
|
||||
modifier=planetary_computer.sign_inplace,
|
||||
)
|
||||
search = catalog.search(
|
||||
collections=["sentinel-2-l2a"],
|
||||
bbox=bbox,
|
||||
datetime=f"{date_range[0]}/{date_range[1]}",
|
||||
)
|
||||
items = list(search.items())
|
||||
|
||||
data = odc.stac.load(
|
||||
items,
|
||||
bands=["red", "nir", "SCL"],
|
||||
bbox=bbox,
|
||||
crs="EPSG:32648",
|
||||
resolution=RESOLUTION,
|
||||
chunks={"x": 2048, "y": 2048, "time": 1},
|
||||
groupby="solar_day"
|
||||
)
|
||||
if "SCL" in data.data_vars:
|
||||
data = data.rename({"SCL": "scl"})
|
||||
return data
|
||||
|
||||
|
||||
def mask_clean(data):
|
||||
flag_name = 'scl'
|
||||
flag_desc = masking.describe_variable_flags(data[flag_name]) # Pandas dataframe
|
||||
display(flag_desc)
|
||||
display(flag_desc.loc['qa'].values[1])
|
||||
# Create a "data quality" Mask layer
|
||||
flags_def = flag_desc.loc['qa'].values[1]
|
||||
good_pixel_flags = [flags_def[str(i)] for i in [2, 4, 5, 6]] # To pass strings to enum_to_bool()
|
||||
|
||||
# enum_to_bool calculates the pixel-wise "or" of each set of pixels given by good_pixel_flags
|
||||
# 1 = good data
|
||||
# 0 = "bad" data
|
||||
good_pixel_mask = enum_to_bool(data[flag_name], good_pixel_flags)
|
||||
# For Sentinel-2 L2A SCL:
|
||||
# 2: Dark Area Pixels, 4: Vegetation, 5: Not Vegetated, 6: Water
|
||||
good_pixel_mask = data['scl'].isin([2, 4, 5, 6])
|
||||
data_layer_names = [x for x in data.data_vars if x != 'scl']
|
||||
# Apply good pixel mask to blue, green, red and nir.
|
||||
# Apply good pixel mask
|
||||
result = data[data_layer_names].where(good_pixel_mask).persist()
|
||||
return result
|
||||
|
||||
|
||||
def fill_nan(ndvi, time_split):
|
||||
if len(ndvi.time) == 0:
|
||||
return ndvi
|
||||
|
||||
# If the total time duration is less than 90 days, skip seasonal splitting
|
||||
try:
|
||||
total_days = (ndvi.time[-1] - ndvi.time[0]).dt.days.item()
|
||||
if total_days < 90:
|
||||
return ndvi.bfill(dim="time").ffill(dim="time")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
rs = []
|
||||
for times in time_split:
|
||||
try:
|
||||
tmp = ndvi.sel(time=times)
|
||||
fill_ds = tmp.sel(time=times).bfill(dim='time')
|
||||
fill_ds = fill_ds.sel(time=times).ffill(dim='time')
|
||||
if len(tmp.time) == 0:
|
||||
continue
|
||||
fill_ds = tmp.bfill(dim='time').ffill(dim='time')
|
||||
rs.append(fill_ds)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if len(rs) == 0:
|
||||
return ndvi.bfill(dim="time").ffill(dim="time")
|
||||
|
||||
merged_ndvi = xr.concat([i for i in rs], dim="time")
|
||||
fill_m = merged_ndvi.bfill(dim="time")
|
||||
fill_m = fill_m.ffill(dim="time")
|
||||
@@ -144,10 +167,49 @@ def load_train_data(train_path):
|
||||
return train
|
||||
|
||||
|
||||
def load_sen1(name_vh, name_vv):
|
||||
dsvv = rioxarray.open_rasterio(name_vv)
|
||||
dsvh = rioxarray.open_rasterio(name_vh)
|
||||
return dsvh, dsvv
|
||||
def load_sen1(bbox, time_range):
|
||||
import pystac_client
|
||||
import planetary_computer
|
||||
import odc.stac
|
||||
|
||||
# Kết nối STAC Client
|
||||
catalog = pystac_client.Client.open(
|
||||
"https://planetarycomputer.microsoft.com/api/stac/v1",
|
||||
modifier=planetary_computer.sign_inplace,
|
||||
)
|
||||
|
||||
# Tìm kiếm Items
|
||||
search = catalog.search(
|
||||
collections=["sentinel-1-rtc"],
|
||||
bbox=bbox,
|
||||
datetime=time_range,
|
||||
)
|
||||
items = list(search.items())
|
||||
|
||||
# Tải dữ liệu thành xarray Dataset
|
||||
ds_s1 = odc.stac.load(
|
||||
items,
|
||||
bands=["vv", "vh"],
|
||||
bbox=bbox,
|
||||
crs="EPSG:32648",
|
||||
resolution=RESOLUTION,
|
||||
chunks={"x": 2048, "y": 2048, "time": 1}
|
||||
)
|
||||
|
||||
# Tính giá trị trung vị theo thời gian
|
||||
ds_median = ds_s1.median(dim="time").compute()
|
||||
vv = ds_median["vv"]
|
||||
vh = ds_median["vh"]
|
||||
|
||||
# Thêm chiều 'band' để giống hệt rioxarray
|
||||
vv = vv.expand_dims(dim="band")
|
||||
vh = vh.expand_dims(dim="band")
|
||||
|
||||
# Phục hồi metadata về toạ độ
|
||||
vv = vv.rio.write_crs("EPSG:32648")
|
||||
vh = vh.rio.write_crs("EPSG:32648")
|
||||
|
||||
return vh, vv
|
||||
|
||||
|
||||
def get_data_sen1_and_sen2(train, average_ndvi, dsvh, dsvv):
|
||||
@@ -267,6 +329,10 @@ def save_model(name_file, model, metadata=None, label_encoder=None):
|
||||
|
||||
|
||||
def predict(model, data_crs, ndvi, vh, vv):
|
||||
# Unpack model if it is wrapped in a dictionary (from ModelManager)
|
||||
if isinstance(model, dict) and 'model' in model:
|
||||
model = model['model']
|
||||
|
||||
data_predict = []
|
||||
for i in range(ndvi.shape[1]):
|
||||
ndvi_tmp = ndvi.isel(y=i).values
|
||||
@@ -361,21 +427,35 @@ def save_result(result, HT_MAP):
|
||||
|
||||
def load_data_sen1(dc, date_range, coordinates):
|
||||
longtitude_range, latitude_range = coordinates
|
||||
data_sen1 = dc.load(
|
||||
product="sentinel1_grd_gamma0_10m",
|
||||
x=longtitude_range,
|
||||
y=latitude_range,
|
||||
time=date_range,
|
||||
measurements=["vv", "vh"],
|
||||
output_crs="EPSG:32648",
|
||||
resolution=(-10,10),
|
||||
dask_chunks={"x":2048, "y":2048},
|
||||
skip_broken_datasets=True,
|
||||
group_by='solar_day'
|
||||
bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
|
||||
|
||||
import pystac_client
|
||||
import planetary_computer
|
||||
import odc.stac
|
||||
|
||||
catalog = pystac_client.Client.open(
|
||||
"https://planetarycomputer.microsoft.com/api/stac/v1",
|
||||
modifier=planetary_computer.sign_inplace,
|
||||
)
|
||||
search = catalog.search(
|
||||
collections=["sentinel-1-rtc"],
|
||||
bbox=bbox,
|
||||
datetime=f"{date_range[0]}/{date_range[1]}",
|
||||
)
|
||||
items = list(search.items())
|
||||
|
||||
data_sen1 = odc.stac.load(
|
||||
items,
|
||||
bands=["vv", "vh"],
|
||||
bbox=bbox,
|
||||
crs="EPSG:32648",
|
||||
resolution=RESOLUTION,
|
||||
chunks={"x": 2048, "y": 2048, "time": 1},
|
||||
groupby="solar_day"
|
||||
)
|
||||
|
||||
notebook_utils.heading(notebook_utils.xarray_object_size(data_sen1))
|
||||
display(data_sen1)
|
||||
# notebook_utils.heading(notebook_utils.xarray_object_size(data_sen1))
|
||||
# display(data_sen1)
|
||||
dsvh = data_sen1.vh
|
||||
dsvv = data_sen1.vv
|
||||
|
||||
@@ -387,46 +467,42 @@ def calculate_average(data, time_pattern='1M'):
|
||||
|
||||
def load_data_sen2(dc, date_range, coordinates):
|
||||
longtitude_range, latitude_range = coordinates
|
||||
product = 's2_l2a'
|
||||
query = {
|
||||
'product': product, # Product name
|
||||
'x': longtitude_range, # "x" axis bounds
|
||||
'y': latitude_range, # "y" axis bounds
|
||||
'time': date_range, # Any parsable date strings
|
||||
}
|
||||
native_crs = notebook_utils.mostcommon_crs(dc, query)
|
||||
print(f'Most common native CRS: {native_crs}')
|
||||
bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
|
||||
|
||||
# measurements = ['red','green', 'blue', 'nir', 'scl']
|
||||
measurements = ['red', 'nir', 'scl']
|
||||
import pystac_client
|
||||
import planetary_computer
|
||||
import odc.stac
|
||||
|
||||
load_params = {
|
||||
'measurements': measurements, # Selected measurement or alias names
|
||||
'output_crs': native_crs, # Target EPSG code
|
||||
'resolution': (-10, 10), # Target resolution
|
||||
'group_by': 'solar_day', # Scene grouping
|
||||
'dask_chunks': {'x': 2048, 'y': 2048}, # Dask chunks
|
||||
}
|
||||
data = load_s2l2a_with_offset(
|
||||
dc,
|
||||
query | load_params # Combine the two dicts that contain our search and load parameters
|
||||
catalog = pystac_client.Client.open(
|
||||
"https://planetarycomputer.microsoft.com/api/stac/v1",
|
||||
modifier=planetary_computer.sign_inplace,
|
||||
)
|
||||
search = catalog.search(
|
||||
collections=["sentinel-2-l2a"],
|
||||
bbox=bbox,
|
||||
datetime=f"{date_range[0]}/{date_range[1]}",
|
||||
)
|
||||
items = list(search.items())
|
||||
|
||||
data = odc.stac.load(
|
||||
items,
|
||||
bands=["red", "nir", "SCL"],
|
||||
bbox=bbox,
|
||||
crs="EPSG:32648",
|
||||
resolution=RESOLUTION,
|
||||
chunks={"x": 2048, "y": 2048, "time": 1},
|
||||
groupby="solar_day"
|
||||
)
|
||||
if "SCL" in data.data_vars:
|
||||
data = data.rename({"SCL": "scl"})
|
||||
return data
|
||||
|
||||
def mask_cloud(data):
|
||||
flag_name = 'scl'
|
||||
flag_desc = masking.describe_variable_flags(data[flag_name]) # Pandas dataframe
|
||||
display(flag_desc.loc['qa'].values[1])
|
||||
# Create a "data quality" Mask layer
|
||||
flags_def = flag_desc.loc['qa'].values[1]
|
||||
good_pixel_flags = [flags_def[str(i)] for i in [2, 4, 5, 6]] # To pass strings to enum_to_bool()
|
||||
|
||||
# enum_to_bool calculates the pixel-wise "or" of each set of pixels given by good_pixel_flags
|
||||
# 1 = good data
|
||||
# 0 = "bad" data
|
||||
good_pixel_mask = enum_to_bool(data[flag_name], good_pixel_flags)
|
||||
# For Sentinel-2 L2A SCL:
|
||||
# 2: Dark Area Pixels, 4: Vegetation, 5: Not Vegetated, 6: Water
|
||||
good_pixel_mask = data['scl'].isin([2, 4, 5, 6])
|
||||
data_layer_names = [x for x in data.data_vars if x != 'scl']
|
||||
# Apply good pixel mask to blue, green, red and nir.
|
||||
# Apply good pixel mask
|
||||
result = data[data_layer_names].where(good_pixel_mask).persist()
|
||||
return result
|
||||
|
||||
|
||||
+2926
-2239
File diff suppressed because one or more lines are too long
+264
@@ -0,0 +1,264 @@
|
||||
#!/usr/bin/env python
|
||||
# coding: utf-8
|
||||
|
||||
# In[49]:
|
||||
|
||||
|
||||
get_ipython().run_cell_magic('time', '', '%matplotlib inline\nfrom new_import import *\n')
|
||||
|
||||
|
||||
# In[2]:
|
||||
|
||||
|
||||
get_ipython().run_cell_magic('time', '', '# Cấu hình Daskgateway\ncluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,10))\n# Khai báo 1 Datacube là dc\ndc = datacube.Datacube()\n\n# Cấu hình truy cập dịch vụ S3\nconfigure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n\nclient\n')
|
||||
|
||||
|
||||
# LOAD VH, VV
|
||||
|
||||
# In[47]:
|
||||
|
||||
|
||||
## cấu hình thời gian lấy ảnh và tọa độ
|
||||
date_range = ('2022-09-01', '2023-10-01')
|
||||
longtitude_range = (105.5, 106.4)
|
||||
latitude_range = (9.2, 10.0)
|
||||
|
||||
|
||||
# In[3]:
|
||||
|
||||
|
||||
## cấu hình dữ liệu train và vh vv file
|
||||
train_path = "train/ST_training data_updated_1130points.shp" # đường dẫn shp file train
|
||||
name_vh = "vh-0922_0923-full_ST.tif"
|
||||
name_vv = "vv-0922_0923-full_ST.tif"
|
||||
|
||||
|
||||
train = load_train_data(train_path)
|
||||
|
||||
|
||||
# In[4]:
|
||||
|
||||
|
||||
# %%time
|
||||
# ## tải về dữ liệu sen1
|
||||
# import os
|
||||
# if not os.path.exists(name_vh):
|
||||
# !aws s3 cp s3://easi-asia-dc-data/staging/ctu/sentinel-1/vh-0922_0923-full_ST.tif vh-0922_0923-full_ST.tif
|
||||
# if not os.path.exists(name_vv):
|
||||
# !aws s3 cp s3://easi-asia-dc-data/staging/ctu/sentinel-1/vv-0922_0923-full_ST.tif vv-0922_0923-full_ST.tif
|
||||
|
||||
|
||||
# In[5]:
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# In[38]:
|
||||
|
||||
|
||||
ds = dc.load(
|
||||
product="sentinel1_grd_gamma0_20m",
|
||||
x=(105.5, 106.4),
|
||||
y=(9.2, 10.0),
|
||||
time=("2022-09-01", "2023-10-01"),
|
||||
measurements=["vv", "vh"],
|
||||
output_crs="EPSG:32648",
|
||||
resolution=(-10,10),
|
||||
dask_chunks={"x":2048, "y":2048},
|
||||
skip_broken_datasets=True,
|
||||
group_by="solar_day"
|
||||
)
|
||||
notebook_utils.heading(notebook_utils.xarray_object_size(ds))
|
||||
ds
|
||||
|
||||
|
||||
# In[43]:
|
||||
|
||||
|
||||
vv_data = ds.vv
|
||||
vv_data
|
||||
|
||||
|
||||
# In[44]:
|
||||
|
||||
|
||||
bbox = [105.5, 9.2, 106.4, 10.0]
|
||||
time_range = "2022-09-01/2023-10-01"
|
||||
dsvh, dsvv = load_sen1(bbox, time_range)
|
||||
dsvv
|
||||
|
||||
|
||||
# LOAD SENTINEL 2
|
||||
#
|
||||
#
|
||||
|
||||
# In[50]:
|
||||
|
||||
|
||||
data = load_data(dc, date_range, longtitude_range, latitude_range)
|
||||
notebook_utils.heading(notebook_utils.xarray_object_size(data))
|
||||
display(data)
|
||||
|
||||
|
||||
# In[8]:
|
||||
|
||||
|
||||
get_ipython().run_cell_magic('time', '', '# Tiến hành loại bỏ các vị trí bị mây ảnh hưởng\nresult = mask_clean(data)\nprogress(result)\n')
|
||||
|
||||
|
||||
# CALCULATING THE MEAN VALUE AND FILL TO NAN POINT
|
||||
|
||||
# In[9]:
|
||||
|
||||
|
||||
ds1 = calculate_indices(result, index='NDVI', satellite_mission='s2')
|
||||
ndvi = ds1["NDVI"]
|
||||
average_ndvi = ndvi.resample(time='1M').mean().persist() ## tính mean cho từng tháng -> time = 12
|
||||
progress(average_ndvi)
|
||||
|
||||
|
||||
# In[10]:
|
||||
|
||||
|
||||
dsvh.shape
|
||||
|
||||
|
||||
# In[11]:
|
||||
|
||||
|
||||
average_ndvi = average_ndvi.compute()
|
||||
average_ndvi = average_ndvi[:, :dsvh.shape[1], :dsvh.shape[2]]
|
||||
|
||||
|
||||
# In[12]:
|
||||
|
||||
|
||||
get_ipython().run_cell_magic('time', '', "filled_ds = average_ndvi.bfill(dim='time')\nfilled_ds = filled_ds.ffill(dim='time')\n")
|
||||
|
||||
|
||||
# FIND NAN POINT AFTER FILLING AND FILLING AGAIN WITH LINEARREGRESSION ALGORITHM
|
||||
|
||||
# In[13]:
|
||||
|
||||
|
||||
nan_mask = filled_ds.isnull()
|
||||
|
||||
# Print the NaN mask
|
||||
# print(nan_mask)
|
||||
|
||||
# Count the number of NaNs
|
||||
num_nans = nan_mask.sum()
|
||||
print(f'Number of NaNs: {num_nans.values}')
|
||||
|
||||
|
||||
# In[14]:
|
||||
|
||||
|
||||
from sklearn.preprocessing import PolynomialFeatures
|
||||
from sklearn.linear_model import LinearRegression
|
||||
from sklearn.ensemble import RandomForestRegressor
|
||||
|
||||
mask = ~np.isnan(filled_ds)
|
||||
X_train = np.stack([dsvh.values[mask], dsvv.values[mask]], axis=1)
|
||||
y_train = filled_ds.values[mask]
|
||||
|
||||
|
||||
# In[15]:
|
||||
|
||||
|
||||
model = LinearRegression()
|
||||
model.fit(X_train, y_train)
|
||||
|
||||
|
||||
# In[16]:
|
||||
|
||||
|
||||
X_pred = np.stack([dsvh.values[~mask], dsvv.values[~mask]], axis=1)
|
||||
filled_ds.values[~mask] = model.predict(X_pred)
|
||||
|
||||
|
||||
# MATCH LABEL TO DATASET
|
||||
|
||||
# In[17]:
|
||||
|
||||
|
||||
get_ipython().run_cell_magic('time', '', '\n# Takes 1 minute to complete.\nloaded_datasets = {}\nfor idx, point in train.iterrows():\n key = f"point_{idx + 1}"\n try:\n ndvi_data = filled_ds.sel(x=point.geometry.x, y=point.geometry.y, method=\'nearest\').values\n vh_data = dsvh.sel(x=point.geometry.x, y=point.geometry.y, method=\'nearest\').values\n vv_data = dsvv.sel(x=point.geometry.x, y=point.geometry.y, method=\'nearest\').values\n loaded_datasets[key] = {\n "data": np.concatenate((ndvi_data, vh_data, vv_data)),\n "label": point.HT_code\n }\n except Exception as e:\n # loaded_datasets[key] = None\n print(e)\n')
|
||||
|
||||
|
||||
# In[18]:
|
||||
|
||||
|
||||
label_mapping = {
|
||||
"Lua tom": "0",
|
||||
"Lua": "1",
|
||||
"CHN": "2",
|
||||
"CLN": "3",
|
||||
"TS": "4",
|
||||
"Song": "5",
|
||||
"Dat xay dung": "6",
|
||||
"Rung": "7"
|
||||
}
|
||||
label_encoder = LabelEncoder()
|
||||
|
||||
# Fit and transform the labels
|
||||
labels = train.Hientrang.values
|
||||
numeric_labels = label_encoder.fit_transform([label_mapping[label] for label in labels])
|
||||
|
||||
|
||||
# In[19]:
|
||||
|
||||
|
||||
X = []
|
||||
x_new = []
|
||||
lb_new = []
|
||||
for k, v in loaded_datasets.items():
|
||||
X.append(v)
|
||||
for i in range(len(X)):
|
||||
if X[i] is not None:
|
||||
x_new.append(X[i]["data"])
|
||||
lb_new.append(numeric_labels[i])
|
||||
|
||||
|
||||
# BUILDING DATASETS
|
||||
|
||||
# In[20]:
|
||||
|
||||
|
||||
X_train, X_temp, y_train, y_temp= train_test_split(x_new, lb_new, test_size=0.4, random_state=42)
|
||||
X_val, X_test, y_val, y_test = train_test_split(X_temp, y_temp, test_size=0.5, random_state=42)
|
||||
|
||||
|
||||
# TRAIN MODEL
|
||||
|
||||
# In[21]:
|
||||
|
||||
|
||||
get_ipython().run_cell_magic('time', '', 'from sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.svm import SVC\nfrom sklearn.metrics import accuracy_score\n\n# Define the models\nrf_model = RandomForestClassifier(random_state=42, n_jobs=-1)\nknn_model = KNeighborsClassifier()\nnb_model = GaussianNB()\nsvm_model = SVC()\n\n# Create a pipeline\npipeline = Pipeline([\n (\'scaler\', StandardScaler()), # Apply scaling\n (\'classifier\', rf_model) # Placeholder, will be set by param_grid\n])\n\n# Define the parameter grid for each classifier\nparam_grid = [\n # RandomForest\n {\n \'classifier\': [rf_model],\n \'classifier__n_estimators\': [100, 300, 500, 700],\n \'classifier__max_depth\': [6, 8, 10, 15],\n \'classifier__criterion\': [\'gini\', \'entropy\'],\n },\n # KNeighborsClassifier\n {\n \'classifier\': [knn_model],\n \'classifier__n_neighbors\': [3, 5, 7, 9],\n \'classifier__weights\': [\'uniform\', \'distance\'],\n \'classifier__metric\': [\'euclidean\', \'manhattan\']\n },\n # Naive Bayes (GaussianNB doesn\'t have hyperparameters to tune here)\n {\n \'classifier\': [nb_model],\n },\n # SVM\n {\n \'classifier\': [svm_model],\n \'classifier__C\': [0.1, 1, 10, 100],\n \'classifier__kernel\': [\'linear\', \'rbf\'],\n \'classifier__gamma\': [\'scale\', \'auto\']\n }\n]\n\n# Use GridSearchCV to find the best classifier and hyperparameters\ngrid_search = GridSearchCV(pipeline, param_grid, cv=5, scoring=\'accuracy\', n_jobs=-1)\ngrid_search.fit(X_train, y_train)\n\n# Print out the best parameters and classifier\nbest_params = grid_search.best_params_\nprint("Best Parameters:", best_params)\n\n# Make predictions on the validation set\ny_pred = grid_search.predict(X_val)\n\n# Evaluate the results\naccuracy = accuracy_score(y_val, y_pred)\nprint(f"Accuracy: {round(accuracy, 2)*100} %")\n')
|
||||
|
||||
|
||||
# In[22]:
|
||||
|
||||
|
||||
## check accuracy score
|
||||
|
||||
y_pred_test = grid_search.predict(X_test)
|
||||
test_accuracy = accuracy_score(y_test, y_pred_test)
|
||||
print(f"Accuracy for test data {round(test_accuracy, 2)*100} %")
|
||||
|
||||
|
||||
# In[23]:
|
||||
|
||||
|
||||
dir_save_model = "model_train"
|
||||
if not os.path.exists(dir_save_model):
|
||||
os.mkdir(dir_save_model)
|
||||
joblib.dump(grid_search, os.path.join(dir_save_model, "model_new2.joblib"))
|
||||
|
||||
|
||||
# In[24]:
|
||||
|
||||
|
||||
client.close()
|
||||
cluster.close()
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
import json
|
||||
import glob
|
||||
|
||||
def fix_load_sen1(file_path):
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
nb = json.load(f)
|
||||
|
||||
changed = False
|
||||
for cell in nb.get('cells', []):
|
||||
if cell.get('cell_type') == 'code':
|
||||
source = cell.get('source', [])
|
||||
for i, line in enumerate(source):
|
||||
if 'load_sen1(name_vh, name_vv)' in line:
|
||||
indent = line[:len(line) - len(line.lstrip())]
|
||||
replacement = (
|
||||
f"{indent}bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]\n"
|
||||
f"{indent}time_range = f'{{date_range[0]}}/{{date_range[1]}}'\n"
|
||||
f"{indent}{line.lstrip().replace('load_sen1(name_vh, name_vv)', 'load_sen1(bbox, time_range)')}"
|
||||
)
|
||||
source[i] = replacement
|
||||
changed = True
|
||||
|
||||
if changed:
|
||||
with open(file_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(nb, f, indent=1)
|
||||
print(f"Patched load_sen1 in {file_path}")
|
||||
|
||||
for nb in glob.glob("*.ipynb"):
|
||||
fix_load_sen1(nb)
|
||||
@@ -0,0 +1,43 @@
|
||||
import json
|
||||
import glob
|
||||
|
||||
def patch_notebook(file_path):
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
nb = json.load(f)
|
||||
|
||||
changed = False
|
||||
for cell in nb.get('cells', []):
|
||||
if cell.get('cell_type') == 'code':
|
||||
source = cell.get('source', [])
|
||||
|
||||
# Check if this cell should be fully commented out
|
||||
full_source = ''.join(source)
|
||||
if 'dc.load(' in full_source or 'ds.vv' in full_source:
|
||||
for i in range(len(source)):
|
||||
if not source[i].startswith('#'):
|
||||
source[i] = '# ' + source[i]
|
||||
changed = True
|
||||
continue
|
||||
|
||||
# Otherwise, do line-by-line replacements
|
||||
for i, line in enumerate(source):
|
||||
if 'ST_training data_updated_1130points.shp' in line:
|
||||
source[i] = line.replace('ST_training data_updated_1130points.shp', 'ST_training_data_updated_1130points.shp')
|
||||
changed = True
|
||||
if 'from new_import import *' in line:
|
||||
source[i] = line.replace('from new_import import *', 'from new_import_ODC import *')
|
||||
changed = True
|
||||
if 'dc = datacube.Datacube()' in line:
|
||||
source[i] = line.replace('dc = datacube.Datacube()', 'dc = None')
|
||||
changed = True
|
||||
if 'load_data(dc,' in line:
|
||||
source[i] = line.replace('load_data(dc,', 'load_data(None,')
|
||||
changed = True
|
||||
|
||||
if changed:
|
||||
with open(file_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(nb, f, indent=1)
|
||||
print(f"Patched {file_path}")
|
||||
|
||||
for nb in glob.glob("*.ipynb"):
|
||||
patch_notebook(nb)
|
||||
+45
-45
@@ -1132,7 +1132,7 @@
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"%matplotlib inline\n",
|
||||
"from new_import import *"
|
||||
"from new_import_ODC import *"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1216,7 +1216,7 @@
|
||||
"%%time\n",
|
||||
"# Dask gateway\n",
|
||||
"cluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))\n",
|
||||
"dc = datacube.Datacube()\n",
|
||||
"dc = None\n",
|
||||
"\n",
|
||||
"# Configure s3 access\n",
|
||||
"configure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n",
|
||||
@@ -1233,11 +1233,11 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"## cấu hình thời gian lấy ảnh và tọa độ\n",
|
||||
"# date_range = ('2022-09-01', '2023-10-01')\n",
|
||||
"## c\u1ea5u h\u00ecnh th\u1eddi gian l\u1ea5y \u1ea3nh v\u00e0 t\u1ecda \u0111\u1ed9\n",
|
||||
"# date_range = ('2022-09-01', '2022-10-01')\n",
|
||||
"# longtitude_range = (105.86575, 105.94120)\n",
|
||||
"# latitude_range = (9.65070, 9.69850)\n",
|
||||
"date_range = ('2022-09-01', '2023-10-01')\n",
|
||||
"date_range = ('2022-09-01', '2022-10-01')\n",
|
||||
"longtitude_range = (105.5, 106.4)\n",
|
||||
"latitude_range = (9.2, 10.0) "
|
||||
]
|
||||
@@ -1396,7 +1396,7 @@
|
||||
"\n",
|
||||
".xr-section-summary-in + label:before {\n",
|
||||
" display: inline-block;\n",
|
||||
" content: '►';\n",
|
||||
" content: '\u25ba';\n",
|
||||
" font-size: 11px;\n",
|
||||
" width: 15px;\n",
|
||||
" text-align: center;\n",
|
||||
@@ -1407,7 +1407,7 @@
|
||||
"}\n",
|
||||
"\n",
|
||||
".xr-section-summary-in:checked + label:before {\n",
|
||||
" content: '▼';\n",
|
||||
" content: '\u25bc';\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
".xr-section-summary-in:checked + label > span {\n",
|
||||
@@ -2424,8 +2424,8 @@
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"## truy vấn ảnh vệ tinh sen2\n",
|
||||
"data = load_data(dc, date_range, longtitude_range, latitude_range)\n",
|
||||
"## truy v\u1ea5n \u1ea3nh v\u1ec7 tinh sen2\n",
|
||||
"data = load_data(None, date_range, longtitude_range, latitude_range)\n",
|
||||
"notebook_utils.heading(notebook_utils.xarray_object_size(data))\n",
|
||||
"display(data)"
|
||||
]
|
||||
@@ -2440,10 +2440,10 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Specify the start and end times \n",
|
||||
"min_date = '2022-09-01' # Thời gian bắt đầu lấy data cho quá trình train\n",
|
||||
"max_date = '2023-10-01' # Thời gian kết thúc lấy data cho quá trình train\n",
|
||||
"min_date = '2022-09-01' # Th\u1eddi gian b\u1eaft \u0111\u1ea7u l\u1ea5y data cho qu\u00e1 tr\u00ecnh train\n",
|
||||
"max_date = '2022-10-01' # Th\u1eddi gian k\u1ebft th\u00fac l\u1ea5y data cho qu\u00e1 tr\u00ecnh train\n",
|
||||
"# Just do 1 month for testing\n",
|
||||
"# max_date = '2022-10-01' # Thời gian kết thúc lấy data cho quá trình train\n",
|
||||
"# max_date = '2022-10-01' # Th\u1eddi gian k\u1ebft th\u00fac l\u1ea5y data cho qu\u00e1 tr\u00ecnh train\n",
|
||||
"\n",
|
||||
"# Specify a spatail region to search using latitude/longitude cooridinates\n",
|
||||
"min_longitude, max_longitude = (105.5, 106.4)\n",
|
||||
@@ -2657,7 +2657,7 @@
|
||||
"\n",
|
||||
".xr-section-summary-in + label:before {\n",
|
||||
" display: inline-block;\n",
|
||||
" content: '►';\n",
|
||||
" content: '\u25ba';\n",
|
||||
" font-size: 11px;\n",
|
||||
" width: 15px;\n",
|
||||
" text-align: center;\n",
|
||||
@@ -2668,7 +2668,7 @@
|
||||
"}\n",
|
||||
"\n",
|
||||
".xr-section-summary-in:checked + label:before {\n",
|
||||
" content: '▼';\n",
|
||||
" content: '\u25bc';\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
".xr-section-summary-in:checked + label > span {\n",
|
||||
@@ -3423,23 +3423,23 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"CPU times: user 1.57 s, sys: 177 µs, total: 1.57 s\n",
|
||||
"CPU times: user 1.57 s, sys: 177 \u00b5s, total: 1.57 s\n",
|
||||
"Wall time: 1.66 s\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"# %%time\n",
|
||||
"# The replacement \"dc.load()\" function for this product\n",
|
||||
"data = load_s2l2a_with_offset(\n",
|
||||
" dc,\n",
|
||||
" query | load_params # Combine the two dicts that contain our search and load parameters\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# data = load_s2l2a_with_offset(\n",
|
||||
"# dc,\n",
|
||||
"# query | load_params # Combine the two dicts that contain our search and load parameters\n",
|
||||
"# )\n",
|
||||
"# \n",
|
||||
"# This line prints the total size of the dataset hat was loaded\n",
|
||||
"notebook_utils.heading(notebook_utils.xarray_object_size(data))\n",
|
||||
"\n",
|
||||
"display(data)"
|
||||
"# notebook_utils.heading(notebook_utils.xarray_object_size(data))\n",
|
||||
"# \n",
|
||||
"# display(data)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -3546,9 +3546,9 @@
|
||||
],
|
||||
"source": [
|
||||
"# %%time\n",
|
||||
"# # Tiến hành loại bỏ các vị trí bị mây ảnh hưởng\n",
|
||||
"# # Ti\u1ebfn h\u00e0nh lo\u1ea1i b\u1ecf c\u00e1c v\u1ecb tr\u00ed b\u1ecb m\u00e2y \u1ea3nh h\u01b0\u1edfng\n",
|
||||
"# result = mask_clean(data)\n",
|
||||
"# progress(result)"
|
||||
"# # progress(result)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -3684,7 +3684,7 @@
|
||||
"\n",
|
||||
".xr-section-summary-in + label:before {\n",
|
||||
" display: inline-block;\n",
|
||||
" content: '►';\n",
|
||||
" content: '\u25ba';\n",
|
||||
" font-size: 11px;\n",
|
||||
" width: 15px;\n",
|
||||
" text-align: center;\n",
|
||||
@@ -3695,7 +3695,7 @@
|
||||
"}\n",
|
||||
"\n",
|
||||
".xr-section-summary-in:checked + label:before {\n",
|
||||
" content: '▼';\n",
|
||||
" content: '\u25bc';\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
".xr-section-summary-in:checked + label > span {\n",
|
||||
@@ -4174,7 +4174,7 @@
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Tiến hành tính toán NDVI\n",
|
||||
"# Ti\u1ebfn h\u00e0nh t\u00ednh to\u00e1n NDVI\n",
|
||||
"ds1 = calculate_indices(result, index='NDVI', satellite_mission='s2')\n",
|
||||
"ndvi = ds1[\"NDVI\"]\n",
|
||||
"display(ndvi)"
|
||||
@@ -4213,9 +4213,9 @@
|
||||
],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"## tính ndvi theo tháng\n",
|
||||
"## t\u00ednh ndvi theo th\u00e1ng\n",
|
||||
"average_ndvi = ndvi.resample(time='1M').mean().persist()\n",
|
||||
"progress(average_ndvi)"
|
||||
"# progress(average_ndvi)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -4253,12 +4253,12 @@
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# cấu hình vh vv file\n",
|
||||
"# c\u1ea5u h\u00ecnh vh vv file\n",
|
||||
"# name_vh = \"ThuanHoa/ThuanHoa_VH.tif\"\n",
|
||||
"# name_vv = \"ThuanHoa/ThuanHoa_VV.tif\"\n",
|
||||
"\n",
|
||||
"# load dữ liệu sen1\n",
|
||||
"# dsvh, dsvv = load_sen1(name_vh, name_vv)\n",
|
||||
"# load d\u1eef li\u1ec7u sen1\n",
|
||||
"bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]\ntime_range = f'{date_range[0]}/{date_range[1]}'\n# dsvh, dsvv = load_sen1(bbox, time_range)\n",
|
||||
"\n",
|
||||
"name_vh = \"vh-0922_0923-full_ST.tif\"\n",
|
||||
"name_vv = \"vv-0922_0923-full_ST.tif\"\n",
|
||||
@@ -4268,7 +4268,7 @@
|
||||
"if not os.path.exists(name_vv):\n",
|
||||
" !aws s3 cp s3://easi-asia-dc-data/staging/ctu/sentinel-1/vv-0922_0923-full_ST.tif vv-0922_0923-full_ST.tif\n",
|
||||
" \n",
|
||||
"dsvh, dsvv = load_sen1(name_vh, name_vv)"
|
||||
"bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]\ntime_range = f'{date_range[0]}/{date_range[1]}'\ndsvh, dsvv = load_sen1(bbox, time_range)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -4326,7 +4326,7 @@
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<style>#sk-container-id-1 {color: black;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>LinearRegression()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LinearRegression</label><div class=\"sk-toggleable__content\"><pre>LinearRegression()</pre></div></div></div></div></div>"
|
||||
"<style>#sk-container-id-1 {color: black;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"\u25b8\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"\u25be\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>LinearRegression()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LinearRegression</label><div class=\"sk-toggleable__content\"><pre>LinearRegression()</pre></div></div></div></div></div>"
|
||||
],
|
||||
"text/plain": [
|
||||
"LinearRegression()"
|
||||
@@ -4397,7 +4397,7 @@
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"plt.imshow(average_ndvi_filled.isel(time=6))"
|
||||
"plt.imshow(average_ndvi_filled.isel(time=0))"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -4430,7 +4430,7 @@
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"plt.imshow(average_ndvi.isel(time=6))"
|
||||
"plt.imshow(average_ndvi.isel(time=0))"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -4440,7 +4440,7 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"train_path = \"train/ST_training data_updated_1130points.shp\""
|
||||
"train_path = \"train/ST_training_data_updated_1130points.shp\""
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -4476,7 +4476,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# cấu hình nhãn dữ liệu\n",
|
||||
"# c\u1ea5u h\u00ecnh nh\u00e3n d\u1eef li\u1ec7u\n",
|
||||
"label_mapping = {\n",
|
||||
" \"Lua tom\": \"0\",\n",
|
||||
" \"Lua\": \"1\",\n",
|
||||
@@ -4488,7 +4488,7 @@
|
||||
" \"Rung\": \"7\"\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"# chia tập dữ liệu train, val, test\n",
|
||||
"# chia t\u1eadp d\u1eef li\u1ec7u train, val, test\n",
|
||||
"X_train, X_val, X_test, y_train, y_val, y_test = split_train_data(train, label_mapping, datasets)"
|
||||
]
|
||||
},
|
||||
@@ -4510,7 +4510,7 @@
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Huấn luyện mô hình\n",
|
||||
"# Hu\u1ea5n luy\u1ec7n m\u00f4 h\u00ecnh\n",
|
||||
"grid_search = train_with_rf(X_train, X_val, y_train, y_val)"
|
||||
]
|
||||
},
|
||||
@@ -4531,7 +4531,7 @@
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# kiểm tra độ chính xác với tập test\n",
|
||||
"# ki\u1ec3m tra \u0111\u1ed9 ch\u00ednh x\u00e1c v\u1edbi t\u1eadp test\n",
|
||||
"y_pred_test = grid_search.predict(X_test)\n",
|
||||
"test_accuracy = accuracy_score(y_test, y_pred_test)\n",
|
||||
"print(f\"Accuracy for test data {round(test_accuracy, 2)*100} %\")"
|
||||
@@ -4554,7 +4554,7 @@
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Lưu mô hình huấn luyện\n",
|
||||
"# L\u01b0u m\u00f4 h\u00ecnh hu\u1ea5n luy\u1ec7n\n",
|
||||
"save_model(\"model_new.joblib\", grid_search)"
|
||||
]
|
||||
},
|
||||
@@ -4567,7 +4567,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# đóng client, cluster\n",
|
||||
"# \u0111\u00f3ng client, cluster\n",
|
||||
"client.close()\n",
|
||||
"cluster.close()"
|
||||
]
|
||||
|
||||
@@ -0,0 +1,272 @@
|
||||
#!/usr/bin/env python
|
||||
# coding: utf-8
|
||||
|
||||
# In[1]:
|
||||
|
||||
|
||||
get_ipython().run_cell_magic('time', '', '%matplotlib inline\nfrom new_import import *\n')
|
||||
|
||||
|
||||
# In[2]:
|
||||
|
||||
|
||||
get_ipython().run_cell_magic('time', '', '# Dask gateway\ncluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))\ndc = datacube.Datacube()\n\n# Configure s3 access\nconfigure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n\nclient\n')
|
||||
|
||||
|
||||
# In[3]:
|
||||
|
||||
|
||||
## cấu hình thời gian lấy ảnh và tọa độ
|
||||
# date_range = ('2022-09-01', '2023-10-01')
|
||||
# longtitude_range = (105.86575, 105.94120)
|
||||
# latitude_range = (9.65070, 9.69850)
|
||||
date_range = ('2022-09-01', '2023-10-01')
|
||||
longtitude_range = (105.5, 106.4)
|
||||
latitude_range = (9.2, 10.0)
|
||||
|
||||
|
||||
# In[4]:
|
||||
|
||||
|
||||
## truy vấn ảnh vệ tinh sen2
|
||||
data = load_data(dc, date_range, longtitude_range, latitude_range)
|
||||
notebook_utils.heading(notebook_utils.xarray_object_size(data))
|
||||
display(data)
|
||||
|
||||
|
||||
# In[5]:
|
||||
|
||||
|
||||
# Specify the start and end times
|
||||
min_date = '2022-09-01' # Thời gian bắt đầu lấy data cho quá trình train
|
||||
max_date = '2023-10-01' # Thời gian kết thúc lấy data cho quá trình train
|
||||
# Just do 1 month for testing
|
||||
# max_date = '2022-10-01' # Thời gian kết thúc lấy data cho quá trình train
|
||||
|
||||
# Specify a spatail region to search using latitude/longitude cooridinates
|
||||
min_longitude, max_longitude = (105.5, 106.4)
|
||||
min_latitude, max_latitude = (9.2, 10.0)
|
||||
|
||||
# Specify the product. In this case we want to use Sentinel-2 Level-2A data
|
||||
product = 's2_l2a'
|
||||
|
||||
# Construct the search query dictionary
|
||||
query = {
|
||||
'product': product, # Product name
|
||||
'x': (min_longitude, max_longitude), # "x" axis bounds
|
||||
'y': (min_latitude, max_latitude), # "y" axis bounds
|
||||
'time': (min_date, max_date), # Any parsable date strings
|
||||
}
|
||||
|
||||
|
||||
# In[6]:
|
||||
|
||||
|
||||
# Most common CRS
|
||||
native_crs = notebook_utils.mostcommon_crs(dc, query)
|
||||
print(f'Most common native CRS: {native_crs}')
|
||||
|
||||
|
||||
# In[7]:
|
||||
|
||||
|
||||
# Specify the spectral band measurements we want to use for a classification algorithm
|
||||
measurements = ['red', 'nir', 'scl']
|
||||
|
||||
load_params = {
|
||||
'measurements': measurements, # Selected measurement or alias names
|
||||
'output_crs': native_crs, # Target EPSG code
|
||||
'resolution': (-10, 10), # Target resolution
|
||||
'group_by': 'solar_day', # Scene grouping
|
||||
'dask_chunks': {'x': 2048, 'y': 2048}, # Dask chunks
|
||||
}
|
||||
|
||||
|
||||
# In[8]:
|
||||
|
||||
|
||||
get_ipython().run_cell_magic('time', '', '# The replacement "dc.load()" function for this product\ndata = load_s2l2a_with_offset(\n dc,\n query | load_params # Combine the two dicts that contain our search and load parameters\n)\n\n# This line prints the total size of the dataset hat was loaded\nnotebook_utils.heading(notebook_utils.xarray_object_size(data))\n\ndisplay(data)\n')
|
||||
|
||||
|
||||
# In[9]:
|
||||
|
||||
|
||||
# %%time
|
||||
# # Tiến hành loại bỏ các vị trí bị mây ảnh hưởng
|
||||
# result = mask_clean(data)
|
||||
# progress(result)
|
||||
|
||||
|
||||
# In[10]:
|
||||
|
||||
|
||||
# Tiến hành tính toán NDVI
|
||||
ds1 = calculate_indices(result, index='NDVI', satellite_mission='s2')
|
||||
ndvi = ds1["NDVI"]
|
||||
display(ndvi)
|
||||
|
||||
|
||||
# In[11]:
|
||||
|
||||
|
||||
get_ipython().run_cell_magic('time', '', "## tính ndvi theo tháng\naverage_ndvi = ndvi.resample(time='1M').mean().persist()\nprogress(average_ndvi)\n")
|
||||
|
||||
|
||||
# In[12]:
|
||||
|
||||
|
||||
# compute average_ndvi
|
||||
average_ndvi = average_ndvi.compute()
|
||||
|
||||
|
||||
# In[13]:
|
||||
|
||||
|
||||
# cấu hình vh vv file
|
||||
# name_vh = "ThuanHoa/ThuanHoa_VH.tif"
|
||||
# name_vv = "ThuanHoa/ThuanHoa_VV.tif"
|
||||
|
||||
# load dữ liệu sen1
|
||||
bbox = [105.5, 9.2, 106.4, 10.0]
|
||||
time_range = '2022-09-01/2023-10-01'
|
||||
# dsvh, dsvv = load_sen1(bbox, time_range)
|
||||
|
||||
name_vh = "vh-0922_0923-full_ST.tif"
|
||||
name_vv = "vv-0922_0923-full_ST.tif"
|
||||
|
||||
if not os.path.exists(name_vh):
|
||||
get_ipython().system('aws s3 cp s3://easi-asia-dc-data/staging/ctu/sentinel-1/vh-0922_0923-full_ST.tif vh-0922_0923-full_ST.tif')
|
||||
if not os.path.exists(name_vv):
|
||||
get_ipython().system('aws s3 cp s3://easi-asia-dc-data/staging/ctu/sentinel-1/vv-0922_0923-full_ST.tif vv-0922_0923-full_ST.tif')
|
||||
|
||||
bbox = [105.5, 9.2, 106.4, 10.0]
|
||||
time_range = '2022-09-01/2023-10-01'
|
||||
dsvh, dsvv = load_sen1(bbox, time_range)
|
||||
|
||||
|
||||
# In[27]:
|
||||
|
||||
|
||||
from sklearn.preprocessing import PolynomialFeatures
|
||||
from sklearn.linear_model import LinearRegression
|
||||
from sklearn.ensemble import RandomForestRegressor
|
||||
|
||||
|
||||
# In[28]:
|
||||
|
||||
|
||||
average_ndvi = average_ndvi[:, :7680, :8687]
|
||||
mask = ~np.isnan(average_ndvi)
|
||||
print(average_ndvi.shape)
|
||||
print(dsvh.shape)
|
||||
print(dsvv.shape)
|
||||
print(mask.shape)
|
||||
X_train = np.stack([dsvh.values[mask], dsvv.values[mask]], axis=1)
|
||||
y_train = average_ndvi.values[mask]
|
||||
|
||||
|
||||
# In[29]:
|
||||
|
||||
|
||||
model = LinearRegression()
|
||||
model.fit(X_train, y_train)
|
||||
|
||||
|
||||
# In[30]:
|
||||
|
||||
|
||||
X_pred = np.stack([dsvh.values[~mask], dsvv.values[~mask]], axis=1)
|
||||
average_ndvi.values[~mask] = model.predict(X_pred)
|
||||
|
||||
|
||||
# In[31]:
|
||||
|
||||
|
||||
average_ndvi_filled = xr.DataArray(average_ndvi, dims=average_ndvi.dims)
|
||||
|
||||
|
||||
# In[32]:
|
||||
|
||||
|
||||
plt.imshow(average_ndvi_filled.isel(time=6))
|
||||
|
||||
|
||||
# In[65]:
|
||||
|
||||
|
||||
plt.imshow(average_ndvi.isel(time=6))
|
||||
|
||||
|
||||
# In[33]:
|
||||
|
||||
|
||||
train_path = "train/ST_training data_updated_1130points.shp"
|
||||
|
||||
|
||||
# In[34]:
|
||||
|
||||
|
||||
train = load_train_data(train_path)
|
||||
|
||||
|
||||
# In[37]:
|
||||
|
||||
|
||||
datasets = get_data_sen1_and_sen2(train, average_ndvi_filled, dsvh, dsvv)
|
||||
|
||||
|
||||
# In[39]:
|
||||
|
||||
|
||||
# cấu hình nhãn dữ liệu
|
||||
label_mapping = {
|
||||
"Lua tom": "0",
|
||||
"Lua": "1",
|
||||
"CHN": "2",
|
||||
"CLN": "3",
|
||||
"TS": "4",
|
||||
"Song": "5",
|
||||
"Dat xay dung": "6",
|
||||
"Rung": "7"
|
||||
}
|
||||
|
||||
# chia tập dữ liệu train, val, test
|
||||
X_train, X_val, X_test, y_train, y_val, y_test = split_train_data(train, label_mapping, datasets)
|
||||
|
||||
|
||||
# In[40]:
|
||||
|
||||
|
||||
# Huấn luyện mô hình
|
||||
grid_search = train_with_rf(X_train, X_val, y_train, y_val)
|
||||
|
||||
|
||||
# In[41]:
|
||||
|
||||
|
||||
# kiểm tra độ chính xác với tập test
|
||||
y_pred_test = grid_search.predict(X_test)
|
||||
test_accuracy = accuracy_score(y_test, y_pred_test)
|
||||
print(f"Accuracy for test data {round(test_accuracy, 2)*100} %")
|
||||
|
||||
|
||||
# In[42]:
|
||||
|
||||
|
||||
# Lưu mô hình huấn luyện
|
||||
save_model("model_new.joblib", grid_search)
|
||||
|
||||
|
||||
# In[43]:
|
||||
|
||||
|
||||
# đóng client, cluster
|
||||
client.close()
|
||||
cluster.close()
|
||||
|
||||
|
||||
# In[ ]:
|
||||
|
||||
|
||||
|
||||
|
||||
+19
-19
@@ -1132,7 +1132,7 @@
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"%matplotlib inline\n",
|
||||
"from new_import import *"
|
||||
"from new_import_ODC import *"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1216,7 +1216,7 @@
|
||||
"%%time\n",
|
||||
"# Dask gateway\n",
|
||||
"cluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))\n",
|
||||
"dc = datacube.Datacube()\n",
|
||||
"dc = None\n",
|
||||
"\n",
|
||||
"# Configure s3 access\n",
|
||||
"configure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n",
|
||||
@@ -1233,8 +1233,8 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"## cấu hình thời gian lấy ảnh và tọa độ\n",
|
||||
"date_range = ('2022-09-01', '2023-10-01')\n",
|
||||
"## c\u1ea5u h\u00ecnh th\u1eddi gian l\u1ea5y \u1ea3nh v\u00e0 t\u1ecda \u0111\u1ed9\n",
|
||||
"date_range = ('2022-09-01', '2022-10-01')\n",
|
||||
"longtitude_range = (105.86575, 105.94120)\n",
|
||||
"latitude_range = (9.65070, 9.69850)"
|
||||
]
|
||||
@@ -1393,7 +1393,7 @@
|
||||
"\n",
|
||||
".xr-section-summary-in + label:before {\n",
|
||||
" display: inline-block;\n",
|
||||
" content: '►';\n",
|
||||
" content: '\u25ba';\n",
|
||||
" font-size: 11px;\n",
|
||||
" width: 15px;\n",
|
||||
" text-align: center;\n",
|
||||
@@ -1404,7 +1404,7 @@
|
||||
"}\n",
|
||||
"\n",
|
||||
".xr-section-summary-in:checked + label:before {\n",
|
||||
" content: '▼';\n",
|
||||
" content: '\u25bc';\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
".xr-section-summary-in:checked + label > span {\n",
|
||||
@@ -2333,8 +2333,8 @@
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"## truy vấn ảnh vệ tinh sen2\n",
|
||||
"data = load_data(dc, date_range, longtitude_range, latitude_range)\n",
|
||||
"## truy v\u1ea5n \u1ea3nh v\u1ec7 tinh sen2\n",
|
||||
"data = load_data(None, date_range, longtitude_range, latitude_range)\n",
|
||||
"notebook_utils.heading(notebook_utils.xarray_object_size(data))\n",
|
||||
"display(data)"
|
||||
]
|
||||
@@ -2443,9 +2443,9 @@
|
||||
],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"# Tiến hành loại bỏ các vị trí bị mây ảnh hưởng\n",
|
||||
"# Ti\u1ebfn h\u00e0nh lo\u1ea1i b\u1ecf c\u00e1c v\u1ecb tr\u00ed b\u1ecb m\u00e2y \u1ea3nh h\u01b0\u1edfng\n",
|
||||
"result = mask_clean(data)\n",
|
||||
"progress(result)"
|
||||
"# progress(result)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -2581,7 +2581,7 @@
|
||||
"\n",
|
||||
".xr-section-summary-in + label:before {\n",
|
||||
" display: inline-block;\n",
|
||||
" content: '►';\n",
|
||||
" content: '\u25ba';\n",
|
||||
" font-size: 11px;\n",
|
||||
" width: 15px;\n",
|
||||
" text-align: center;\n",
|
||||
@@ -2592,7 +2592,7 @@
|
||||
"}\n",
|
||||
"\n",
|
||||
".xr-section-summary-in:checked + label:before {\n",
|
||||
" content: '▼';\n",
|
||||
" content: '\u25bc';\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
".xr-section-summary-in:checked + label > span {\n",
|
||||
@@ -3047,7 +3047,7 @@
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Tiến hành tính toán NDVI\n",
|
||||
"# Ti\u1ebfn h\u00e0nh t\u00ednh to\u00e1n NDVI\n",
|
||||
"ds1 = calculate_indices(result, index='NDVI', satellite_mission='s2')\n",
|
||||
"ndvi = ds1[\"NDVI\"]\n",
|
||||
"display(ndvi)"
|
||||
@@ -3086,9 +3086,9 @@
|
||||
],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"## tính ndvi theo tháng\n",
|
||||
"## t\u00ednh ndvi theo th\u00e1ng\n",
|
||||
"average_ndvi = ndvi.resample(time='1M').mean().persist()\n",
|
||||
"progress(average_ndvi)"
|
||||
"# progress(average_ndvi)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -3113,12 +3113,12 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# cấu hình vh vv file\n",
|
||||
"# c\u1ea5u h\u00ecnh vh vv file\n",
|
||||
"name_vh = \"ThuanHoa/ThuanHoa_VH.tif\"\n",
|
||||
"name_vv = \"ThuanHoa/ThuanHoa_VV.tif\"\n",
|
||||
"\n",
|
||||
"# load dữ liệu sen1\n",
|
||||
"dsvh, dsvv = load_sen1(name_vh, name_vv)"
|
||||
"# load d\u1eef li\u1ec7u sen1\n",
|
||||
"bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]\ntime_range = f'{date_range[0]}/{date_range[1]}'\ndsvh, dsvv = load_sen1(bbox, time_range)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -3160,7 +3160,7 @@
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<style>#sk-container-id-1 {color: black;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>LinearRegression()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LinearRegression</label><div class=\"sk-toggleable__content\"><pre>LinearRegression()</pre></div></div></div></div></div>"
|
||||
"<style>#sk-container-id-1 {color: black;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"\u25b8\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"\u25be\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>LinearRegression()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LinearRegression</label><div class=\"sk-toggleable__content\"><pre>LinearRegression()</pre></div></div></div></div></div>"
|
||||
],
|
||||
"text/plain": [
|
||||
"LinearRegression()"
|
||||
|
||||
+127
@@ -0,0 +1,127 @@
|
||||
#!/usr/bin/env python
|
||||
# coding: utf-8
|
||||
|
||||
# In[1]:
|
||||
|
||||
|
||||
get_ipython().run_cell_magic('time', '', '%matplotlib inline\nfrom new_import import *\n')
|
||||
|
||||
|
||||
# In[2]:
|
||||
|
||||
|
||||
get_ipython().run_cell_magic('time', '', '# Dask gateway\ncluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))\ndc = datacube.Datacube()\n\n# Configure s3 access\nconfigure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n\nclient\n')
|
||||
|
||||
|
||||
# In[3]:
|
||||
|
||||
|
||||
## cấu hình thời gian lấy ảnh và tọa độ
|
||||
date_range = ('2022-09-01', '2023-10-01')
|
||||
longtitude_range = (105.86575, 105.94120)
|
||||
latitude_range = (9.65070, 9.69850)
|
||||
|
||||
|
||||
# In[4]:
|
||||
|
||||
|
||||
## truy vấn ảnh vệ tinh sen2
|
||||
data = load_data(dc, date_range, longtitude_range, latitude_range)
|
||||
notebook_utils.heading(notebook_utils.xarray_object_size(data))
|
||||
display(data)
|
||||
|
||||
|
||||
# In[5]:
|
||||
|
||||
|
||||
get_ipython().run_cell_magic('time', '', '# Tiến hành loại bỏ các vị trí bị mây ảnh hưởng\nresult = mask_clean(data)\nprogress(result)\n')
|
||||
|
||||
|
||||
# In[6]:
|
||||
|
||||
|
||||
# Tiến hành tính toán NDVI
|
||||
ds1 = calculate_indices(result, index='NDVI', satellite_mission='s2')
|
||||
ndvi = ds1["NDVI"]
|
||||
display(ndvi)
|
||||
|
||||
|
||||
# In[17]:
|
||||
|
||||
|
||||
get_ipython().run_cell_magic('time', '', "## tính ndvi theo tháng\naverage_ndvi = ndvi.resample(time='1M').mean().persist()\nprogress(average_ndvi)\n")
|
||||
|
||||
|
||||
# In[18]:
|
||||
|
||||
|
||||
# compute average_ndvi
|
||||
average_ndvi = average_ndvi.compute()
|
||||
|
||||
|
||||
# In[9]:
|
||||
|
||||
|
||||
# cấu hình vh vv file
|
||||
name_vh = "ThuanHoa/ThuanHoa_VH.tif"
|
||||
name_vv = "ThuanHoa/ThuanHoa_VV.tif"
|
||||
|
||||
# load dữ liệu sen1
|
||||
bbox = [105.5, 9.2, 106.4, 10.0]
|
||||
time_range = '2022-09-01/2023-10-01'
|
||||
dsvh, dsvv = load_sen1(bbox, time_range)
|
||||
|
||||
|
||||
# In[10]:
|
||||
|
||||
|
||||
from sklearn.preprocessing import PolynomialFeatures
|
||||
from sklearn.linear_model import LinearRegression
|
||||
from sklearn.ensemble import RandomForestRegressor
|
||||
|
||||
|
||||
# In[11]:
|
||||
|
||||
|
||||
mask = ~np.isnan(average_ndvi)
|
||||
X_train = np.stack([dsvh.values[mask], dsvv.values[mask]], axis=1)
|
||||
y_train = average_ndvi.values[mask]
|
||||
|
||||
|
||||
# In[12]:
|
||||
|
||||
|
||||
model = LinearRegression()
|
||||
model.fit(X_train, y_train)
|
||||
|
||||
|
||||
# In[13]:
|
||||
|
||||
|
||||
X_pred = np.stack([dsvh.values[~mask], dsvv.values[~mask]], axis=1)
|
||||
average_ndvi.values[~mask] = model.predict(X_pred)
|
||||
|
||||
|
||||
# In[14]:
|
||||
|
||||
|
||||
average_ndvi_filled = xr.DataArray(average_ndvi, dims=average_ndvi.dims)
|
||||
|
||||
|
||||
# In[16]:
|
||||
|
||||
|
||||
plt.imshow(average_ndvi_filled.isel(time=1))
|
||||
|
||||
|
||||
# In[19]:
|
||||
|
||||
|
||||
plt.imshow(average_ndvi.isel(time=1))
|
||||
|
||||
|
||||
# In[ ]:
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
import json
|
||||
import glob
|
||||
|
||||
def fix_notebook(file_path):
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
nb = json.load(f)
|
||||
changed = False
|
||||
for cell in nb.get('cells', []):
|
||||
if cell.get('cell_type') == 'code':
|
||||
source = cell.get('source', [])
|
||||
if isinstance(source, list):
|
||||
for i, line in enumerate(source):
|
||||
if "dc = datacube.Datacube()" in line:
|
||||
source[i] = "dc = None\n"
|
||||
changed = True
|
||||
if "ds = dc.load(" in line:
|
||||
source[i] = "ds = None\n"
|
||||
changed = True
|
||||
if "data = dc.load(" in line:
|
||||
source[i] = "data = None\n"
|
||||
changed = True
|
||||
# If ds is None, ds.vv will fail
|
||||
if "vv_data = ds.vv" in line:
|
||||
source[i] = "vv_data = None\n"
|
||||
changed = True
|
||||
if "notebook_utils.xarray_object_size(ds)" in line:
|
||||
source[i] = line.replace("notebook_utils.xarray_object_size(ds)", "'ds is None'")
|
||||
changed = True
|
||||
if changed:
|
||||
with open(file_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(nb, f, indent=1)
|
||||
print(f"Removed datacube from {file_path}")
|
||||
|
||||
for nb in glob.glob("*.ipynb"):
|
||||
fix_notebook(nb)
|
||||
@@ -0,0 +1,82 @@
|
||||
import json
|
||||
import glob
|
||||
import subprocess
|
||||
import time
|
||||
import os
|
||||
|
||||
NOTEBOOKS_TO_RUN = [
|
||||
"01.train_ODC.ipynb",
|
||||
"01.train_ODC_XGBoost.ipynb",
|
||||
"02.predict_ODC.ipynb",
|
||||
"new_train.ipynb"
|
||||
]
|
||||
|
||||
def limit_time_range(file_path):
|
||||
try:
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
nb = json.load(f)
|
||||
|
||||
changed = False
|
||||
for cell in nb.get('cells', []):
|
||||
if cell.get('cell_type') == 'code':
|
||||
source = cell.get('source', [])
|
||||
if isinstance(source, list):
|
||||
for i, line in enumerate(source):
|
||||
# Replace 2023-12-31 with 2023-04-01
|
||||
if '"2023-12-31"' in line:
|
||||
source[i] = line.replace('"2023-12-31"', '"2023-04-01"')
|
||||
changed = True
|
||||
if "'2023-10-01'" in line:
|
||||
source[i] = line.replace("'2023-10-01'", "'2022-10-01'")
|
||||
changed = True
|
||||
if '"2023-10-01"' in line:
|
||||
source[i] = line.replace('"2023-10-01"', '"2022-10-01"')
|
||||
changed = True
|
||||
# For time_range="2022-09-01/2023-10-01"
|
||||
if "2022-09-01/2023-10-01" in line:
|
||||
source[i] = line.replace("2022-09-01/2023-10-01", "2022-09-01/2022-10-01")
|
||||
changed = True
|
||||
|
||||
elif isinstance(source, str):
|
||||
new_source = source.replace('"2023-12-31"', '"2023-04-01"')
|
||||
new_source = new_source.replace("'2023-10-01'", "'2022-10-01'")
|
||||
new_source = new_source.replace('"2023-10-01"', '"2022-10-01"')
|
||||
new_source = new_source.replace("2022-09-01/2023-10-01", "2022-09-01/2022-10-01")
|
||||
if new_source != source:
|
||||
cell['source'] = new_source
|
||||
changed = True
|
||||
|
||||
if changed:
|
||||
with open(file_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(nb, f, indent=1)
|
||||
print(f"Limited time_range to 1 month in {file_path}")
|
||||
except Exception as e:
|
||||
print(f"Error on {file_path}: {e}")
|
||||
|
||||
# 1. Modify the time ranges
|
||||
for nb_file in glob.glob("*.ipynb"):
|
||||
limit_time_range(nb_file)
|
||||
|
||||
# 2. Run them in parallel
|
||||
print("\nStarting parallel execution of notebooks...")
|
||||
processes = []
|
||||
for nb_file in NOTEBOOKS_TO_RUN:
|
||||
if os.path.exists(nb_file):
|
||||
print(f"Launching {nb_file}...")
|
||||
cmd = f"source /home/x79/miniconda3/etc/profile.d/conda.sh && conda activate env_01 && jupyter nbconvert --execute --ExecutePreprocessor.timeout=-1 --inplace {nb_file}"
|
||||
p = subprocess.Popen(["bash", "-c", cmd], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
processes.append((nb_file, p))
|
||||
|
||||
# 3. Wait and print output
|
||||
for nb_file, p in processes:
|
||||
p.wait()
|
||||
output = p.stdout.read().decode('utf-8')
|
||||
if p.returncode == 0:
|
||||
print(f"[{nb_file}] SUCCESS")
|
||||
else:
|
||||
print(f"[{nb_file}] FAILED (code {p.returncode})")
|
||||
print(f"--- OUTPUT START ({nb_file}) ---")
|
||||
print(output)
|
||||
print(f"--- OUTPUT END ({nb_file}) ---")
|
||||
|
||||
print("\nAll tasks finished.")
|
||||
@@ -0,0 +1,27 @@
|
||||
#!/bin/bash
|
||||
source /home/x79/miniconda3/etc/profile.d/conda.sh
|
||||
conda activate env_01
|
||||
set -e
|
||||
|
||||
# Configure GDAL for vsicurl stability
|
||||
export GDAL_HTTP_MAX_RETRY=5
|
||||
export GDAL_HTTP_RETRY_DELAY=2
|
||||
export GDAL_HTTP_CONNECTION_TIMEOUT=10
|
||||
export GDAL_HTTP_TIMEOUT=30
|
||||
export CPL_VSIL_CURL_ALLOWED_EXTENSIONS=.tif,.tiff
|
||||
export GDAL_DISABLE_READDIR_ON_OPEN=YES
|
||||
|
||||
|
||||
echo "=== [1/4] Running RF Training ==="
|
||||
jupyter nbconvert --execute --ExecutePreprocessor.timeout=-1 --inplace 01.train_ODC.ipynb
|
||||
|
||||
echo "=== [2/4] Running XGBoost Training ==="
|
||||
jupyter nbconvert --execute --ExecutePreprocessor.timeout=-1 --inplace 01.train_ODC_XGBoost.ipynb
|
||||
|
||||
echo "=== [3/4] Running Prediction ==="
|
||||
jupyter nbconvert --execute --ExecutePreprocessor.timeout=-1 --inplace 02.predict_ODC.ipynb
|
||||
|
||||
echo "=== [4/4] Running New Train ==="
|
||||
jupyter nbconvert --execute --ExecutePreprocessor.timeout=-1 --inplace new_train.ipynb
|
||||
|
||||
echo "=== ALL DONE SUCCESSFULLY ==="
|
||||
+25
-25
@@ -1141,12 +1141,12 @@
|
||||
"source": [
|
||||
"\n",
|
||||
"%matplotlib inline\n",
|
||||
"from new_import import *\n",
|
||||
"from new_import_ODC import *\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Dask gateway\n",
|
||||
"cluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))\n",
|
||||
"dc = datacube.Datacube()\n",
|
||||
"dc = None\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Configure s3 access\n",
|
||||
@@ -1300,7 +1300,7 @@
|
||||
"\n",
|
||||
".xr-section-summary-in + label:before {\n",
|
||||
" display: inline-block;\n",
|
||||
" content: '►';\n",
|
||||
" content: '\u25ba';\n",
|
||||
" font-size: 11px;\n",
|
||||
" width: 15px;\n",
|
||||
" text-align: center;\n",
|
||||
@@ -1311,7 +1311,7 @@
|
||||
"}\n",
|
||||
"\n",
|
||||
".xr-section-summary-in:checked + label:before {\n",
|
||||
" content: '▼';\n",
|
||||
" content: '\u25bc';\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
".xr-section-summary-in:checked + label > span {\n",
|
||||
@@ -1880,20 +1880,20 @@
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"ds = dc.load(\n",
|
||||
" product=\"sentinel1_grd_gamma0_20m\",\n",
|
||||
" x=(105.5, 106.4),\n",
|
||||
" y=(9.2, 10.0),\n",
|
||||
" time=(\"2022-09-01\", \"2023-10-01\"),\n",
|
||||
" measurements=[\"vv\", \"vh\"],\n",
|
||||
" output_crs=\"EPSG:32648\",\n",
|
||||
" resolution=(-10,10),\n",
|
||||
" dask_chunks={\"x\":2048, \"y\":2048},\n",
|
||||
" skip_broken_datasets=True,\n",
|
||||
" group_by=\"solar_day\"\n",
|
||||
")\n",
|
||||
"notebook_utils.heading(notebook_utils.xarray_object_size(ds))\n",
|
||||
"ds"
|
||||
"# ds = dc.load(\n",
|
||||
"# product=\"sentinel1_grd_gamma0_20m\",\n",
|
||||
"# x=(105.5, 106.4),\n",
|
||||
"# y=(9.2, 10.0),\n",
|
||||
"# time=(\"2022-09-01\", \"2022-10-01\"),\n",
|
||||
"# measurements=[\"vv\", \"vh\"],\n",
|
||||
"# output_crs=\"EPSG:32648\",\n",
|
||||
"# resolution=(-10,10),\n",
|
||||
"# dask_chunks={\"x\":2048, \"y\":2048},\n",
|
||||
"# skip_broken_datasets=True,\n",
|
||||
"# group_by=\"solar_day\"\n",
|
||||
"# )\n",
|
||||
"# notebook_utils.heading(notebook_utils.xarray_object_size(ds))\n",
|
||||
"# ds"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1905,11 +1905,11 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"vh = ds.vh.resample(time='1M').mean().persist()\n",
|
||||
"vh = vh.compute()\n",
|
||||
"vv = ds.vv.resample(time='1M').mean().persist()\n",
|
||||
"vv = vv.compute()\n",
|
||||
"\n"
|
||||
"# vh = ds.vh.resample(time='1M').mean().persist()\n",
|
||||
"# vh = vh.compute()\n",
|
||||
"# vv = ds.vv.resample(time='1M').mean().persist()\n",
|
||||
"# vv = vv.compute()\n",
|
||||
"# \n"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -2045,7 +2045,7 @@
|
||||
"\n",
|
||||
".xr-section-summary-in + label:before {\n",
|
||||
" display: inline-block;\n",
|
||||
" content: '►';\n",
|
||||
" content: '\u25ba';\n",
|
||||
" font-size: 11px;\n",
|
||||
" width: 15px;\n",
|
||||
" text-align: center;\n",
|
||||
@@ -2056,7 +2056,7 @@
|
||||
"}\n",
|
||||
"\n",
|
||||
".xr-section-summary-in:checked + label:before {\n",
|
||||
" content: '▼';\n",
|
||||
" content: '\u25bc';\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
".xr-section-summary-in:checked + label > span {\n",
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
#!/usr/bin/env python
|
||||
# coding: utf-8
|
||||
|
||||
# In[2]:
|
||||
|
||||
|
||||
get_ipython().run_line_magic('matplotlib', 'inline')
|
||||
from new_import import *
|
||||
|
||||
|
||||
# Dask gateway
|
||||
cluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))
|
||||
dc = datacube.Datacube()
|
||||
|
||||
|
||||
# Configure s3 access
|
||||
configure_s3_access(aws_unsigned=False, requester_pays=True, client=client)
|
||||
|
||||
|
||||
|
||||
|
||||
# In[3]:
|
||||
|
||||
|
||||
ds = dc.load(
|
||||
product="sentinel1_grd_gamma0_20m",
|
||||
x=(105.5, 106.4),
|
||||
y=(9.2, 10.0),
|
||||
time=("2022-09-01", "2023-10-01"),
|
||||
measurements=["vv", "vh"],
|
||||
output_crs="EPSG:32648",
|
||||
resolution=(-10,10),
|
||||
dask_chunks={"x":2048, "y":2048},
|
||||
skip_broken_datasets=True,
|
||||
group_by="solar_day"
|
||||
)
|
||||
notebook_utils.heading(notebook_utils.xarray_object_size(ds))
|
||||
ds
|
||||
|
||||
|
||||
# In[18]:
|
||||
|
||||
|
||||
vh = ds.vh.resample(time='1M').mean().persist()
|
||||
vh = vh.compute()
|
||||
vv = ds.vv.resample(time='1M').mean().persist()
|
||||
vv = vv.compute()
|
||||
|
||||
|
||||
|
||||
# In[28]:
|
||||
|
||||
|
||||
vv.min()
|
||||
|
||||
|
||||
# In[33]:
|
||||
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
# Plot the data
|
||||
plt.imshow(vh.isel(time=0), cmap='viridis', vmin=0, vmax=1)
|
||||
plt.colorbar() # Add colorbar for reference
|
||||
plt.show()
|
||||
|
||||
|
||||
# In[ ]:
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
import sys
|
||||
# Thêm đường dẫn hiện tại vào PYTHONPATH để import được new_import_ODC nếu cần
|
||||
sys.path.append('.')
|
||||
import warnings
|
||||
warnings.filterwarnings('ignore')
|
||||
from new_import_ODC import load_sen1
|
||||
|
||||
print("Testing load_sen1 with a short time range to speed up Dask compute...")
|
||||
bbox = [105.5, 9.2, 106.4, 10.0]
|
||||
time_range = "2023-01-01/2023-01-31" # Short time range for fast testing
|
||||
vh, vv = load_sen1(bbox, time_range)
|
||||
print("VH shape:", vh.shape)
|
||||
print("VV shape:", vv.shape)
|
||||
print("VH CRS:", vh.rio.crs)
|
||||
print("Success!")
|
||||
@@ -0,0 +1,49 @@
|
||||
import warnings
|
||||
warnings.filterwarnings('ignore')
|
||||
|
||||
def load_sen1(bbox, time_range):
|
||||
import pystac_client
|
||||
import planetary_computer
|
||||
import odc.stac
|
||||
|
||||
catalog = pystac_client.Client.open(
|
||||
"https://planetarycomputer.microsoft.com/api/stac/v1",
|
||||
modifier=planetary_computer.sign_inplace,
|
||||
)
|
||||
|
||||
search = catalog.search(
|
||||
collections=["sentinel-1-rtc"],
|
||||
bbox=bbox,
|
||||
datetime=time_range,
|
||||
)
|
||||
items = list(search.items())
|
||||
print("Found items:", len(items))
|
||||
|
||||
ds_s1 = odc.stac.load(
|
||||
items,
|
||||
bands=["vv", "vh"],
|
||||
bbox=bbox,
|
||||
crs="EPSG:32648",
|
||||
resolution=10,
|
||||
chunks={"x": 2048, "y": 2048, "time": 1}
|
||||
)
|
||||
|
||||
ds_median = ds_s1.median(dim="time").compute()
|
||||
vv = ds_median["vv"]
|
||||
vh = ds_median["vh"]
|
||||
|
||||
vv = vv.expand_dims(dim="band")
|
||||
vh = vh.expand_dims(dim="band")
|
||||
|
||||
vv = vv.rio.write_crs("EPSG:32648")
|
||||
vh = vh.rio.write_crs("EPSG:32648")
|
||||
|
||||
return vh, vv
|
||||
|
||||
print("Testing load_sen1...")
|
||||
bbox = [105.5, 9.2, 106.4, 10.0]
|
||||
time_range = "2022-09-01/2023-10-01"
|
||||
vh, vv = load_sen1(bbox, time_range)
|
||||
print("VH shape:", vh.shape)
|
||||
print("VV shape:", vv.shape)
|
||||
print("Success!")
|
||||
Reference in New Issue
Block a user