refactor: reorganize project structure by moving core modules and update import paths in API server
This commit is contained in:
@@ -0,0 +1,659 @@
|
||||
TEST_MODE = True
|
||||
RESOLUTION = 1000 if TEST_MODE else 10
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
# Common imports and settings
|
||||
import os, sys
|
||||
os.environ['USE_PYGEOS'] = '0'
|
||||
os.environ["GDAL_HTTP_MAX_RETRY"] = "5"
|
||||
os.environ["GDAL_HTTP_RETRY_DELAY"] = "2"
|
||||
os.environ["GDAL_HTTP_CONNECTION_TIMEOUT"] = "10"
|
||||
os.environ["GDAL_HTTP_TIMEOUT"] = "30"
|
||||
os.environ["CPL_VSIL_CURL_ALLOWED_EXTENSIONS"] = ".tif,.tiff"
|
||||
os.environ["GDAL_DISABLE_READDIR_ON_OPEN"] = "YES"
|
||||
from IPython.display import Markdown
|
||||
import pandas as pd
|
||||
pd.set_option("display.max_rows", None)
|
||||
import xarray as xr
|
||||
|
||||
# Datacube
|
||||
import datacube
|
||||
from datacube.utils.rio import configure_s3_access
|
||||
from datacube.utils import masking
|
||||
from datacube.utils.cog import write_cog
|
||||
# removed deafrica_tools imports to avoid ipyleaflet error
|
||||
|
||||
# EASI defaults
|
||||
easinotebooksrepo = '/home/x79/CSIROBoeingPhase4-Vietnam'
|
||||
if easinotebooksrepo not in sys.path: sys.path.append(easinotebooksrepo)
|
||||
from easi_tools import EasiDefaults, xarray_object_size, notebook_utils, unset_cachingproxy
|
||||
# from easi_tools.load_s2l2a import load_s2l2a_with_offset
|
||||
from dask.distributed import progress
|
||||
|
||||
# Data tools
|
||||
import numpy as np
|
||||
from datetime import datetime
|
||||
|
||||
# Datacube
|
||||
from datacube.utils import masking # https://github.com/opendatacube/datacube-core/blob/develop/datacube/utils/masking.py
|
||||
from odc.algo import enum_to_bool # https://github.com/opendatacube/odc-algo/blob/main/odc/algo/_masking.py
|
||||
# removed xr_reproject
|
||||
from datacube.utils.geometry import GeoBox, box # https://github.com/opendatacube/datacube-core/blob/develop/datacube/utils/geometry/_base.py
|
||||
|
||||
# Holoviews, Datashader and Bokeh
|
||||
import hvplot.pandas
|
||||
import hvplot.xarray
|
||||
import holoviews as hv
|
||||
import panel as pn
|
||||
import colorcet as cc
|
||||
import cartopy.crs as ccrs
|
||||
from datashader import reductions
|
||||
from holoviews import opts
|
||||
from utils import load_data_geo
|
||||
import rasterio
|
||||
import rioxarray
|
||||
# import geoviews as gv
|
||||
# from holoviews.operation.datashader import rasterize
|
||||
hv.extension('bokeh', logo=False)
|
||||
|
||||
from deafrica_tools.bandindices import calculate_indices
|
||||
from xgboost import XGBClassifier
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.metrics import accuracy_score, classification_report
|
||||
from sklearn.preprocessing import LabelEncoder
|
||||
|
||||
from sklearn.pipeline import Pipeline
|
||||
from sklearn.impute import SimpleImputer
|
||||
from sklearn.preprocessing import StandardScaler
|
||||
from sklearn.model_selection import GridSearchCV
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.metrics import accuracy_score
|
||||
from shapely.geometry import Point, Polygon
|
||||
import geopandas as gpd
|
||||
from pyproj import CRS
|
||||
from matplotlib.colors import ListedColormap
|
||||
from holoviews import opts
|
||||
from datashader import reductions
|
||||
from bokeh.models.tickers import FixedTicker
|
||||
from rioxarray.merge import merge_arrays
|
||||
|
||||
from sklearn.preprocessing import PolynomialFeatures
|
||||
from sklearn.linear_model import LinearRegression
|
||||
from sklearn.ensemble import RandomForestRegressor
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.metrics import mean_squared_error, r2_score
|
||||
|
||||
import joblib
|
||||
|
||||
|
||||
def load_data(dc, date_range, longtitude_range, latitude_range):
|
||||
import os, hashlib
|
||||
cache_dir = "dataset_cache"
|
||||
os.makedirs(cache_dir, exist_ok=True)
|
||||
key_str = f"s2_{date_range}_{longtitude_range}_{latitude_range}"
|
||||
cache_key = hashlib.md5(key_str.encode()).hexdigest() + ".nc"
|
||||
cache_path = os.path.join(cache_dir, cache_key)
|
||||
|
||||
if os.path.exists(cache_path):
|
||||
print(f"✅ Loading cached S2 data from {cache_path}")
|
||||
return xr.open_dataset(cache_path, engine='netcdf4')
|
||||
|
||||
product = 's2_l2a'
|
||||
bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
|
||||
|
||||
import pystac_client
|
||||
import planetary_computer
|
||||
import odc.stac
|
||||
|
||||
catalog = pystac_client.Client.open(
|
||||
"https://planetarycomputer.microsoft.com/api/stac/v1",
|
||||
modifier=planetary_computer.sign_inplace,
|
||||
)
|
||||
search = catalog.search(
|
||||
collections=["sentinel-2-l2a"],
|
||||
bbox=bbox,
|
||||
datetime=f"{date_range[0]}/{date_range[1]}",
|
||||
)
|
||||
items = list(search.items())
|
||||
|
||||
data = odc.stac.load(
|
||||
items,
|
||||
bands=["red", "nir", "SCL"],
|
||||
bbox=bbox,
|
||||
crs="EPSG:32648",
|
||||
resolution=RESOLUTION,
|
||||
chunks={"x": 2048, "y": 2048, "time": 1},
|
||||
groupby="solar_day"
|
||||
)
|
||||
if "SCL" in data.data_vars:
|
||||
data = data.rename({"SCL": "scl"})
|
||||
|
||||
print(f"💾 Caching S2 data to {cache_path}")
|
||||
data = data.compute()
|
||||
data.to_netcdf(cache_path, engine='netcdf4')
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def mask_clean(data):
|
||||
# For Sentinel-2 L2A SCL:
|
||||
# 2: Dark Area Pixels, 4: Vegetation, 5: Not Vegetated, 6: Water
|
||||
good_pixel_mask = data['scl'].isin([2, 4, 5, 6])
|
||||
data_layer_names = [x for x in data.data_vars if x != 'scl']
|
||||
# Apply good pixel mask
|
||||
result = data[data_layer_names].where(good_pixel_mask).persist()
|
||||
return result
|
||||
|
||||
|
||||
def fill_nan(ndvi, time_split):
|
||||
if len(ndvi.time) == 0:
|
||||
return ndvi
|
||||
|
||||
# If the total time duration is less than 90 days, skip seasonal splitting
|
||||
try:
|
||||
total_days = (ndvi.time[-1] - ndvi.time[0]).dt.days.item()
|
||||
if total_days < 90:
|
||||
return ndvi.bfill(dim="time").ffill(dim="time")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
rs = []
|
||||
for times in time_split:
|
||||
try:
|
||||
tmp = ndvi.sel(time=times)
|
||||
if len(tmp.time) == 0:
|
||||
continue
|
||||
fill_ds = tmp.bfill(dim='time').ffill(dim='time')
|
||||
rs.append(fill_ds)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if len(rs) == 0:
|
||||
return ndvi.bfill(dim="time").ffill(dim="time")
|
||||
|
||||
merged_ndvi = xr.concat([i for i in rs], dim="time")
|
||||
fill_m = merged_ndvi.bfill(dim="time")
|
||||
fill_m = fill_m.ffill(dim="time")
|
||||
return fill_m
|
||||
|
||||
|
||||
def load_train_data(train_path):
|
||||
train = load_data_geo(train_path)
|
||||
return train
|
||||
|
||||
|
||||
def load_sen1(bbox, time_range):
|
||||
import os, hashlib
|
||||
cache_dir = "dataset_cache"
|
||||
os.makedirs(cache_dir, exist_ok=True)
|
||||
key_str = f"s1_vh_vv_{bbox}_{time_range}"
|
||||
cache_key_vh = hashlib.md5((key_str + "vh").encode()).hexdigest() + ".nc"
|
||||
cache_key_vv = hashlib.md5((key_str + "vv").encode()).hexdigest() + ".nc"
|
||||
cache_path_vh = os.path.join(cache_dir, cache_key_vh)
|
||||
cache_path_vv = os.path.join(cache_dir, cache_key_vv)
|
||||
|
||||
if os.path.exists(cache_path_vh) and os.path.exists(cache_path_vv):
|
||||
print(f"✅ Loading cached S1 data from {cache_path_vh} and {cache_path_vv}")
|
||||
ds_vh = xr.open_dataset(cache_path_vh, engine='netcdf4')
|
||||
ds_vv = xr.open_dataset(cache_path_vv, engine='netcdf4')
|
||||
return ds_vh[list(ds_vh.data_vars)[0]], ds_vv[list(ds_vv.data_vars)[0]]
|
||||
|
||||
import pystac_client
|
||||
import planetary_computer
|
||||
import odc.stac
|
||||
|
||||
# Kết nối STAC Client
|
||||
catalog = pystac_client.Client.open(
|
||||
"https://planetarycomputer.microsoft.com/api/stac/v1",
|
||||
modifier=planetary_computer.sign_inplace,
|
||||
)
|
||||
|
||||
# Tìm kiếm Items
|
||||
search = catalog.search(
|
||||
collections=["sentinel-1-rtc"],
|
||||
bbox=bbox,
|
||||
datetime=time_range,
|
||||
)
|
||||
items = list(search.items())
|
||||
|
||||
# Tải dữ liệu thành xarray Dataset
|
||||
ds_s1 = odc.stac.load(
|
||||
items,
|
||||
bands=["vv", "vh"],
|
||||
bbox=bbox,
|
||||
crs="EPSG:32648",
|
||||
resolution=RESOLUTION,
|
||||
chunks={"x": 2048, "y": 2048, "time": 1}
|
||||
)
|
||||
|
||||
# Tính giá trị trung vị theo thời gian
|
||||
ds_median = ds_s1.median(dim="time").compute()
|
||||
vv = ds_median["vv"]
|
||||
vh = ds_median["vh"]
|
||||
|
||||
# Thêm chiều 'band' để giống hệt rioxarray
|
||||
vv = vv.expand_dims(dim="band")
|
||||
vh = vh.expand_dims(dim="band")
|
||||
|
||||
# Phục hồi metadata về toạ độ
|
||||
vv = vv.rio.write_crs("EPSG:32648")
|
||||
vh = vh.rio.write_crs("EPSG:32648")
|
||||
|
||||
print(f"💾 Caching S1 data to {cache_dir}")
|
||||
vh.to_netcdf(cache_path_vh, engine='netcdf4')
|
||||
vv.to_netcdf(cache_path_vv, engine='netcdf4')
|
||||
|
||||
return vh, vv
|
||||
|
||||
|
||||
def get_data_sen1_and_sen2(train, average_ndvi, dsvh, dsvv):
|
||||
loaded_datasets = {}
|
||||
for idx, point in train.iterrows():
|
||||
key = f"point_{idx + 1}"
|
||||
try:
|
||||
ndvi_data = average_ndvi.sel(x=point.geometry.x, y=point.geometry.y, method='nearest').values
|
||||
vh_data = dsvh.sel(x=point.geometry.x, y=point.geometry.y, method='nearest').values
|
||||
vv_data = dsvv.sel(x=point.geometry.x, y=point.geometry.y, method='nearest').values
|
||||
loaded_datasets[key] = {
|
||||
"data": np.concatenate((ndvi_data, vh_data, vv_data)),
|
||||
"label": point.HT_code
|
||||
}
|
||||
except Exception as e:
|
||||
# loaded_datasets[key] = None
|
||||
print(e)
|
||||
return loaded_datasets
|
||||
|
||||
|
||||
def split_train_data(train, label_mapping, datasets):
|
||||
label_encoder = LabelEncoder()
|
||||
|
||||
# Fit and transform the labels
|
||||
labels = train.Hientrang.values
|
||||
numeric_labels = label_encoder.fit_transform([label_mapping[label] for label in labels])
|
||||
X = []
|
||||
x_new = []
|
||||
lb_new = []
|
||||
for k, v in datasets.items():
|
||||
X.append(v)
|
||||
for i in range(len(X)):
|
||||
if X[i] is not None:
|
||||
x_new.append(X[i]["data"])
|
||||
lb_new.append(numeric_labels[i])
|
||||
X_train, X_temp, y_train, y_temp= train_test_split(x_new, lb_new, test_size=0.4, random_state=42)
|
||||
X_val, X_test, y_val, y_test = train_test_split(X_temp, y_temp, test_size=0.5, random_state=42)
|
||||
return X_train, X_val, X_test, y_train, y_val, y_test
|
||||
|
||||
|
||||
def train_with_rf(X_train, X_val, y_train, y_val):
|
||||
# Takes 1-2 minutes to complete
|
||||
|
||||
# Tạo RandomForestClassifier mặc định để sử dụng làm mô hình ban đầu trong pipeline
|
||||
base_model = XGBClassifier(tree_method="hist", device="cuda", random_state=42, n_jobs=-1)
|
||||
|
||||
# Tạo pipeline
|
||||
pipeline = Pipeline([
|
||||
# ('imputer', SimpleImputer(strategy='mean')),
|
||||
('scaler', StandardScaler()),
|
||||
('classifier', base_model),
|
||||
])
|
||||
# Thiết lập các tham số bạn muốn tối ưu hóa
|
||||
param_grid = {
|
||||
'classifier__n_estimators': [100, 300, 500, 700, 1000],
|
||||
'classifier__max_depth': [6, 8, 10, 15, 20],
|
||||
'classifier__learning_rate': [0.01, 0.1, 0.2],
|
||||
}
|
||||
|
||||
# Sử dụng GridSearchCV để tìm bộ tham số tốt nhất
|
||||
grid_search = GridSearchCV(pipeline, param_grid, cv=5, scoring='accuracy', n_jobs=-1)
|
||||
grid_search.fit(X_train, y_train)
|
||||
|
||||
# In ra bộ tham số tốt nhất
|
||||
best_params = grid_search.best_params_
|
||||
print("Best Parameters:", best_params)
|
||||
|
||||
# Dự đoán trên tập kiểm tra
|
||||
y_pred = grid_search.predict(X_val)
|
||||
|
||||
# Đánh giá kết quả
|
||||
accuracy = accuracy_score(y_val, y_pred)
|
||||
print(f"Accuracy: {round(accuracy, 2)*100} %")
|
||||
return grid_search
|
||||
|
||||
|
||||
def save_model(name_file, model, metadata=None, label_encoder=None):
|
||||
"""
|
||||
Save model với metadata để tương thích với ModelManager
|
||||
|
||||
Args:
|
||||
name_file: Tên file model
|
||||
model: Model object
|
||||
metadata: Dict chứa thông tin về model (optional)
|
||||
label_encoder: Label encoder (optional)
|
||||
"""
|
||||
from model_manager import get_model_manager
|
||||
|
||||
dir_save_model = "model_train"
|
||||
if not os.path.exists(dir_save_model):
|
||||
os.mkdir(dir_save_model)
|
||||
|
||||
# Nếu có metadata, sử dụng ModelManager
|
||||
if metadata is not None:
|
||||
model_manager = get_model_manager()
|
||||
model_manager.save_model(
|
||||
model=model,
|
||||
metadata=metadata,
|
||||
model_filename=name_file,
|
||||
label_encoder=label_encoder
|
||||
)
|
||||
else:
|
||||
# Legacy mode: save trực tiếp (backward compatibility)
|
||||
model_data = {
|
||||
'model': model,
|
||||
'label_encoder': label_encoder
|
||||
} if label_encoder is not None else model
|
||||
|
||||
joblib.dump(model_data, os.path.join(dir_save_model, name_file))
|
||||
|
||||
print(f"✅ Model saved: {name_file}")
|
||||
if metadata:
|
||||
print(f" - Type: {metadata.get('model_type', 'N/A')}")
|
||||
print(f" - Features: {metadata.get('n_features', 'N/A')}")
|
||||
print(f" - Accuracy: {metadata.get('test_accuracy', 'N/A')}")
|
||||
|
||||
|
||||
|
||||
def predict(model, data_crs, ndvi, vh, vv):
|
||||
# Unpack model if it is wrapped in a dictionary (from ModelManager)
|
||||
if isinstance(model, dict) and 'model' in model:
|
||||
model = model['model']
|
||||
|
||||
data_predict = []
|
||||
for i in range(ndvi.shape[1]):
|
||||
ndvi_tmp = ndvi.isel(y=i).values
|
||||
vh_data = vh.sel(y=ndvi.y.values[i], method='nearest').values
|
||||
vv_data = vv.sel(y=ndvi.y.values[i], method='nearest').values
|
||||
all_tmp = np.concatenate((ndvi_tmp, vh_data, vv_data), axis=0)
|
||||
data_predict.extend(all_tmp.T)
|
||||
y_pred = model.predict(data_predict)
|
||||
final_label = y_pred.reshape(ndvi.y.shape[0], ndvi.x.shape[0])
|
||||
|
||||
final_xarray_save = xr.DataArray(final_label, dims=("y", "x"))
|
||||
final_xarray_save = final_xarray_save.rio.write_crs(data_crs)
|
||||
|
||||
x_values = ndvi.x.values
|
||||
y_values = ndvi.y.values
|
||||
|
||||
data_array = xr.DataArray(final_xarray_save,
|
||||
coords={'x': x_values, 'y': y_values},
|
||||
dims=['y', 'x'])
|
||||
data_array = data_array.rio.write_crs(ndvi.rio.crs)
|
||||
return data_array
|
||||
|
||||
|
||||
def cut_according_shp(thuanhoa_path, average_ndvi, data_array):
|
||||
gdf = gpd.read_file(thuanhoa_path)
|
||||
gdf = gdf.to_crs(average_ndvi.rio.crs)
|
||||
polygon_coords = list(gdf.geometry.values[0].exterior.coords)
|
||||
polygon_coordinates = [(x, y) for x, y in polygon_coords]
|
||||
|
||||
geometries = [
|
||||
{
|
||||
'type': 'Polygon',
|
||||
'coordinates': [polygon_coordinates]
|
||||
}
|
||||
]
|
||||
region_result = data_array.rio.clip(geometries, data_array.rio.crs, drop=False)
|
||||
region_result = region_result.where(region_result >= 0, float('nan'))
|
||||
return region_result
|
||||
|
||||
|
||||
def compare(KD_path, KetQuaPhanLoaiDat, CODE_MAP, HT_MAP):
|
||||
gdf = gpd.read_file(KD_path, crs="EPSG:9209")
|
||||
polygon = gdf.geometry.values
|
||||
label = gdf.tenchu.values
|
||||
ouput_image = rioxarray.open_rasterio(KetQuaPhanLoaiDat)
|
||||
code_tq = HT_MAP["TQ"]["data"][0]
|
||||
code_pnn = HT_MAP["PNN"]["data"][0]
|
||||
result = {}
|
||||
for key, values in HT_MAP.items():
|
||||
print(f"process {key}")
|
||||
array_list = []
|
||||
for i in range(len(polygon)):
|
||||
po = polygon[i]
|
||||
lb = label[i]
|
||||
code_lb = CODE_MAP.get(lb, code_tq)
|
||||
try:
|
||||
qr = ouput_image.rio.clip([po], "EPSG:9209")
|
||||
if code_lb in values["data"]:
|
||||
if code_lb == code_pnn:
|
||||
qr = qr.where((qr != float(code_pnn)), np.nan)
|
||||
# qr = qr.where((qr != 3.0), np.nan)
|
||||
elif code_lb == code_tq:
|
||||
qr = qr.where((qr != float(code_pnn)), np.nan)
|
||||
qr = qr.where((qr != 3.0), np.nan)
|
||||
else:
|
||||
qr = qr.where(qr != float(code_lb), np.nan)
|
||||
else:
|
||||
qr.values[:, :, :] = np.nan
|
||||
array_list.append(qr)
|
||||
except Exception as e:
|
||||
pass
|
||||
result.update({key: array_list})
|
||||
return result
|
||||
|
||||
|
||||
def save_result(result, HT_MAP):
|
||||
# cmap = ListedColormap(colors)
|
||||
save_path = "ThuanHoa/KetQua"
|
||||
if not os.path.exists(save_path):
|
||||
os.mkdir(save_path)
|
||||
|
||||
for k, v in result.items():
|
||||
rs = merge_arrays(v, nodata = np.nan)
|
||||
rs.rio.to_raster(f"{save_path}/{k}.tif")
|
||||
print(f"save {save_path}/{k}.tif")
|
||||
# img = rs.plot(cmap=cmap, add_colorbar=False)
|
||||
# cbar = plt.colorbar(img)
|
||||
# cbar.ax.set_yticklabels(labels)
|
||||
# plt.title(f'{HT_MAP[k]["name"]}')
|
||||
# plt.axis('off')
|
||||
# plt.show()
|
||||
|
||||
def load_data_sen1(dc, date_range, coordinates):
|
||||
import os, hashlib
|
||||
longtitude_range, latitude_range = coordinates
|
||||
bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
|
||||
|
||||
cache_dir = "dataset_cache"
|
||||
os.makedirs(cache_dir, exist_ok=True)
|
||||
key_str = f"data_sen1_{date_range}_{bbox}"
|
||||
cache_key_vh = hashlib.md5((key_str + "vh").encode()).hexdigest() + ".nc"
|
||||
cache_key_vv = hashlib.md5((key_str + "vv").encode()).hexdigest() + ".nc"
|
||||
cache_path_vh = os.path.join(cache_dir, cache_key_vh)
|
||||
cache_path_vv = os.path.join(cache_dir, cache_key_vv)
|
||||
|
||||
if os.path.exists(cache_path_vh) and os.path.exists(cache_path_vv):
|
||||
print(f"✅ Loading cached S1 (coord) data")
|
||||
ds_vh = xr.open_dataset(cache_path_vh, engine='netcdf4')
|
||||
ds_vv = xr.open_dataset(cache_path_vv, engine='netcdf4')
|
||||
var_vh = [v for v in ds_vh.data_vars if v != 'spatial_ref'][0]
|
||||
var_vv = [v for v in ds_vv.data_vars if v != 'spatial_ref'][0]
|
||||
return ds_vh[var_vh], ds_vv[var_vv]
|
||||
|
||||
import pystac_client
|
||||
import planetary_computer
|
||||
import odc.stac
|
||||
|
||||
catalog = pystac_client.Client.open(
|
||||
"https://planetarycomputer.microsoft.com/api/stac/v1",
|
||||
modifier=planetary_computer.sign_inplace,
|
||||
)
|
||||
search = catalog.search(
|
||||
collections=["sentinel-1-rtc"],
|
||||
bbox=bbox,
|
||||
datetime=f"{date_range[0]}/{date_range[1]}",
|
||||
)
|
||||
items = list(search.items())
|
||||
|
||||
data_sen1 = odc.stac.load(
|
||||
items,
|
||||
bands=["vv", "vh"],
|
||||
bbox=bbox,
|
||||
crs="EPSG:32648",
|
||||
resolution=RESOLUTION,
|
||||
chunks={"x": 2048, "y": 2048, "time": 1},
|
||||
groupby="solar_day"
|
||||
)
|
||||
|
||||
data_sen1 = data_sen1.compute()
|
||||
dsvh = data_sen1.vh
|
||||
dsvv = data_sen1.vv
|
||||
|
||||
print(f"💾 Caching S1 (coord) data")
|
||||
dsvh.to_netcdf(cache_path_vh, engine='netcdf4')
|
||||
dsvv.to_netcdf(cache_path_vv, engine='netcdf4')
|
||||
|
||||
return dsvh, dsvv
|
||||
|
||||
def calculate_average(data, time_pattern='1M'):
|
||||
return data.resample(time=time_pattern).mean().persist()
|
||||
|
||||
|
||||
def load_data_sen2(dc, date_range, coordinates):
|
||||
import os, hashlib
|
||||
longtitude_range, latitude_range = coordinates
|
||||
bbox = [longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1]]
|
||||
|
||||
cache_dir = "dataset_cache"
|
||||
os.makedirs(cache_dir, exist_ok=True)
|
||||
key_str = f"data_sen2_{date_range}_{bbox}"
|
||||
cache_key = hashlib.md5(key_str.encode()).hexdigest() + ".nc"
|
||||
cache_path = os.path.join(cache_dir, cache_key)
|
||||
|
||||
if os.path.exists(cache_path):
|
||||
print(f"✅ Loading cached S2 (coord) data from {cache_path}")
|
||||
return xr.open_dataset(cache_path, engine='netcdf4')
|
||||
|
||||
import pystac_client
|
||||
import planetary_computer
|
||||
import odc.stac
|
||||
|
||||
catalog = pystac_client.Client.open(
|
||||
"https://planetarycomputer.microsoft.com/api/stac/v1",
|
||||
modifier=planetary_computer.sign_inplace,
|
||||
)
|
||||
search = catalog.search(
|
||||
collections=["sentinel-2-l2a"],
|
||||
bbox=bbox,
|
||||
datetime=f"{date_range[0]}/{date_range[1]}",
|
||||
)
|
||||
items = list(search.items())
|
||||
|
||||
data = odc.stac.load(
|
||||
items,
|
||||
bands=["red", "nir", "SCL"],
|
||||
bbox=bbox,
|
||||
crs="EPSG:32648",
|
||||
resolution=RESOLUTION,
|
||||
chunks={"x": 2048, "y": 2048, "time": 1},
|
||||
groupby="solar_day"
|
||||
)
|
||||
if "SCL" in data.data_vars:
|
||||
data = data.rename({"SCL": "scl"})
|
||||
|
||||
data = data.compute()
|
||||
print(f"💾 Caching S2 (coord) data to {cache_path}")
|
||||
data.to_netcdf(cache_path, engine='netcdf4')
|
||||
|
||||
return data
|
||||
|
||||
def mask_cloud(data):
|
||||
# For Sentinel-2 L2A SCL:
|
||||
# 2: Dark Area Pixels, 4: Vegetation, 5: Not Vegetated, 6: Water
|
||||
good_pixel_mask = data['scl'].isin([2, 4, 5, 6])
|
||||
data_layer_names = [x for x in data.data_vars if x != 'scl']
|
||||
# Apply good pixel mask
|
||||
result = data[data_layer_names].where(good_pixel_mask).persist()
|
||||
return result
|
||||
|
||||
def find_best_model(dataset):
|
||||
X_train, X_val, y_train, y_val = dataset
|
||||
# Tạo RandomForestClassifier mặc định để sử dụng làm mô hình ban đầu trong pipeline
|
||||
base_model = XGBClassifier(tree_method="hist", device="cuda", random_state=42, n_jobs=-1)
|
||||
|
||||
# Tạo pipeline
|
||||
pipeline = Pipeline([
|
||||
# ('imputer', SimpleImputer(strategy='mean')),
|
||||
('scaler', StandardScaler()),
|
||||
('classifier', base_model),
|
||||
])
|
||||
# Thiết lập các tham số bạn muốn tối ưu hóa
|
||||
param_grid = {
|
||||
'classifier__n_estimators': [100, 300, 500, 700, 1000],
|
||||
'classifier__max_depth': [6, 8, 10, 15, 20],
|
||||
'classifier__learning_rate': [0.01, 0.1, 0.2],
|
||||
}
|
||||
|
||||
# Sử dụng GridSearchCV để tìm bộ tham số tốt nhất
|
||||
grid_search = GridSearchCV(pipeline, param_grid, cv=5, scoring='accuracy', n_jobs=-1)
|
||||
grid_search.fit(X_train, y_train)
|
||||
|
||||
# In ra bộ tham số tốt nhất
|
||||
best_params = grid_search.best_params_
|
||||
print("Best Parameters:", best_params)
|
||||
|
||||
# Dự đoán trên tập kiểm tra
|
||||
y_pred = grid_search.predict(X_val)
|
||||
|
||||
# Đánh giá kết quả
|
||||
accuracy = accuracy_score(y_val, y_pred)
|
||||
print(f"Accuracy: {round(accuracy, 2)*100} %")
|
||||
return grid_search
|
||||
|
||||
def save_result_new(result, save_path, HT_MAP):
|
||||
# cmap = ListedColormap(colors)
|
||||
if not os.path.exists(save_path):
|
||||
os.mkdir(save_path)
|
||||
|
||||
for k, v in result.items():
|
||||
rs = merge_arrays(v, nodata = np.nan)
|
||||
rs.rio.to_raster(f"{save_path}/{k}.tif")
|
||||
print(f"save {save_path}/{k}.tif")
|
||||
# img = rs.plot(cmap=cmap, add_colorbar=False)
|
||||
# cbar = plt.colorbar(img)
|
||||
# cbar.ax.set_yticklabels(labels)
|
||||
# plt.title(f'{HT_MAP[k]["name"]}')
|
||||
# plt.axis('off')
|
||||
# plt.show()
|
||||
def accuracy_test(test, data_array):
|
||||
# cấu hình nhãn dữ liệu
|
||||
label_mapping = {
|
||||
"Lua tom": "0",
|
||||
"Lua": "1",
|
||||
"CHN": "2",
|
||||
"CLN": "3",
|
||||
"TS": "4",
|
||||
"Song": "5",
|
||||
"Dat xay dung": "6",
|
||||
"Rung": "7"
|
||||
}
|
||||
|
||||
chk = []
|
||||
pred = []
|
||||
dd = []
|
||||
for idx, point in test.iterrows():
|
||||
label = point.LULC
|
||||
predict = data_array.sel(x=point.geometry.x, y=point.geometry.y, method='nearest').values
|
||||
pred.append(label_mapping[label])
|
||||
dd.append(str(predict))
|
||||
chk.append(predict == int(label_mapping[label]))
|
||||
test["code"] = pred
|
||||
test["dd"] = dd
|
||||
test["check"] = chk
|
||||
path = "ThuanHoa/TestAccuracy"
|
||||
if not os.path.exists(path):
|
||||
os.mkdir(path)
|
||||
test.to_file(f"{path}/result.shp")
|
||||
|
||||
percentage_true = np.mean(chk) * 100
|
||||
print(f"độ chính xác: {percentage_true:.2f}%")
|
||||
Reference in New Issue
Block a user