refactor: reorganize project structure by moving core modules and update import paths in API server
This commit is contained in:
@@ -0,0 +1,272 @@
|
||||
#!/usr/bin/env python
|
||||
# coding: utf-8
|
||||
|
||||
# In[1]:
|
||||
|
||||
|
||||
get_ipython().run_cell_magic('time', '', '%matplotlib inline\nfrom new_import import *\n')
|
||||
|
||||
|
||||
# In[2]:
|
||||
|
||||
|
||||
get_ipython().run_cell_magic('time', '', '# Dask gateway\ncluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,4))\ndc = datacube.Datacube()\n\n# Configure s3 access\nconfigure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n\nclient\n')
|
||||
|
||||
|
||||
# In[3]:
|
||||
|
||||
|
||||
## cấu hình thời gian lấy ảnh và tọa độ
|
||||
# date_range = ('2022-09-01', '2023-10-01')
|
||||
# longtitude_range = (105.86575, 105.94120)
|
||||
# latitude_range = (9.65070, 9.69850)
|
||||
date_range = ('2022-09-01', '2023-10-01')
|
||||
longtitude_range = (105.5, 106.4)
|
||||
latitude_range = (9.2, 10.0)
|
||||
|
||||
|
||||
# In[4]:
|
||||
|
||||
|
||||
## truy vấn ảnh vệ tinh sen2
|
||||
data = load_data(dc, date_range, longtitude_range, latitude_range)
|
||||
notebook_utils.heading(notebook_utils.xarray_object_size(data))
|
||||
display(data)
|
||||
|
||||
|
||||
# In[5]:
|
||||
|
||||
|
||||
# Specify the start and end times
|
||||
min_date = '2022-09-01' # Thời gian bắt đầu lấy data cho quá trình train
|
||||
max_date = '2023-10-01' # Thời gian kết thúc lấy data cho quá trình train
|
||||
# Just do 1 month for testing
|
||||
# max_date = '2022-10-01' # Thời gian kết thúc lấy data cho quá trình train
|
||||
|
||||
# Specify a spatail region to search using latitude/longitude cooridinates
|
||||
min_longitude, max_longitude = (105.5, 106.4)
|
||||
min_latitude, max_latitude = (9.2, 10.0)
|
||||
|
||||
# Specify the product. In this case we want to use Sentinel-2 Level-2A data
|
||||
product = 's2_l2a'
|
||||
|
||||
# Construct the search query dictionary
|
||||
query = {
|
||||
'product': product, # Product name
|
||||
'x': (min_longitude, max_longitude), # "x" axis bounds
|
||||
'y': (min_latitude, max_latitude), # "y" axis bounds
|
||||
'time': (min_date, max_date), # Any parsable date strings
|
||||
}
|
||||
|
||||
|
||||
# In[6]:
|
||||
|
||||
|
||||
# Most common CRS
|
||||
native_crs = notebook_utils.mostcommon_crs(dc, query)
|
||||
print(f'Most common native CRS: {native_crs}')
|
||||
|
||||
|
||||
# In[7]:
|
||||
|
||||
|
||||
# Specify the spectral band measurements we want to use for a classification algorithm
|
||||
measurements = ['red', 'nir', 'scl']
|
||||
|
||||
load_params = {
|
||||
'measurements': measurements, # Selected measurement or alias names
|
||||
'output_crs': native_crs, # Target EPSG code
|
||||
'resolution': (-10, 10), # Target resolution
|
||||
'group_by': 'solar_day', # Scene grouping
|
||||
'dask_chunks': {'x': 2048, 'y': 2048}, # Dask chunks
|
||||
}
|
||||
|
||||
|
||||
# In[8]:
|
||||
|
||||
|
||||
get_ipython().run_cell_magic('time', '', '# The replacement "dc.load()" function for this product\ndata = load_s2l2a_with_offset(\n dc,\n query | load_params # Combine the two dicts that contain our search and load parameters\n)\n\n# This line prints the total size of the dataset hat was loaded\nnotebook_utils.heading(notebook_utils.xarray_object_size(data))\n\ndisplay(data)\n')
|
||||
|
||||
|
||||
# In[9]:
|
||||
|
||||
|
||||
# %%time
|
||||
# # Tiến hành loại bỏ các vị trí bị mây ảnh hưởng
|
||||
# result = mask_clean(data)
|
||||
# progress(result)
|
||||
|
||||
|
||||
# In[10]:
|
||||
|
||||
|
||||
# Tiến hành tính toán NDVI
|
||||
ds1 = calculate_indices(result, index='NDVI', satellite_mission='s2')
|
||||
ndvi = ds1["NDVI"]
|
||||
display(ndvi)
|
||||
|
||||
|
||||
# In[11]:
|
||||
|
||||
|
||||
get_ipython().run_cell_magic('time', '', "## tính ndvi theo tháng\naverage_ndvi = ndvi.resample(time='1M').mean().persist()\nprogress(average_ndvi)\n")
|
||||
|
||||
|
||||
# In[12]:
|
||||
|
||||
|
||||
# compute average_ndvi
|
||||
average_ndvi = average_ndvi.compute()
|
||||
|
||||
|
||||
# In[13]:
|
||||
|
||||
|
||||
# cấu hình vh vv file
|
||||
# name_vh = "ThuanHoa/ThuanHoa_VH.tif"
|
||||
# name_vv = "ThuanHoa/ThuanHoa_VV.tif"
|
||||
|
||||
# load dữ liệu sen1
|
||||
bbox = [105.5, 9.2, 106.4, 10.0]
|
||||
time_range = '2022-09-01/2023-10-01'
|
||||
# dsvh, dsvv = load_sen1(bbox, time_range)
|
||||
|
||||
name_vh = "vh-0922_0923-full_ST.tif"
|
||||
name_vv = "vv-0922_0923-full_ST.tif"
|
||||
|
||||
if not os.path.exists(name_vh):
|
||||
get_ipython().system('aws s3 cp s3://easi-asia-dc-data/staging/ctu/sentinel-1/vh-0922_0923-full_ST.tif vh-0922_0923-full_ST.tif')
|
||||
if not os.path.exists(name_vv):
|
||||
get_ipython().system('aws s3 cp s3://easi-asia-dc-data/staging/ctu/sentinel-1/vv-0922_0923-full_ST.tif vv-0922_0923-full_ST.tif')
|
||||
|
||||
bbox = [105.5, 9.2, 106.4, 10.0]
|
||||
time_range = '2022-09-01/2023-10-01'
|
||||
dsvh, dsvv = load_sen1(bbox, time_range)
|
||||
|
||||
|
||||
# In[27]:
|
||||
|
||||
|
||||
from sklearn.preprocessing import PolynomialFeatures
|
||||
from sklearn.linear_model import LinearRegression
|
||||
from sklearn.ensemble import RandomForestRegressor
|
||||
|
||||
|
||||
# In[28]:
|
||||
|
||||
|
||||
average_ndvi = average_ndvi[:, :7680, :8687]
|
||||
mask = ~np.isnan(average_ndvi)
|
||||
print(average_ndvi.shape)
|
||||
print(dsvh.shape)
|
||||
print(dsvv.shape)
|
||||
print(mask.shape)
|
||||
X_train = np.stack([dsvh.values[mask], dsvv.values[mask]], axis=1)
|
||||
y_train = average_ndvi.values[mask]
|
||||
|
||||
|
||||
# In[29]:
|
||||
|
||||
|
||||
model = LinearRegression()
|
||||
model.fit(X_train, y_train)
|
||||
|
||||
|
||||
# In[30]:
|
||||
|
||||
|
||||
X_pred = np.stack([dsvh.values[~mask], dsvv.values[~mask]], axis=1)
|
||||
average_ndvi.values[~mask] = model.predict(X_pred)
|
||||
|
||||
|
||||
# In[31]:
|
||||
|
||||
|
||||
average_ndvi_filled = xr.DataArray(average_ndvi, dims=average_ndvi.dims)
|
||||
|
||||
|
||||
# In[32]:
|
||||
|
||||
|
||||
plt.imshow(average_ndvi_filled.isel(time=6))
|
||||
|
||||
|
||||
# In[65]:
|
||||
|
||||
|
||||
plt.imshow(average_ndvi.isel(time=6))
|
||||
|
||||
|
||||
# In[33]:
|
||||
|
||||
|
||||
train_path = "train/ST_training data_updated_1130points.shp"
|
||||
|
||||
|
||||
# In[34]:
|
||||
|
||||
|
||||
train = load_train_data(train_path)
|
||||
|
||||
|
||||
# In[37]:
|
||||
|
||||
|
||||
datasets = get_data_sen1_and_sen2(train, average_ndvi_filled, dsvh, dsvv)
|
||||
|
||||
|
||||
# In[39]:
|
||||
|
||||
|
||||
# cấu hình nhãn dữ liệu
|
||||
label_mapping = {
|
||||
"Lua tom": "0",
|
||||
"Lua": "1",
|
||||
"CHN": "2",
|
||||
"CLN": "3",
|
||||
"TS": "4",
|
||||
"Song": "5",
|
||||
"Dat xay dung": "6",
|
||||
"Rung": "7"
|
||||
}
|
||||
|
||||
# chia tập dữ liệu train, val, test
|
||||
X_train, X_val, X_test, y_train, y_val, y_test = split_train_data(train, label_mapping, datasets)
|
||||
|
||||
|
||||
# In[40]:
|
||||
|
||||
|
||||
# Huấn luyện mô hình
|
||||
grid_search = train_with_rf(X_train, X_val, y_train, y_val)
|
||||
|
||||
|
||||
# In[41]:
|
||||
|
||||
|
||||
# kiểm tra độ chính xác với tập test
|
||||
y_pred_test = grid_search.predict(X_test)
|
||||
test_accuracy = accuracy_score(y_test, y_pred_test)
|
||||
print(f"Accuracy for test data {round(test_accuracy, 2)*100} %")
|
||||
|
||||
|
||||
# In[42]:
|
||||
|
||||
|
||||
# Lưu mô hình huấn luyện
|
||||
save_model("model_new.joblib", grid_search)
|
||||
|
||||
|
||||
# In[43]:
|
||||
|
||||
|
||||
# đóng client, cluster
|
||||
client.close()
|
||||
cluster.close()
|
||||
|
||||
|
||||
# In[ ]:
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user