diff --git a/01.train_ODC.ipynb b/.temp/01.train_ODC.ipynb similarity index 100% rename from 01.train_ODC.ipynb rename to .temp/01.train_ODC.ipynb diff --git a/01.train_ODC.py b/.temp/01.train_ODC.py similarity index 100% rename from 01.train_ODC.py rename to .temp/01.train_ODC.py diff --git a/01.train_ODC_XGBoost.ipynb b/.temp/01.train_ODC_XGBoost.ipynb similarity index 100% rename from 01.train_ODC_XGBoost.ipynb rename to .temp/01.train_ODC_XGBoost.ipynb diff --git a/01.train_ODC_XGBoost.py b/.temp/01.train_ODC_XGBoost.py similarity index 100% rename from 01.train_ODC_XGBoost.py rename to .temp/01.train_ODC_XGBoost.py diff --git a/01.train_ODC_local_with_Mic_supplyer.ipynb b/.temp/01.train_ODC_local_with_Mic_supplyer.ipynb similarity index 100% rename from 01.train_ODC_local_with_Mic_supplyer.ipynb rename to .temp/01.train_ODC_local_with_Mic_supplyer.ipynb diff --git a/01.train_ODC_local_with_Mic_supplyer.py b/.temp/01.train_ODC_local_with_Mic_supplyer.py similarity index 100% rename from 01.train_ODC_local_with_Mic_supplyer.py rename to .temp/01.train_ODC_local_with_Mic_supplyer.py diff --git a/02.predict_ODC.ipynb b/.temp/02.predict_ODC.ipynb similarity index 100% rename from 02.predict_ODC.ipynb rename to .temp/02.predict_ODC.ipynb diff --git a/02.predict_ODC.py b/.temp/02.predict_ODC.py similarity index 100% rename from 02.predict_ODC.py rename to .temp/02.predict_ODC.py diff --git a/03.compare_ODC.ipynb b/.temp/03.compare_ODC.ipynb similarity index 100% rename from 03.compare_ODC.ipynb rename to .temp/03.compare_ODC.ipynb diff --git a/03.compare_ODC.py b/.temp/03.compare_ODC.py similarity index 100% rename from 03.compare_ODC.py rename to .temp/03.compare_ODC.py diff --git a/Download_Cache_Colab.ipynb b/.temp/Download_Cache_Colab.ipynb similarity index 100% rename from Download_Cache_Colab.ipynb rename to .temp/Download_Cache_Colab.ipynb diff --git a/check_X.py b/.temp/check_X.py similarity index 100% rename from check_X.py rename to .temp/check_X.py diff --git a/check_missing.py b/.temp/check_missing.py similarity index 100% rename from check_missing.py rename to .temp/check_missing.py diff --git a/check_versions.py b/.temp/check_versions.py similarity index 100% rename from check_versions.py rename to .temp/check_versions.py diff --git a/check_xgboost_data.py b/.temp/check_xgboost_data.py similarity index 100% rename from check_xgboost_data.py rename to .temp/check_xgboost_data.py diff --git a/check_zeros.py b/.temp/check_zeros.py similarity index 100% rename from check_zeros.py rename to .temp/check_zeros.py diff --git a/cloud_removal_train.ipynb b/.temp/cloud_removal_train.ipynb similarity index 100% rename from cloud_removal_train.ipynb rename to .temp/cloud_removal_train.ipynb diff --git a/fix_all.py b/.temp/fix_all.py similarity index 100% rename from fix_all.py rename to .temp/fix_all.py diff --git a/fix_cache.py b/.temp/fix_cache.py similarity index 100% rename from fix_cache.py rename to .temp/fix_cache.py diff --git a/fix_new_train.py b/.temp/fix_new_train.py similarity index 100% rename from fix_new_train.py rename to .temp/fix_new_train.py diff --git a/fix_train.py b/.temp/fix_train.py similarity index 100% rename from fix_train.py rename to .temp/fix_train.py diff --git a/fix_typo.py b/.temp/fix_typo.py similarity index 100% rename from fix_typo.py rename to .temp/fix_typo.py diff --git a/generate_colab_notebook.py b/.temp/generate_colab_notebook.py similarity index 100% rename from generate_colab_notebook.py rename to .temp/generate_colab_notebook.py diff --git a/generate_previews.py b/.temp/generate_previews.py similarity index 100% rename from generate_previews.py rename to .temp/generate_previews.py diff --git a/generate_report_only.py b/.temp/generate_report_only.py similarity index 100% rename from generate_report_only.py rename to .temp/generate_report_only.py diff --git a/inspect_cells_train.py b/.temp/inspect_cells_train.py similarity index 100% rename from inspect_cells_train.py rename to .temp/inspect_cells_train.py diff --git a/inspect_model_odc.py b/.temp/inspect_model_odc.py similarity index 100% rename from inspect_model_odc.py rename to .temp/inspect_model_odc.py diff --git a/log_cnn.txt b/.temp/log_cnn.txt similarity index 100% rename from log_cnn.txt rename to .temp/log_cnn.txt diff --git a/log_dt.txt b/.temp/log_dt.txt similarity index 100% rename from log_dt.txt rename to .temp/log_dt.txt diff --git a/log_extract_fast.txt b/.temp/log_extract_fast.txt similarity index 100% rename from log_extract_fast.txt rename to .temp/log_extract_fast.txt diff --git a/log_extract_fusion.txt b/.temp/log_extract_fusion.txt similarity index 100% rename from log_extract_fusion.txt rename to .temp/log_extract_fusion.txt diff --git a/log_lightgbm.txt b/.temp/log_lightgbm.txt similarity index 100% rename from log_lightgbm.txt rename to .temp/log_lightgbm.txt diff --git a/log_mobilenet.txt b/.temp/log_mobilenet.txt similarity index 100% rename from log_mobilenet.txt rename to .temp/log_mobilenet.txt diff --git a/log_rf.txt b/.temp/log_rf.txt similarity index 100% rename from log_rf.txt rename to .temp/log_rf.txt diff --git a/log_svm.txt b/.temp/log_svm.txt similarity index 100% rename from log_svm.txt rename to .temp/log_svm.txt diff --git a/log_swin.txt b/.temp/log_swin.txt similarity index 100% rename from log_swin.txt rename to .temp/log_swin.txt diff --git a/log_train_2d.txt b/.temp/log_train_2d.txt similarity index 100% rename from log_train_2d.txt rename to .temp/log_train_2d.txt diff --git a/log_train_2d_final.txt b/.temp/log_train_2d_final.txt similarity index 100% rename from log_train_2d_final.txt rename to .temp/log_train_2d_final.txt diff --git a/log_train_2d_temporal.txt b/.temp/log_train_2d_temporal.txt similarity index 100% rename from log_train_2d_temporal.txt rename to .temp/log_train_2d_temporal.txt diff --git a/log_train_2d_temporal_pretrained.txt b/.temp/log_train_2d_temporal_pretrained.txt similarity index 100% rename from log_train_2d_temporal_pretrained.txt rename to .temp/log_train_2d_temporal_pretrained.txt diff --git a/log_train_v4.txt b/.temp/log_train_v4.txt similarity index 100% rename from log_train_v4.txt rename to .temp/log_train_v4.txt diff --git a/log_train_v5.txt b/.temp/log_train_v5.txt similarity index 100% rename from log_train_v5.txt rename to .temp/log_train_v5.txt diff --git a/log_train_v6.txt b/.temp/log_train_v6.txt similarity index 100% rename from log_train_v6.txt rename to .temp/log_train_v6.txt diff --git a/log_tune_swin_unet.txt b/.temp/log_tune_swin_unet.txt similarity index 100% rename from log_tune_swin_unet.txt rename to .temp/log_tune_swin_unet.txt diff --git a/log_ultimate_95.txt b/.temp/log_ultimate_95.txt similarity index 100% rename from log_ultimate_95.txt rename to .temp/log_ultimate_95.txt diff --git a/log_ultimate_v2.txt b/.temp/log_ultimate_v2.txt similarity index 100% rename from log_ultimate_v2.txt rename to .temp/log_ultimate_v2.txt diff --git a/log_ultimate_v3.txt b/.temp/log_ultimate_v3.txt similarity index 100% rename from log_ultimate_v3.txt rename to .temp/log_ultimate_v3.txt diff --git a/log_xgboost.txt b/.temp/log_xgboost.txt similarity index 100% rename from log_xgboost.txt rename to .temp/log_xgboost.txt diff --git a/new_import_ODC.py b/.temp/new_import_ODC.py similarity index 100% rename from new_import_ODC.py rename to .temp/new_import_ODC.py diff --git a/new_train.ipynb b/.temp/new_train.ipynb similarity index 100% rename from new_train.ipynb rename to .temp/new_train.ipynb diff --git a/patch_caching.py b/.temp/patch_caching.py similarity index 100% rename from patch_caching.py rename to .temp/patch_caching.py diff --git a/patch_load_sen1.py b/.temp/patch_load_sen1.py similarity index 100% rename from patch_load_sen1.py rename to .temp/patch_load_sen1.py diff --git a/patch_notebooks.py b/.temp/patch_notebooks.py similarity index 100% rename from patch_notebooks.py rename to .temp/patch_notebooks.py diff --git a/patch_rf_to_xgb.py b/.temp/patch_rf_to_xgb.py similarity index 100% rename from patch_rf_to_xgb.py rename to .temp/patch_rf_to_xgb.py diff --git a/processMask-test.ipynb b/.temp/processMask-test.ipynb similarity index 100% rename from processMask-test.ipynb rename to .temp/processMask-test.ipynb diff --git a/processMask-test.py b/.temp/processMask-test.py similarity index 100% rename from processMask-test.py rename to .temp/processMask-test.py diff --git a/processMask.ipynb b/.temp/processMask.ipynb similarity index 100% rename from processMask.ipynb rename to .temp/processMask.ipynb diff --git a/processMask.py b/.temp/processMask.py similarity index 100% rename from processMask.py rename to .temp/processMask.py diff --git a/rebuild_notebook.py b/.temp/rebuild_notebook.py similarity index 100% rename from rebuild_notebook.py rename to .temp/rebuild_notebook.py diff --git a/remote-sensing.zip:Zone.Identifier b/.temp/remote-sensing.zip:Zone.Identifier similarity index 100% rename from remote-sensing.zip:Zone.Identifier rename to .temp/remote-sensing.zip:Zone.Identifier diff --git a/remove_datacube.py b/.temp/remove_datacube.py similarity index 100% rename from remove_datacube.py rename to .temp/remove_datacube.py diff --git a/run_mock_parallel_and_compare.py b/.temp/run_mock_parallel_and_compare.py similarity index 100% rename from run_mock_parallel_and_compare.py rename to .temp/run_mock_parallel_and_compare.py diff --git a/test_VH_VV.ipynb b/.temp/test_VH_VV.ipynb similarity index 100% rename from test_VH_VV.ipynb rename to .temp/test_VH_VV.ipynb diff --git a/test_VH_VV.py b/.temp/test_VH_VV.py similarity index 100% rename from test_VH_VV.py rename to .temp/test_VH_VV.py diff --git a/test_bug.py b/.temp/test_bug.py similarity index 100% rename from test_bug.py rename to .temp/test_bug.py diff --git a/test_caching.py b/.temp/test_caching.py similarity index 100% rename from test_caching.py rename to .temp/test_caching.py diff --git a/test_checkpoint.py b/.temp/test_checkpoint.py similarity index 100% rename from test_checkpoint.py rename to .temp/test_checkpoint.py diff --git a/test_cloud.py b/.temp/test_cloud.py similarity index 100% rename from test_cloud.py rename to .temp/test_cloud.py diff --git a/test_cloud2.py b/.temp/test_cloud2.py similarity index 100% rename from test_cloud2.py rename to .temp/test_cloud2.py diff --git a/test_cloud_removal.py b/.temp/test_cloud_removal.py similarity index 100% rename from test_cloud_removal.py rename to .temp/test_cloud_removal.py diff --git a/test_cloud_training.py b/.temp/test_cloud_training.py similarity index 100% rename from test_cloud_training.py rename to .temp/test_cloud_training.py diff --git a/test_cloud_upload.py b/.temp/test_cloud_upload.py similarity index 100% rename from test_cloud_upload.py rename to .temp/test_cloud_upload.py diff --git a/test_empty_points.py b/.temp/test_empty_points.py similarity index 100% rename from test_empty_points.py rename to .temp/test_empty_points.py diff --git a/test_extract_one.py b/.temp/test_extract_one.py similarity index 100% rename from test_extract_one.py rename to .temp/test_extract_one.py diff --git a/test_feature_extractor.py b/.temp/test_feature_extractor.py similarity index 100% rename from test_feature_extractor.py rename to .temp/test_feature_extractor.py diff --git a/test_gdf.py b/.temp/test_gdf.py similarity index 100% rename from test_gdf.py rename to .temp/test_gdf.py diff --git a/test_gdf_cols.py b/.temp/test_gdf_cols.py similarity index 100% rename from test_gdf_cols.py rename to .temp/test_gdf_cols.py diff --git a/test_gpu.py b/.temp/test_gpu.py similarity index 100% rename from test_gpu.py rename to .temp/test_gpu.py diff --git a/test_keys.py b/.temp/test_keys.py similarity index 100% rename from test_keys.py rename to .temp/test_keys.py diff --git a/test_model_manager.py b/.temp/test_model_manager.py similarity index 100% rename from test_model_manager.py rename to .temp/test_model_manager.py diff --git a/test_model_struct.py b/.temp/test_model_struct.py similarity index 100% rename from test_model_struct.py rename to .temp/test_model_struct.py diff --git a/test_new_features.py b/.temp/test_new_features.py similarity index 100% rename from test_new_features.py rename to .temp/test_new_features.py diff --git a/test_odc_stac.py b/.temp/test_odc_stac.py similarity index 100% rename from test_odc_stac.py rename to .temp/test_odc_stac.py diff --git a/test_overlap.py b/.temp/test_overlap.py similarity index 100% rename from test_overlap.py rename to .temp/test_overlap.py diff --git a/test_overlap2.py b/.temp/test_overlap2.py similarity index 100% rename from test_overlap2.py rename to .temp/test_overlap2.py diff --git a/test_overlap_issue.py b/.temp/test_overlap_issue.py similarity index 100% rename from test_overlap_issue.py rename to .temp/test_overlap_issue.py diff --git a/test_planetary.py b/.temp/test_planetary.py similarity index 100% rename from test_planetary.py rename to .temp/test_planetary.py diff --git a/test_planetary2.py b/.temp/test_planetary2.py similarity index 100% rename from test_planetary2.py rename to .temp/test_planetary2.py diff --git a/test_planetary_computer.py b/.temp/test_planetary_computer.py similarity index 100% rename from test_planetary_computer.py rename to .temp/test_planetary_computer.py diff --git a/test_raw.py b/.temp/test_raw.py similarity index 100% rename from test_raw.py rename to .temp/test_raw.py diff --git a/test_shape.py b/.temp/test_shape.py similarity index 100% rename from test_shape.py rename to .temp/test_shape.py diff --git a/test_spatial_filter.py b/.temp/test_spatial_filter.py similarity index 100% rename from test_spatial_filter.py rename to .temp/test_spatial_filter.py diff --git a/test_training_api.py b/.temp/test_training_api.py similarity index 100% rename from test_training_api.py rename to .temp/test_training_api.py diff --git a/test_unet.py b/.temp/test_unet.py similarity index 100% rename from test_unet.py rename to .temp/test_unet.py diff --git a/test_y_mapped.py b/.temp/test_y_mapped.py similarity index 100% rename from test_y_mapped.py rename to .temp/test_y_mapped.py diff --git a/api_server.py b/api_server.py index 3a87376..eaa9e74 100644 --- a/api_server.py +++ b/api_server.py @@ -26,20 +26,20 @@ import socket import urllib.request # Import report generator -from report_generator import generate_training_report, generate_prediction_report +from scripts.inference.report_generator import generate_training_report, generate_prediction_report # Import Model Manager -from model_manager import ModelManager, get_model_manager +from core.model_manager import ModelManager, get_model_manager # Import Vietnam provinces data -from vietnam_provinces import get_all_provinces, get_provinces_by_region, get_province_bbox, search_province -from vietnam_provinces_merged import ( +from core.vietnam_provinces import get_all_provinces, get_provinces_by_region, get_province_bbox, search_province +from core.vietnam_provinces_merged import ( get_all_provinces_32, get_provinces_by_region_32, get_province_bbox_32, search_province_32, get_merged_info, get_provinces_statistics ) # Import cloud removal module -from cloud_removal import process_cloud_removal, get_available_methods +from core.cloud_removal import process_cloud_removal, get_available_methods # Import planetary computer libraries (conditional) try: diff --git a/backup_S3_download_Amazon/new_import_S3.py b/backup_S3_download_Amazon/new_import_S3.py deleted file mode 100644 index 6f1e52a..0000000 --- a/backup_S3_download_Amazon/new_import_S3.py +++ /dev/null @@ -1,354 +0,0 @@ -import matplotlib.pyplot as plt - -# Common imports and settings -import os, sys -os.environ['USE_PYGEOS'] = '0' -from IPython.display import Markdown -import pandas as pd -pd.set_option("display.max_rows", None) -import xarray as xr - -# Datacube -import datacube -from datacube.utils.rio import configure_s3_access -from datacube.utils import masking -from datacube.utils.cog import write_cog -# https://github.com/GeoscienceAustralia/dea-notebooks/tree/develop/Tools -from dea_tools.plotting import display_map, rgb -from dea_tools.datahandling import mostcommon_crs - -# EASI defaults -easinotebooksrepo = '/home/jovyan/easi-notebooks' -if easinotebooksrepo not in sys.path: sys.path.append(easinotebooksrepo) -from easi_tools import EasiDefaults, xarray_object_size, notebook_utils, unset_cachingproxy -from easi_tools.load_s2l2a import load_s2l2a_with_offset -from dask.distributed import progress - -# Data tools -import numpy as np -from datetime import datetime - -# Datacube -from datacube.utils import masking # https://github.com/opendatacube/datacube-core/blob/develop/datacube/utils/masking.py -from odc.algo import enum_to_bool # https://github.com/opendatacube/odc-algo/blob/main/odc/algo/_masking.py -from odc.algo import xr_reproject # https://github.com/opendatacube/odc-algo/blob/main/odc/algo/_warp.py -from datacube.utils.geometry import GeoBox, box # https://github.com/opendatacube/datacube-core/blob/develop/datacube/utils/geometry/_base.py - -# Holoviews, Datashader and Bokeh -import hvplot.pandas -import hvplot.xarray -import holoviews as hv -import panel as pn -import colorcet as cc -import cartopy.crs as ccrs -from datashader import reductions -from holoviews import opts -from utils import load_data_geo -import rasterio -import rioxarray -# import geoviews as gv -# from holoviews.operation.datashader import rasterize -hv.extension('bokeh', logo=False) - -from deafrica_tools.bandindices import calculate_indices -from sklearn.ensemble import RandomForestClassifier -from sklearn.model_selection import train_test_split -from sklearn.metrics import accuracy_score, classification_report -from sklearn.preprocessing import LabelEncoder - -from sklearn.pipeline import Pipeline -from sklearn.ensemble import RandomForestClassifier -from sklearn.impute import SimpleImputer -from sklearn.preprocessing import StandardScaler -from sklearn.model_selection import GridSearchCV -from sklearn.model_selection import train_test_split -from sklearn.metrics import accuracy_score -from shapely.geometry import Point, Polygon -import geopandas as gpd -from pyproj import CRS -from matplotlib.colors import ListedColormap -from holoviews import opts -from datashader import reductions -from bokeh.models.tickers import FixedTicker -from rioxarray.merge import merge_arrays - -import joblib - - -def load_data(dc, date_range, longtitude_range, latitude_range): - product = 's2_l2a' - query = { - 'product': product, # Product name - 'x': longtitude_range, # "x" axis bounds - 'y': latitude_range, # "y" axis bounds - 'time': date_range, # Any parsable date strings - } - native_crs = notebook_utils.mostcommon_crs(dc, query) - print(f'Most common native CRS: {native_crs}') - measurements = [ 'red', 'nir', 'scl'] - - load_params = { - 'measurements': measurements, # Selected measurement or alias names - 'output_crs': native_crs, # Target EPSG code - 'resolution': (-10, 10), # Target resolution - 'group_by': 'solar_day', # Scene grouping - 'dask_chunks': {'x': 2048, 'y': 2048}, # Dask chunks - } - data = load_s2l2a_with_offset( - dc, - query | load_params # Combine the two dicts that contain our search and load parameters - ) - return data - - -def mask_clean(data): - flag_name = 'scl' - flag_desc = masking.describe_variable_flags(data[flag_name]) # Pandas dataframe - display(flag_desc) - display(flag_desc.loc['qa'].values[1]) - # Create a "data quality" Mask layer - flags_def = flag_desc.loc['qa'].values[1] - good_pixel_flags = [flags_def[str(i)] for i in [2, 4, 5, 6]] # To pass strings to enum_to_bool() - - # enum_to_bool calculates the pixel-wise "or" of each set of pixels given by good_pixel_flags - # 1 = good data - # 0 = "bad" data - good_pixel_mask = enum_to_bool(data[flag_name], good_pixel_flags) - data_layer_names = [x for x in data.data_vars if x != 'scl'] - # Apply good pixel mask to blue, green, red and nir. - result = data[data_layer_names].where(good_pixel_mask).persist() - return result - - -def fill_nan(ndvi, time_split): - rs = [] - for times in time_split: - tmp = ndvi.sel(time=times) - fill_ds = tmp.sel(time=times).bfill(dim='time') - fill_ds = fill_ds.sel(time=times).ffill(dim='time') - rs.append(fill_ds) - merged_ndvi = xr.concat([i for i in rs], dim="time") - fill_m = merged_ndvi.bfill(dim="time") - fill_m = fill_m.ffill(dim="time") - return fill_m - - -def load_train_data(train_path): - train = load_data_geo(train_path) - return train - - -def load_sen1(name_vh, name_vv): - dsvv = rioxarray.open_rasterio(name_vv) - dsvh = rioxarray.open_rasterio(name_vh) - return dsvh, dsvv - - -def get_data_sen1_and_sen2(train, average_ndvi, dsvh, dsvv): - loaded_datasets = {} - for idx, point in train.iterrows(): - key = f"point_{idx + 1}" - try: - ndvi_data = average_ndvi.sel(x=point.geometry.x, y=point.geometry.y, method='nearest').values - vh_data = dsvh.sel(x=point.geometry.x, y=point.geometry.y, method='nearest').values - vv_data = dsvv.sel(x=point.geometry.x, y=point.geometry.y, method='nearest').values - loaded_datasets[key] = { - "data": np.concatenate((ndvi_data, vh_data, vv_data)), - "label": point.HT_code - } - except Exception as e: - # loaded_datasets[key] = None - print(e) - return loaded_datasets - - -def split_train_data(train, label_mapping, datasets): - label_encoder = LabelEncoder() - - # Fit and transform the labels - labels = train.Hientrang.values - numeric_labels = label_encoder.fit_transform([label_mapping[label] for label in labels]) - X = [] - x_new = [] - lb_new = [] - for k, v in datasets.items(): - X.append(v) - for i in range(len(X)): - if X[i] is not None: - x_new.append(X[i]["data"]) - lb_new.append(numeric_labels[i]) - X_train, X_temp, y_train, y_temp= train_test_split(x_new, lb_new, test_size=0.4, random_state=42) - X_val, X_test, y_val, y_test = train_test_split(X_temp, y_temp, test_size=0.5, random_state=42) - return X_train, X_val, X_test, y_train, y_val, y_test - - -def train_with_rf(X_train, X_val, y_train, y_val): - # Takes 1-2 minutes to complete - - # Tạo RandomForestClassifier mặc định để sử dụng làm mô hình ban đầu trong pipeline - base_model = RandomForestClassifier(random_state=42, n_jobs=-1) - - # Tạo pipeline - pipeline = Pipeline([ - # ('imputer', SimpleImputer(strategy='mean')), - ('scaler', StandardScaler()), - ('classifier', base_model), - ]) - # Thiết lập các tham số bạn muốn tối ưu hóa - param_grid = { - 'classifier__n_estimators': [100, 300, 500, 700, 1000], - 'classifier__max_depth': [6, 8, 10, 15, 20], - 'classifier__criterion': ['gini', 'entropy'], - } - - # Sử dụng GridSearchCV để tìm bộ tham số tốt nhất - grid_search = GridSearchCV(pipeline, param_grid, cv=5, scoring='accuracy', n_jobs=-1) - grid_search.fit(X_train, y_train) - - # In ra bộ tham số tốt nhất - best_params = grid_search.best_params_ - print("Best Parameters:", best_params) - - # Dự đoán trên tập kiểm tra - y_pred = grid_search.predict(X_val) - - # Đánh giá kết quả - accuracy = accuracy_score(y_val, y_pred) - print(f"Accuracy: {round(accuracy, 2)*100} %") - return grid_search - - -def save_model(name_file, grid_search): - dir_save_model = "model_train" - if not os.path.exists(dir_save_model): - os.mkdir(dir_save_model) - joblib.dump(grid_search, os.path.join(dir_save_model, name_file)) - print("Done!") - - -def predict(model, data_crs, ndvi, vh, vv): - data_predict = [] - for i in range(ndvi.shape[1]): - ndvi_tmp = ndvi.isel(y=i).values - vh_data = vh.sel(y=ndvi.y.values[i], method='nearest').values - vv_data = vv.sel(y=ndvi.y.values[i], method='nearest').values - all_tmp = np.concatenate((ndvi_tmp, vh_data, vv_data), axis=0) - data_predict.extend(all_tmp.T) - y_pred = model.predict(data_predict) - final_label = y_pred.reshape(ndvi.y.shape[0], ndvi.x.shape[0]) - - final_xarray_save = xr.DataArray(final_label, dims=("y", "x")) - final_xarray_save = final_xarray_save.rio.write_crs(data_crs) - - x_values = ndvi.x.values - y_values = ndvi.y.values - - data_array = xr.DataArray(final_xarray_save, - coords={'x': x_values, 'y': y_values}, - dims=['y', 'x']) - data_array = data_array.rio.write_crs(ndvi.rio.crs) - return data_array - - -def cut_according_shp(thuanhoa_path, average_ndvi, data_array): - gdf = gpd.read_file(thuanhoa_path) - gdf = gdf.to_crs(average_ndvi.rio.crs) - polygon_coords = list(gdf.geometry.values[0].exterior.coords) - polygon_coordinates = [(x, y) for x, y in polygon_coords] - - geometries = [ - { - 'type': 'Polygon', - 'coordinates': [polygon_coordinates] - } - ] - region_result = data_array.rio.clip(geometries, data_array.rio.crs, drop=False) - region_result = region_result.where(region_result >= 0, float('nan')) - return region_result - - -def compare(KD_path, KetQuaPhanLoaiDat, CODE_MAP, HT_MAP): - gdf = gpd.read_file(KD_path, crs="EPSG:9209") - polygon = gdf.geometry.values - label = gdf.tenchu.values - ouput_image = rioxarray.open_rasterio(KetQuaPhanLoaiDat) - code_tq = HT_MAP["TQ"]["data"][0] - code_pnn = HT_MAP["PNN"]["data"][0] - result = {} - for key, values in HT_MAP.items(): - print(f"process {key}") - array_list = [] - for i in range(len(polygon)): - po = polygon[i] - lb = label[i] - code_lb = CODE_MAP.get(lb, code_tq) - try: - qr = ouput_image.rio.clip([po], "EPSG:9209") - if code_lb in values["data"]: - if code_lb == code_pnn: - qr = qr.where((qr != float(code_pnn)), np.nan) - # qr = qr.where((qr != 3.0), np.nan) - elif code_lb == code_tq: - qr = qr.where((qr != float(code_pnn)), np.nan) - qr = qr.where((qr != 3.0), np.nan) - else: - qr = qr.where(qr != float(code_lb), np.nan) - else: - qr.values[:, :, :] = np.nan - array_list.append(qr) - except Exception as e: - pass - result.update({key: array_list}) - return result - - -def save_result(result, HT_MAP): - # cmap = ListedColormap(colors) - save_path = "ThuanHoa/KetQua" - if not os.path.exists(save_path): - os.mkdir(save_path) - - for k, v in result.items(): - rs = merge_arrays(v, nodata = np.nan) - rs.rio.to_raster(f"{save_path}/{k}.tif") - print(f"save {save_path}/{k}.tif") - # img = rs.plot(cmap=cmap, add_colorbar=False) - # cbar = plt.colorbar(img) - # cbar.ax.set_yticklabels(labels) - # plt.title(f'{HT_MAP[k]["name"]}') - # plt.axis('off') - # plt.show() - - -def accuracy_test(test, data_array): - # cấu hình nhãn dữ liệu - label_mapping = { - "Lua tom": "0", - "Lua": "1", - "CHN": "2", - "CLN": "3", - "TS": "4", - "Song": "5", - "Dat xay dung": "6", - "Rung": "7" - } - - chk = [] - pred = [] - dd = [] - for idx, point in test.iterrows(): - label = point.LULC - predict = data_array.sel(x=point.geometry.x, y=point.geometry.y, method='nearest').values - pred.append(label_mapping[label]) - dd.append(str(predict)) - chk.append(predict == int(label_mapping[label])) - test["code"] = pred - test["dd"] = dd - test["check"] = chk - path = "ThuanHoa/TestAccuracy" - if not os.path.exists(path): - os.mkdir(path) - test.to_file(f"{path}/result.shp") - - percentage_true = np.mean(chk) * 100 - print(f"độ chính xác: {percentage_true:.2f}%") \ No newline at end of file diff --git a/backup_code_training/CSIROBoeingPhase5-Vietnam b/backup_code_training/CSIROBoeingPhase5-Vietnam deleted file mode 160000 index 8f0cb55..0000000 --- a/backup_code_training/CSIROBoeingPhase5-Vietnam +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8f0cb55cba309998800203d4a8982d9994be3a12 diff --git a/core/__init__.py b/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cloud_removal.py b/core/cloud_removal.py similarity index 100% rename from cloud_removal.py rename to core/cloud_removal.py diff --git a/feature_extractor.py b/core/feature_extractor.py similarity index 100% rename from feature_extractor.py rename to core/feature_extractor.py diff --git a/model_manager.py b/core/model_manager.py similarity index 100% rename from model_manager.py rename to core/model_manager.py diff --git a/ndvi_data_loader.py b/core/ndvi_data_loader.py similarity index 100% rename from ndvi_data_loader.py rename to core/ndvi_data_loader.py diff --git a/utils.py b/core/utils.py similarity index 100% rename from utils.py rename to core/utils.py diff --git a/vietnam_provinces.py b/core/vietnam_provinces.py similarity index 100% rename from vietnam_provinces.py rename to core/vietnam_provinces.py diff --git a/vietnam_provinces_merged.py b/core/vietnam_provinces_merged.py similarity index 100% rename from vietnam_provinces_merged.py rename to core/vietnam_provinces_merged.py diff --git a/md/GopYCuaHoiDong.md b/md/GopYCuaHoiDong.md new file mode 100644 index 0000000..fa8aeac --- /dev/null +++ b/md/GopYCuaHoiDong.md @@ -0,0 +1,10 @@ +Dear Author(s): Mr./Ms. Phan Vo Dinh Hien, Phan Duyen, Truong Minh Thai + +Thank you for submitting your manuscript to CTU Journal of Innovation and Sustainable Development. After an initial editorial assessment, we regret to inform you that the manuscript is not yet ready to be sent out for peer review in its present form. + +The topic is relevant and has potential, particularly in relation to the application of AI, satellite imagery, and land cover classification in a practical local context. However, the current version still needs substantial improvement before it can be considered further. In particular, the methodology should be described in much greater detail, including the study area, data sources, ground-truth data, sampling and validation procedures, model settings, and evaluation approach. The results should also be expanded with stronger evidence, clearer tables and figures, and more detailed interpretation to support the main claims of the study. In addition, the manuscript should be carefully revised to follow the journal format and to adopt a more formal academic writing style. + +We therefore recommend that the authors revise the manuscript thoroughly and resubmit it as a new submission when it is more fully developed. We hope the comments provided here will be helpful in strengthening the paper, and we would be pleased to consider a substantially improved version in the future. + +Sincerely, +Editor. \ No newline at end of file diff --git a/document/Microstation V8_Thuan Hoa land use inventory.dgn b/md/Microstation V8_Thuan Hoa land use inventory.dgn similarity index 100% rename from document/Microstation V8_Thuan Hoa land use inventory.dgn rename to md/Microstation V8_Thuan Hoa land use inventory.dgn diff --git a/document/README.md b/md/README.md similarity index 100% rename from document/README.md rename to md/README.md diff --git a/document/Swin-Unet.md b/md/Swin-Unet.md similarity index 100% rename from document/Swin-Unet.md rename to md/Swin-Unet.md diff --git a/document/TaiDuLieuVaTrainV4.md b/md/TaiDuLieuVaTrainV4.md similarity index 100% rename from document/TaiDuLieuVaTrainV4.md rename to md/TaiDuLieuVaTrainV4.md diff --git a/document/giai_thich_quy_trinh_phan_loai_dat.md b/md/giai_thich_quy_trinh_phan_loai_dat.md similarity index 100% rename from document/giai_thich_quy_trinh_phan_loai_dat.md rename to md/giai_thich_quy_trinh_phan_loai_dat.md diff --git a/document/implementation_plan_radar_1.md b/md/implementation_plan_radar_1.md similarity index 100% rename from document/implementation_plan_radar_1.md rename to md/implementation_plan_radar_1.md diff --git a/document/ketquacuoicung.md b/md/ketquacuoicung.md similarity index 100% rename from document/ketquacuoicung.md rename to md/ketquacuoicung.md diff --git a/document/plan.txt b/md/plan.txt similarity index 100% rename from document/plan.txt rename to md/plan.txt diff --git a/document/v5_ketqua.md b/md/v5_ketqua.md similarity index 100% rename from document/v5_ketqua.md rename to md/v5_ketqua.md diff --git a/cache_all_data.py b/scripts/extraction/cache_all_data.py similarity index 100% rename from cache_all_data.py rename to scripts/extraction/cache_all_data.py diff --git a/create_odc_metadata.py b/scripts/extraction/create_odc_metadata.py similarity index 100% rename from create_odc_metadata.py rename to scripts/extraction/create_odc_metadata.py diff --git a/extract_fusion_32ch.py b/scripts/extraction/extract_fusion_32ch.py similarity index 99% rename from extract_fusion_32ch.py rename to scripts/extraction/extract_fusion_32ch.py index 2dc3f75..1b8a991 100644 --- a/extract_fusion_32ch.py +++ b/scripts/extraction/extract_fusion_32ch.py @@ -17,7 +17,7 @@ from pyproj import Transformer import warnings warnings.filterwarnings('ignore') -from cloud_removal import DeepInpaintingStrategy +from core.cloud_removal import DeepInpaintingStrategy def get_s2_items(bbox, time_range): catalog = pystac_client.Client.open( diff --git a/extract_fusion_fast.py b/scripts/extraction/extract_fusion_fast.py similarity index 99% rename from extract_fusion_fast.py rename to scripts/extraction/extract_fusion_fast.py index 93a0c4b..f517d6d 100644 --- a/extract_fusion_fast.py +++ b/scripts/extraction/extract_fusion_fast.py @@ -17,7 +17,7 @@ from pyproj import Transformer import warnings warnings.filterwarnings('ignore') -from cloud_removal import DeepInpaintingStrategy +from core.cloud_removal import DeepInpaintingStrategy # Add GDAL optimizations for fast HTTP access os.environ["GDAL_HTTP_MAX_RETRY"] = "5" diff --git a/extract_s1_s2_fusion.py b/scripts/extraction/extract_s1_s2_fusion.py similarity index 99% rename from extract_s1_s2_fusion.py rename to scripts/extraction/extract_s1_s2_fusion.py index a6bbb28..3787690 100644 --- a/extract_s1_s2_fusion.py +++ b/scripts/extraction/extract_s1_s2_fusion.py @@ -19,7 +19,7 @@ from sklearn.metrics import accuracy_score, classification_report from tqdm import tqdm from joblib import Parallel, delayed -from cloud_removal import DeepInpaintingStrategy +from core.cloud_removal import DeepInpaintingStrategy def get_s2_items(bbox, time_range): catalog = pystac_client.Client.open( diff --git a/analyze_data.py b/scripts/inference/analyze_data.py similarity index 100% rename from analyze_data.py rename to scripts/inference/analyze_data.py diff --git a/report_generator.py b/scripts/inference/report_generator.py similarity index 100% rename from report_generator.py rename to scripts/inference/report_generator.py diff --git a/run_prediction_new.py b/scripts/inference/run_prediction_new.py similarity index 100% rename from run_prediction_new.py rename to scripts/inference/run_prediction_new.py diff --git a/create_train_scripts.py b/scripts/training/create_train_scripts.py similarity index 100% rename from create_train_scripts.py rename to scripts/training/create_train_scripts.py diff --git a/new_train.py b/scripts/training/new_train.py similarity index 100% rename from new_train.py rename to scripts/training/new_train.py diff --git a/run_all_parallel.py b/scripts/training/run_all_parallel.py similarity index 100% rename from run_all_parallel.py rename to scripts/training/run_all_parallel.py diff --git a/train_all_land_models.py b/scripts/training/train_all_land_models.py similarity index 100% rename from train_all_land_models.py rename to scripts/training/train_all_land_models.py diff --git a/train_cloud_cnn.py b/scripts/training/train_cloud_cnn.py similarity index 100% rename from train_cloud_cnn.py rename to scripts/training/train_cloud_cnn.py diff --git a/train_cloud_removal.py b/scripts/training/train_cloud_removal.py similarity index 100% rename from train_cloud_removal.py rename to scripts/training/train_cloud_removal.py diff --git a/train_cloud_swin_unet.py b/scripts/training/train_cloud_swin_unet.py similarity index 100% rename from train_cloud_swin_unet.py rename to scripts/training/train_cloud_swin_unet.py diff --git a/train_land_2d_patch.py b/scripts/training/train_land_2d_patch.py similarity index 99% rename from train_land_2d_patch.py rename to scripts/training/train_land_2d_patch.py index a6bbb28..3787690 100644 --- a/train_land_2d_patch.py +++ b/scripts/training/train_land_2d_patch.py @@ -19,7 +19,7 @@ from sklearn.metrics import accuracy_score, classification_report from tqdm import tqdm from joblib import Parallel, delayed -from cloud_removal import DeepInpaintingStrategy +from core.cloud_removal import DeepInpaintingStrategy def get_s2_items(bbox, time_range): catalog = pystac_client.Client.open( diff --git a/train_land_cnn_gpu.py b/scripts/training/train_land_cnn_gpu.py similarity index 100% rename from train_land_cnn_gpu.py rename to scripts/training/train_land_cnn_gpu.py diff --git a/train_land_decision_tree_gpu.py b/scripts/training/train_land_decision_tree_gpu.py similarity index 100% rename from train_land_decision_tree_gpu.py rename to scripts/training/train_land_decision_tree_gpu.py diff --git a/train_land_lightgbm.py b/scripts/training/train_land_lightgbm.py similarity index 100% rename from train_land_lightgbm.py rename to scripts/training/train_land_lightgbm.py diff --git a/train_land_lightgbm_gpu.py b/scripts/training/train_land_lightgbm_gpu.py similarity index 100% rename from train_land_lightgbm_gpu.py rename to scripts/training/train_land_lightgbm_gpu.py diff --git a/train_land_mobilenet_lraspp_gpu.py b/scripts/training/train_land_mobilenet_lraspp_gpu.py similarity index 100% rename from train_land_mobilenet_lraspp_gpu.py rename to scripts/training/train_land_mobilenet_lraspp_gpu.py diff --git a/train_land_random_forest_gpu.py b/scripts/training/train_land_random_forest_gpu.py similarity index 100% rename from train_land_random_forest_gpu.py rename to scripts/training/train_land_random_forest_gpu.py diff --git a/train_land_randomforest.py b/scripts/training/train_land_randomforest.py similarity index 100% rename from train_land_randomforest.py rename to scripts/training/train_land_randomforest.py diff --git a/train_land_svm_gpu.py b/scripts/training/train_land_svm_gpu.py similarity index 100% rename from train_land_svm_gpu.py rename to scripts/training/train_land_svm_gpu.py diff --git a/train_land_swin_unet_gpu.py b/scripts/training/train_land_swin_unet_gpu.py similarity index 100% rename from train_land_swin_unet_gpu.py rename to scripts/training/train_land_swin_unet_gpu.py diff --git a/train_land_xgboost_gpu.py b/scripts/training/train_land_xgboost_gpu.py similarity index 100% rename from train_land_xgboost_gpu.py rename to scripts/training/train_land_xgboost_gpu.py diff --git a/train_module.py b/scripts/training/train_module.py similarity index 99% rename from train_module.py rename to scripts/training/train_module.py index 0d5dfea..bafeaac 100644 --- a/train_module.py +++ b/scripts/training/train_module.py @@ -351,7 +351,7 @@ from pystac_client import Client from odc.stac import load as stac_load # Feature extraction -from feature_extractor import get_feature_extractor +from core.feature_extractor import get_feature_extractor def train_model( diff --git a/train_ndvi_convlstm.py b/scripts/training/train_ndvi_convlstm.py similarity index 97% rename from train_ndvi_convlstm.py rename to scripts/training/train_ndvi_convlstm.py index 4fad05f..abde914 100644 --- a/train_ndvi_convlstm.py +++ b/scripts/training/train_ndvi_convlstm.py @@ -7,7 +7,7 @@ import torch.nn as nn import torch.optim as optim from torch.utils.data import DataLoader from tqdm import tqdm -from ndvi_data_loader import NDVITimeSeriesDataset +from core.ndvi_data_loader import NDVITimeSeriesDataset print("=" * 70) print("🚀 Training ConvLSTM Spatial-Temporal model for NDVI (REAL DATA & GPU)") diff --git a/train_ndvi_ensemble.py b/scripts/training/train_ndvi_ensemble.py similarity index 96% rename from train_ndvi_ensemble.py rename to scripts/training/train_ndvi_ensemble.py index 466137e..c138f00 100644 --- a/train_ndvi_ensemble.py +++ b/scripts/training/train_ndvi_ensemble.py @@ -7,7 +7,7 @@ import numpy as np from sklearn.ensemble import VotingRegressor from sklearn.linear_model import LinearRegression from sklearn.ensemble import RandomForestRegressor -from ndvi_data_loader import NDVITimeSeriesDataset +from core.ndvi_data_loader import NDVITimeSeriesDataset print("=" * 70) print("🚀 Training Multi-Model Ensemble for NDVI (REAL DATA & CPU)") diff --git a/train_ndvi_hybrid_physics.py b/scripts/training/train_ndvi_hybrid_physics.py similarity index 96% rename from train_ndvi_hybrid_physics.py rename to scripts/training/train_ndvi_hybrid_physics.py index be1b67c..c111a3d 100644 --- a/train_ndvi_hybrid_physics.py +++ b/scripts/training/train_ndvi_hybrid_physics.py @@ -5,7 +5,7 @@ import json import joblib import numpy as np import xgboost as xgb -from ndvi_data_loader import NDVITimeSeriesDataset +from core.ndvi_data_loader import NDVITimeSeriesDataset print("=" * 70) print("🚀 Training Hybrid Physics-ML model for NDVI (REAL DATA & GPU)") diff --git a/train_ndvi_lstm_gru.py b/scripts/training/train_ndvi_lstm_gru.py similarity index 97% rename from train_ndvi_lstm_gru.py rename to scripts/training/train_ndvi_lstm_gru.py index 68f2138..c1f1b25 100644 --- a/train_ndvi_lstm_gru.py +++ b/scripts/training/train_ndvi_lstm_gru.py @@ -7,7 +7,7 @@ import torch.nn as nn import torch.optim as optim from torch.utils.data import DataLoader from tqdm import tqdm -from ndvi_data_loader import NDVITimeSeriesDataset +from core.ndvi_data_loader import NDVITimeSeriesDataset print("=" * 70) print("🚀 Training LSTM/GRU Time Series model for NDVI (REAL DATA & GPU)") diff --git a/train_ndvi_statistical.py b/scripts/training/train_ndvi_statistical.py similarity index 95% rename from train_ndvi_statistical.py rename to scripts/training/train_ndvi_statistical.py index 7811ddb..c9aa2e7 100644 --- a/train_ndvi_statistical.py +++ b/scripts/training/train_ndvi_statistical.py @@ -5,7 +5,7 @@ import json import joblib import numpy as np from statsmodels.tsa.statespace.sarimax import SARIMAX -from ndvi_data_loader import NDVITimeSeriesDataset +from core.ndvi_data_loader import NDVITimeSeriesDataset print("=" * 70) print("🚀 Training Statistical Model (SARIMA) for NDVI (REAL DATA & CPU)") diff --git a/train_ultimate_95.py b/scripts/training/train_ultimate_95.py similarity index 100% rename from train_ultimate_95.py rename to scripts/training/train_ultimate_95.py diff --git a/train_ultimate_v2.py b/scripts/training/train_ultimate_v2.py similarity index 100% rename from train_ultimate_v2.py rename to scripts/training/train_ultimate_v2.py diff --git a/train_ultimate_v3.py b/scripts/training/train_ultimate_v3.py similarity index 100% rename from train_ultimate_v3.py rename to scripts/training/train_ultimate_v3.py diff --git a/train_ultimate_v4.py b/scripts/training/train_ultimate_v4.py similarity index 100% rename from train_ultimate_v4.py rename to scripts/training/train_ultimate_v4.py diff --git a/train_ultimate_v5.py b/scripts/training/train_ultimate_v5.py similarity index 100% rename from train_ultimate_v5.py rename to scripts/training/train_ultimate_v5.py diff --git a/train_ultimate_v6.py b/scripts/training/train_ultimate_v6.py similarity index 100% rename from train_ultimate_v6.py rename to scripts/training/train_ultimate_v6.py diff --git a/tune_swin_unet.py b/scripts/training/tune_swin_unet.py similarity index 100% rename from tune_swin_unet.py rename to scripts/training/tune_swin_unet.py