From 1125a1e2dea249d3275d7fef4cd18438097a3326 Mon Sep 17 00:00:00 2001 From: nghiadang Date: Fri, 30 Aug 2024 00:52:15 +0000 Subject: [PATCH] add caculate accuracy --- 01.train.ipynb | 103 +++--------- 02.predict.ipynb | 192 +++++++++++++---------- ThuanHoa/DataTest/ThuanHoa_DKS_Kappa.cpg | 1 + ThuanHoa/DataTest/ThuanHoa_DKS_Kappa.dbf | Bin 0 -> 33287 bytes ThuanHoa/DataTest/ThuanHoa_DKS_Kappa.prj | 1 + ThuanHoa/DataTest/ThuanHoa_DKS_Kappa.qmd | 44 ++++++ ThuanHoa/DataTest/ThuanHoa_DKS_Kappa.shp | Bin 0 -> 10864 bytes ThuanHoa/DataTest/ThuanHoa_DKS_Kappa.shx | Bin 0 -> 2492 bytes new_import.py | 36 ++++- 9 files changed, 216 insertions(+), 161 deletions(-) create mode 100644 ThuanHoa/DataTest/ThuanHoa_DKS_Kappa.cpg create mode 100644 ThuanHoa/DataTest/ThuanHoa_DKS_Kappa.dbf create mode 100644 ThuanHoa/DataTest/ThuanHoa_DKS_Kappa.prj create mode 100644 ThuanHoa/DataTest/ThuanHoa_DKS_Kappa.qmd create mode 100644 ThuanHoa/DataTest/ThuanHoa_DKS_Kappa.shp create mode 100644 ThuanHoa/DataTest/ThuanHoa_DKS_Kappa.shx diff --git a/01.train.ipynb b/01.train.ipynb index 77a783f..a5f5da4 100644 --- a/01.train.ipynb +++ b/01.train.ipynb @@ -3279,6 +3279,17 @@ "progress(average_ndvi)" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "6be55b09-44d9-4957-a2d9-00f1c58d0aee", + "metadata": {}, + "outputs": [], + "source": [ + "# compute average_ndvi\n", + "average_ndvi = average_ndvi.compute()" + ] + }, { "cell_type": "code", "execution_count": 10, @@ -3394,73 +3405,19 @@ } ], "source": [ - "# compute average_ndvi\n", - "average_ndvi = average_ndvi.compute()\n", - "\n", - "## cấu hình dữ liệu train và vh vv file\n", - "train_path = \"train/ST_training data_updated_1130points.shp\" # đường dẫn shp file train\n", + "## cấu hình dữ liệu vh vv file\n", "name_vh = \"vh-0922_0923-full_ST.tif\"\n", "name_vv = \"vv-0922_0923-full_ST.tif\"\n", "\n", - "## load dữ liệu điểm train\n", - "train = load_train_data(train_path)\n", - "train.head()" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "b32cfb3f-e267-4995-853d-8e0e26f5501f", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 527 µs, sys: 111 µs, total: 638 µs\n", - "Wall time: 375 µs\n" - ] - } - ], - "source": [ "%%time\n", "## tải về dữ liệu sen1\n", - "import os\n", "if not os.path.exists(name_vh):\n", " !aws s3 cp s3://easi-asia-dc-data/staging/ctu/sentinel-1/vh-0922_0923-full_ST.tif vh-0922_0923-full_ST.tif\n", "if not os.path.exists(name_vv):\n", - " !aws s3 cp s3://easi-asia-dc-data/staging/ctu/sentinel-1/vv-0922_0923-full_ST.tif vv-0922_0923-full_ST.tif" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "9838697a-2260-45ef-a810-533428a783e9", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Warning 1: TIFFReadDirectory:Invalid data type for tag StripByteCounts\n", - "Warning 1: TIFFFetchNormalTag:ASCII value for tag \"GeoASCIIParams\" contains null byte in value; value incorrectly truncated during reading due to implementation limitations\n", - "Warning 1: TIFFReadDirectory:Sum of Photometric type-related color channels and ExtraSamples doesn't match SamplesPerPixel. Defining non-color channels as ExtraSamples.\n", - "Warning 1: TIFFReadDirectory:Invalid data type for tag StripByteCounts\n", - "Warning 1: TIFFFetchNormalTag:ASCII value for tag \"GeoASCIIParams\" contains null byte in value; value incorrectly truncated during reading due to implementation limitations\n", - "Warning 1: TIFFReadDirectory:Sum of Photometric type-related color channels and ExtraSamples doesn't match SamplesPerPixel. Defining non-color channels as ExtraSamples.\n" - ] - } - ], - "source": [ + " !aws s3 cp s3://easi-asia-dc-data/staging/ctu/sentinel-1/vv-0922_0923-full_ST.tif vv-0922_0923-full_ST.tif\n", + " \n", "# load dữ liệu sen1\n", - "dsvh, dsvv = load_sen1(name_vh, name_vv)\n", - "\n", - "# xây dựng tập dataset\n", - "datasets = get_data_sen1_and_sen2(train, average_ndvi, dsvh, dsvv)" + "dsvh, dsvv = load_sen1(name_vh, name_vv)" ] }, { @@ -3472,6 +3429,11 @@ }, "outputs": [], "source": [ + "## load dữ liệu điểm train\n", + "train_path = \"train/ST_training data_updated_1130points.shp\" # đường dẫn shp file train\n", + "train = load_train_data(train_path)\n", + "train.head()\n", + "\n", "# cấu hình nhãn dữ liệu\n", "label_mapping = {\n", " \"Lua tom\": \"0\",\n", @@ -3484,6 +3446,9 @@ " \"Rung\": \"7\"\n", "}\n", "\n", + "# xây dựng tập dataset\n", + "datasets = get_data_sen1_and_sen2(train, average_ndvi, dsvh, dsvv)\n", + "\n", "# chia tập dữ liệu train, val, test\n", "X_train, X_val, X_test, y_train, y_val, y_test = split_train_data(train, label_mapping, datasets)" ] @@ -3530,26 +3495,8 @@ "# kiểm tra độ chính xác với tập test\n", "y_pred_test = grid_search.predict(X_test)\n", "test_accuracy = accuracy_score(y_test, y_pred_test)\n", - "print(f\"Accuracy for test data {round(test_accuracy, 2)*100} %\")" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "id": "f1a14379-ed6e-4897-9ca4-2669743fab40", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Done!\n" - ] - } - ], - "source": [ + "print(f\"Accuracy for test data {round(test_accuracy, 2)*100} %\")\n", + "\n", "# Lưu mô hình huấn luyện\n", "save_model(\"model.joblib\", grid_search)" ] diff --git a/02.predict.ipynb b/02.predict.ipynb index cb713f8..dcdc429 100644 --- a/02.predict.ipynb +++ b/02.predict.ipynb @@ -1122,8 +1122,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 13.5 s, sys: 3.97 s, total: 17.5 s\n", - "Wall time: 11 s\n" + "CPU times: user 13.7 s, sys: 4 s, total: 17.7 s\n", + "Wall time: 11.3 s\n" ] } ], @@ -1143,9 +1143,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "Starting new cluster.\n", - "CPU times: user 798 ms, sys: 20.3 ms, total: 818 ms\n", - "Wall time: 3min 46s\n" + "An existing cluster was found. Connecting to: easihub.f1152aa37dff4a11b444ed47e71e9a25\n", + "CPU times: user 749 ms, sys: 46.1 ms, total: 795 ms\n", + "Wall time: 979 ms\n" ] }, { @@ -1155,7 +1155,7 @@ "
\n", "
\n", "

Client

\n", - "

Client-9b1d4f60-6441-11ef-8121-ca4e92f6ea90

\n", + "

Client-44b90909-6668-11ef-825d-56b98e053c49

\n", " \n", "\n", " \n", @@ -1168,7 +1168,7 @@ " \n", " \n", " \n", " \n", " \n", @@ -1177,7 +1177,7 @@ "
\n", - " Dashboard: https://hub.asia.easi-eo.solutions/services/dask-gateway/clusters/easihub.c5bae2a11d48444fae17a2cd2db4b287/status\n", + " Dashboard: https://hub.asia.easi-eo.solutions/services/dask-gateway/clusters/easihub.f1152aa37dff4a11b444ed47e71e9a25/status\n", "
\n", "\n", " \n", - " \n", " \n", @@ -1188,8 +1188,8 @@ "
\n", "

GatewayCluster

\n", " \n", "
\n", "\n", @@ -1200,7 +1200,7 @@ "
" ], "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -1645,7 +1645,7 @@ " scl (time, y, x) uint8 dask.array<chunksize=(1, 532, 830), meta=np.ndarray>\n", "Attributes:\n", " crs: EPSG:32648\n", - " grid_mapping: spatial_ref