{ "cells": [ { "cell_type": "code", "execution_count": 49, "id": "db6b50b8-f9c5-4ee9-b93f-ed38a89156ba", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 300 µs, sys: 172 µs, total: 472 µs\n", "Wall time: 484 µs\n" ] } ], "source": [ "%%time \n", "%matplotlib inline\n", "from new_import import *" ] }, { "cell_type": "code", "execution_count": 2, "id": "bfdefd7d-90d2-43bf-aa58-ebb4caef0f85", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "An existing cluster was found. Connecting to: easihub.26c21d10c6f44900aa4429853b81ac3f\n", "CPU times: user 762 ms, sys: 26.3 ms, total: 788 ms\n", "Wall time: 1 s\n" ] }, { "data": { "text/html": [ "
\n", "
\n", "
\n", "

Client

\n", "

Client-e689cc55-7343-11ef-82c8-eab95fd56520

\n", " \n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", "
Connection method: Cluster objectCluster type: dask_gateway.GatewayCluster
\n", " Dashboard: https://hub.asia.easi-eo.solutions/services/dask-gateway/clusters/easihub.26c21d10c6f44900aa4429853b81ac3f/status\n", "
\n", "\n", " \n", " \n", " \n", "\n", " \n", "
\n", "

Cluster Info

\n", "
\n", "

GatewayCluster

\n", " \n", "
\n", "\n", "
\n", " \n", "\n", "
\n", "
" ], "text/plain": [ "" ] }, "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ "%%time\n", "# Cấu hình Daskgateway\n", "cluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1,10))\n", "# Khai báo 1 Datacube là dc\n", "dc = datacube.Datacube()\n", "\n", "# Cấu hình truy cập dịch vụ S3\n", "configure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n", "\n", "client" ] }, { "cell_type": "markdown", "id": "657547be-5136-439b-b194-e5944c905138", "metadata": {}, "source": [ "LOAD VH, VV" ] }, { "cell_type": "code", "execution_count": 47, "id": "a0a8a450-3725-48d8-837e-2f19c2e6db59", "metadata": { "tags": [] }, "outputs": [], "source": [ "## cấu hình thời gian lấy ảnh và tọa độ\n", "date_range = ('2022-09-01', '2023-10-01')\n", "longtitude_range = (105.5, 106.4)\n", "latitude_range = (9.2, 10.0) " ] }, { "cell_type": "code", "execution_count": 3, "id": "05c2a58f-d0e5-4b20-9320-de05897cbdf7", "metadata": { "tags": [] }, "outputs": [], "source": [ "## cấu hình dữ liệu train và vh vv file\n", "train_path = \"train/ST_training data_updated_1130points.shp\" # đường dẫn shp file train\n", "name_vh = \"vh-0922_0923-full_ST.tif\"\n", "name_vv = \"vv-0922_0923-full_ST.tif\"\n", "\n", "\n", "train = load_train_data(train_path)" ] }, { "cell_type": "code", "execution_count": 4, "id": "33471ca1-6e5a-403e-b9af-2e7b45c1d892", "metadata": { "tags": [] }, "outputs": [], "source": [ "# %%time\n", "# ## tải về dữ liệu sen1\n", "# import os\n", "# if not os.path.exists(name_vh):\n", "# !aws s3 cp s3://easi-asia-dc-data/staging/ctu/sentinel-1/vh-0922_0923-full_ST.tif vh-0922_0923-full_ST.tif\n", "# if not os.path.exists(name_vv):\n", "# !aws s3 cp s3://easi-asia-dc-data/staging/ctu/sentinel-1/vv-0922_0923-full_ST.tif vv-0922_0923-full_ST.tif" ] }, { "cell_type": "code", "execution_count": 5, "id": "6ebfc48a-2bed-4909-b495-6bbfd3f3cc6f", "metadata": { "tags": [] }, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 38, "id": "8471dbbf-c3c6-4f03-94b0-11795b8d2062", "metadata": { "tags": [] }, "outputs": [ { "data": { "text/html": [ "

Dataset size: 21.60 GB

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "
<xarray.Dataset>\n",
       "Dimensions:      (time: 33, y: 8874, x: 9902)\n",
       "Coordinates:\n",
       "  * time         (time) datetime64[ns] 2022-09-06T22:46:14.500000 ... 2023-09...\n",
       "  * y            (y) float64 1.106e+06 1.106e+06 ... 1.017e+06 1.017e+06\n",
       "  * x            (x) float64 5.548e+05 5.548e+05 ... 6.538e+05 6.538e+05\n",
       "    spatial_ref  int32 32648\n",
       "Data variables:\n",
       "    vv           (time, y, x) float32 dask.array<chunksize=(1, 2048, 2048), meta=np.ndarray>\n",
       "    vh           (time, y, x) float32 dask.array<chunksize=(1, 2048, 2048), meta=np.ndarray>\n",
       "Attributes:\n",
       "    crs:           EPSG:32648\n",
       "    grid_mapping:  spatial_ref
" ], "text/plain": [ "\n", "Dimensions: (time: 33, y: 8874, x: 9902)\n", "Coordinates:\n", " * time (time) datetime64[ns] 2022-09-06T22:46:14.500000 ... 2023-09...\n", " * y (y) float64 1.106e+06 1.106e+06 ... 1.017e+06 1.017e+06\n", " * x (x) float64 5.548e+05 5.548e+05 ... 6.538e+05 6.538e+05\n", " spatial_ref int32 32648\n", "Data variables:\n", " vv (time, y, x) float32 dask.array\n", " vh (time, y, x) float32 dask.array\n", "Attributes:\n", " crs: EPSG:32648\n", " grid_mapping: spatial_ref" ] }, "execution_count": 38, "metadata": {}, "output_type": "execute_result" } ], "source": [ "ds = dc.load(\n", " product=\"sentinel1_grd_gamma0_20m\",\n", " x=(105.5, 106.4),\n", " y=(9.2, 10.0),\n", " time=(\"2022-09-01\", \"2023-10-01\"),\n", " measurements=[\"vv\", \"vh\"],\n", " output_crs=\"EPSG:32648\",\n", " resolution=(-10,10),\n", " dask_chunks={\"x\":2048, \"y\":2048},\n", " skip_broken_datasets=True,\n", " group_by=\"solar_day\"\n", ")\n", "notebook_utils.heading(notebook_utils.xarray_object_size(ds))\n", "ds" ] }, { "cell_type": "code", "execution_count": 43, "id": "3b51729a-3cdc-4ec5-957d-6861c0338dd8", "metadata": { "tags": [] }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "
<xarray.DataArray 'vv' (time: 33, y: 8874, x: 9902)>\n",
       "dask.array<dc_load_vv, shape=(33, 8874, 9902), dtype=float32, chunksize=(1, 2048, 2048), chunktype=numpy.ndarray>\n",
       "Coordinates:\n",
       "  * time         (time) datetime64[ns] 2022-09-06T22:46:14.500000 ... 2023-09...\n",
       "  * y            (y) float64 1.106e+06 1.106e+06 ... 1.017e+06 1.017e+06\n",
       "  * x            (x) float64 5.548e+05 5.548e+05 ... 6.538e+05 6.538e+05\n",
       "    spatial_ref  int32 32648\n",
       "Attributes:\n",
       "    units:         intensity\n",
       "    nodata:        nan\n",
       "    crs:           EPSG:32648\n",
       "    grid_mapping:  spatial_ref
" ], "text/plain": [ "\n", "dask.array\n", "Coordinates:\n", " * time (time) datetime64[ns] 2022-09-06T22:46:14.500000 ... 2023-09...\n", " * y (y) float64 1.106e+06 1.106e+06 ... 1.017e+06 1.017e+06\n", " * x (x) float64 5.548e+05 5.548e+05 ... 6.538e+05 6.538e+05\n", " spatial_ref int32 32648\n", "Attributes:\n", " units: intensity\n", " nodata: nan\n", " crs: EPSG:32648\n", " grid_mapping: spatial_ref" ] }, "execution_count": 43, "metadata": {}, "output_type": "execute_result" } ], "source": [ "vv_data = ds.vv\n", "vv_data\n" ] }, { "cell_type": "code", "execution_count": 44, "id": "8e5fd090-d355-45bd-ba1b-b95d0584dd8c", "metadata": { "tags": [] }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "
<xarray.DataArray (band: 13, y: 7680, x: 8687)>\n",
       "[867310080 values with dtype=float64]\n",
       "Coordinates:\n",
       "  * band         (band) int64 1 2 3 4 5 6 7 8 9 10 11 12 13\n",
       "  * x            (x) float64 5.597e+05 5.597e+05 ... 6.465e+05 6.465e+05\n",
       "  * y            (y) float64 1.099e+06 1.099e+06 ... 1.022e+06 1.022e+06\n",
       "    spatial_ref  int64 0\n",
       "Attributes:\n",
       "    AREA_OR_POINT:        Area\n",
       "    TIFFTAG_XRESOLUTION:  1\n",
       "    TIFFTAG_YRESOLUTION:  1\n",
       "    scale_factor:         1.0\n",
       "    add_offset:           0.0
" ], "text/plain": [ "\n", "[867310080 values with dtype=float64]\n", "Coordinates:\n", " * band (band) int64 1 2 3 4 5 6 7 8 9 10 11 12 13\n", " * x (x) float64 5.597e+05 5.597e+05 ... 6.465e+05 6.465e+05\n", " * y (y) float64 1.099e+06 1.099e+06 ... 1.022e+06 1.022e+06\n", " spatial_ref int64 0\n", "Attributes:\n", " AREA_OR_POINT: Area\n", " TIFFTAG_XRESOLUTION: 1\n", " TIFFTAG_YRESOLUTION: 1\n", " scale_factor: 1.0\n", " add_offset: 0.0" ] }, "execution_count": 44, "metadata": {}, "output_type": "execute_result" } ], "source": [ "dsvh, dsvv = load_sen1(name_vh, name_vv)\n", "dsvv" ] }, { "cell_type": "markdown", "id": "26f1d256-48cf-40a2-8de4-6ec7ea220bca", "metadata": {}, "source": [ "LOAD SENTINEL 2\n", "\n" ] }, { "cell_type": "code", "execution_count": 50, "id": "68e134d5-1b22-4b10-bdcb-d65d178e30d2", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Most common native CRS: EPSG:32648\n", "No datasets require offset correction\n", "The valid_data_mask and scale (no offset) have been applied to the reflectance bands\n" ] }, { "data": { "text/html": [ "

Dataset size: 210.07 GB

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "
<xarray.Dataset>\n",
       "Dimensions:      (time: 151, y: 8874, x: 9902)\n",
       "Coordinates:\n",
       "  * time         (time) datetime64[ns] 2022-09-02T03:35:23.960000 ... 2023-09...\n",
       "  * y            (y) float64 1.106e+06 1.106e+06 ... 1.017e+06 1.017e+06\n",
       "  * x            (x) float64 5.548e+05 5.548e+05 ... 6.538e+05 6.538e+05\n",
       "    spatial_ref  int32 32648\n",
       "Data variables:\n",
       "    blue         (time, y, x) float32 dask.array<chunksize=(1, 2048, 2048), meta=np.ndarray>\n",
       "    green        (time, y, x) float32 dask.array<chunksize=(1, 2048, 2048), meta=np.ndarray>\n",
       "    red          (time, y, x) float32 dask.array<chunksize=(1, 2048, 2048), meta=np.ndarray>\n",
       "    nir          (time, y, x) float32 dask.array<chunksize=(1, 2048, 2048), meta=np.ndarray>\n",
       "    scl          (time, y, x) uint8 dask.array<chunksize=(1, 2048, 2048), meta=np.ndarray>\n",
       "Attributes:\n",
       "    crs:           EPSG:32648\n",
       "    grid_mapping:  spatial_ref
" ], "text/plain": [ "\n", "Dimensions: (time: 151, y: 8874, x: 9902)\n", "Coordinates:\n", " * time (time) datetime64[ns] 2022-09-02T03:35:23.960000 ... 2023-09...\n", " * y (y) float64 1.106e+06 1.106e+06 ... 1.017e+06 1.017e+06\n", " * x (x) float64 5.548e+05 5.548e+05 ... 6.538e+05 6.538e+05\n", " spatial_ref int32 32648\n", "Data variables:\n", " blue (time, y, x) float32 dask.array\n", " green (time, y, x) float32 dask.array\n", " red (time, y, x) float32 dask.array\n", " nir (time, y, x) float32 dask.array\n", " scl (time, y, x) uint8 dask.array\n", "Attributes:\n", " crs: EPSG:32648\n", " grid_mapping: spatial_ref" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "data = load_data(dc, date_range, longtitude_range, latitude_range)\n", "notebook_utils.heading(notebook_utils.xarray_object_size(data))\n", "display(data)" ] }, { "cell_type": "code", "execution_count": 8, "id": "0c1ac39c-7a78-425d-afbc-b04a7988f012", "metadata": { "tags": [] }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
bitsvaluesdescription
qa[0, 1, 2, 3, 4, 5, 6, 7]{'0': 'no data', '1': 'saturated or defective'...Sen2Cor Scene Classification
\n", "
" ], "text/plain": [ " bits \\\n", "qa [0, 1, 2, 3, 4, 5, 6, 7] \n", "\n", " values \\\n", "qa {'0': 'no data', '1': 'saturated or defective'... \n", "\n", " description \n", "qa Sen2Cor Scene Classification " ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/plain": [ "{'0': 'no data',\n", " '1': 'saturated or defective',\n", " '2': 'dark area pixels',\n", " '3': 'cloud shadows',\n", " '4': 'vegetation',\n", " '5': 'bare soils',\n", " '6': 'water',\n", " '7': 'unclassified',\n", " '8': 'cloud medium probability',\n", " '9': 'cloud high probability',\n", " '10': 'thin cirrus',\n", " '11': 'snow or ice'}" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 2.06 s, sys: 84.8 ms, total: 2.14 s\n", "Wall time: 2.14 s\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "d9ad365b4b3d46ca9d0d596113c0a9ce", "version_major": 2, "version_minor": 0 }, "text/plain": [ "VBox()" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "%%time\n", "# Tiến hành loại bỏ các vị trí bị mây ảnh hưởng\n", "result = mask_clean(data)\n", "progress(result)" ] }, { "cell_type": "markdown", "id": "cd57a554-a804-4d0f-ac69-136e34d089db", "metadata": {}, "source": [ "CALCULATING THE MEAN VALUE AND FILL TO NAN POINT" ] }, { "cell_type": "code", "execution_count": 9, "id": "973ac9b2-14fd-45e4-934c-b201ac51ec9e", "metadata": { "tags": [] }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "6878e9bc108249bc9c0152ce6b16d463", "version_major": 2, "version_minor": 0 }, "text/plain": [ "VBox()" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "ds1 = calculate_indices(result, index='NDVI', satellite_mission='s2')\n", "ndvi = ds1[\"NDVI\"]\n", "average_ndvi = ndvi.resample(time='1M').mean().persist() ## tính mean cho từng tháng -> time = 12\n", "progress(average_ndvi)" ] }, { "cell_type": "code", "execution_count": 10, "id": "5af8393a-e667-4c93-ae32-ca69ee9d9da7", "metadata": { "tags": [] }, "outputs": [ { "data": { "text/plain": [ "(13, 7680, 8687)" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "dsvh.shape" ] }, { "cell_type": "code", "execution_count": 11, "id": "5b81db73-d533-483e-bab5-44d2c3eff64c", "metadata": { "tags": [] }, "outputs": [], "source": [ "average_ndvi = average_ndvi.compute()\n", "average_ndvi = average_ndvi[:, :dsvh.shape[1], :dsvh.shape[2]]" ] }, { "cell_type": "code", "execution_count": 12, "id": "504a37be-62cc-4121-b0bf-46639cb82fe6", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 3.5 s, sys: 1.2 s, total: 4.7 s\n", "Wall time: 4.69 s\n" ] } ], "source": [ "%%time\n", "filled_ds = average_ndvi.bfill(dim='time')\n", "filled_ds = filled_ds.ffill(dim='time')\n" ] }, { "cell_type": "markdown", "id": "2b6caa44-5793-4bac-ab67-d1e75d19aab3", "metadata": {}, "source": [ "FIND NAN POINT AFTER FILLING AND FILLING AGAIN WITH LINEARREGRESSION ALGORITHM" ] }, { "cell_type": "code", "execution_count": 13, "id": "0c47e46d-2c74-4b96-91b8-fcd5e46e7699", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Number of NaNs: 20800\n" ] } ], "source": [ "nan_mask = filled_ds.isnull()\n", "\n", "# Print the NaN mask\n", "# print(nan_mask)\n", "\n", "# Count the number of NaNs\n", "num_nans = nan_mask.sum()\n", "print(f'Number of NaNs: {num_nans.values}')" ] }, { "cell_type": "code", "execution_count": 14, "id": "ca51b1a5-2499-457b-bb99-65a950092270", "metadata": { "tags": [] }, "outputs": [], "source": [ "from sklearn.preprocessing import PolynomialFeatures\n", "from sklearn.linear_model import LinearRegression\n", "from sklearn.ensemble import RandomForestRegressor\n", "\n", "mask = ~np.isnan(filled_ds)\n", "X_train = np.stack([dsvh.values[mask], dsvv.values[mask]], axis=1)\n", "y_train = filled_ds.values[mask]" ] }, { "cell_type": "code", "execution_count": 15, "id": "c78559a3-a503-4dac-a21d-63b414f932a2", "metadata": { "tags": [] }, "outputs": [ { "data": { "text/html": [ "
LinearRegression()
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
" ], "text/plain": [ "LinearRegression()" ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "model = LinearRegression()\n", "model.fit(X_train, y_train)" ] }, { "cell_type": "code", "execution_count": 16, "id": "6a987dee-50b3-46d0-8fcd-d8fefe44ecd8", "metadata": { "tags": [] }, "outputs": [], "source": [ "X_pred = np.stack([dsvh.values[~mask], dsvv.values[~mask]], axis=1)\n", "filled_ds.values[~mask] = model.predict(X_pred)" ] }, { "cell_type": "markdown", "id": "146aa63d-217f-4c36-be36-e84a1ff3bf52", "metadata": {}, "source": [ "MATCH LABEL TO DATASET" ] }, { "cell_type": "code", "execution_count": 17, "id": "62024ea2-5b65-4aad-b1ea-66adb843f136", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 2.93 s, sys: 0 ns, total: 2.93 s\n", "Wall time: 2.93 s\n" ] } ], "source": [ "%%time\n", "\n", "# Takes 1 minute to complete.\n", "loaded_datasets = {}\n", "for idx, point in train.iterrows():\n", " key = f\"point_{idx + 1}\"\n", " try:\n", " ndvi_data = filled_ds.sel(x=point.geometry.x, y=point.geometry.y, method='nearest').values\n", " vh_data = dsvh.sel(x=point.geometry.x, y=point.geometry.y, method='nearest').values\n", " vv_data = dsvv.sel(x=point.geometry.x, y=point.geometry.y, method='nearest').values\n", " loaded_datasets[key] = {\n", " \"data\": np.concatenate((ndvi_data, vh_data, vv_data)),\n", " \"label\": point.HT_code\n", " }\n", " except Exception as e:\n", " # loaded_datasets[key] = None\n", " print(e)" ] }, { "cell_type": "code", "execution_count": 18, "id": "9d80765f-c25c-465e-8c99-582276477561", "metadata": { "tags": [] }, "outputs": [], "source": [ "label_mapping = {\n", " \"Lua tom\": \"0\",\n", " \"Lua\": \"1\",\n", " \"CHN\": \"2\",\n", " \"CLN\": \"3\",\n", " \"TS\": \"4\",\n", " \"Song\": \"5\",\n", " \"Dat xay dung\": \"6\",\n", " \"Rung\": \"7\"\n", "}\n", "label_encoder = LabelEncoder()\n", "\n", "# Fit and transform the labels\n", "labels = train.Hientrang.values\n", "numeric_labels = label_encoder.fit_transform([label_mapping[label] for label in labels])" ] }, { "cell_type": "code", "execution_count": 19, "id": "dc42088b-3aae-457f-bed8-206692121a37", "metadata": { "tags": [] }, "outputs": [], "source": [ "X = []\n", "x_new = []\n", "lb_new = []\n", "for k, v in loaded_datasets.items():\n", " X.append(v)\n", "for i in range(len(X)):\n", " if X[i] is not None:\n", " x_new.append(X[i][\"data\"])\n", " lb_new.append(numeric_labels[i])" ] }, { "cell_type": "markdown", "id": "6abe9a1b-55a6-48b0-a4ee-a1b2c7f209cf", "metadata": {}, "source": [ "BUILDING DATASETS " ] }, { "cell_type": "code", "execution_count": 20, "id": "e4e234b9-88e0-4f72-bf61-02cd6c6a6cfe", "metadata": { "tags": [] }, "outputs": [], "source": [ "X_train, X_temp, y_train, y_temp= train_test_split(x_new, lb_new, test_size=0.4, random_state=42)\n", "X_val, X_test, y_val, y_test = train_test_split(X_temp, y_temp, test_size=0.5, random_state=42)" ] }, { "cell_type": "markdown", "id": "717baa23-76b6-45e0-8e7b-42d7a79fe773", "metadata": {}, "source": [ "TRAIN MODEL" ] }, { "cell_type": "code", "execution_count": 21, "id": "131df37a-b1c0-4c16-9086-0ba0104fd803", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Best Parameters: {'classifier': RandomForestClassifier(n_jobs=-1, random_state=42), 'classifier__criterion': 'gini', 'classifier__max_depth': 15, 'classifier__n_estimators': 300}\n", "Accuracy: 86.0 %\n", "CPU times: user 8.96 s, sys: 4.88 s, total: 13.8 s\n", "Wall time: 26.8 s\n" ] } ], "source": [ "%%time\n", "from sklearn.pipeline import Pipeline\n", "from sklearn.preprocessing import StandardScaler\n", "from sklearn.model_selection import GridSearchCV\n", "from sklearn.ensemble import RandomForestClassifier\n", "from sklearn.neighbors import KNeighborsClassifier\n", "from sklearn.naive_bayes import GaussianNB\n", "from sklearn.svm import SVC\n", "from sklearn.metrics import accuracy_score\n", "\n", "# Define the models\n", "rf_model = RandomForestClassifier(random_state=42, n_jobs=-1)\n", "knn_model = KNeighborsClassifier()\n", "nb_model = GaussianNB()\n", "svm_model = SVC()\n", "\n", "# Create a pipeline\n", "pipeline = Pipeline([\n", " ('scaler', StandardScaler()), # Apply scaling\n", " ('classifier', rf_model) # Placeholder, will be set by param_grid\n", "])\n", "\n", "# Define the parameter grid for each classifier\n", "param_grid = [\n", " # RandomForest\n", " {\n", " 'classifier': [rf_model],\n", " 'classifier__n_estimators': [100, 300, 500, 700],\n", " 'classifier__max_depth': [6, 8, 10, 15],\n", " 'classifier__criterion': ['gini', 'entropy'],\n", " },\n", " # KNeighborsClassifier\n", " {\n", " 'classifier': [knn_model],\n", " 'classifier__n_neighbors': [3, 5, 7, 9],\n", " 'classifier__weights': ['uniform', 'distance'],\n", " 'classifier__metric': ['euclidean', 'manhattan']\n", " },\n", " # Naive Bayes (GaussianNB doesn't have hyperparameters to tune here)\n", " {\n", " 'classifier': [nb_model],\n", " },\n", " # SVM\n", " {\n", " 'classifier': [svm_model],\n", " 'classifier__C': [0.1, 1, 10, 100],\n", " 'classifier__kernel': ['linear', 'rbf'],\n", " 'classifier__gamma': ['scale', 'auto']\n", " }\n", "]\n", "\n", "# Use GridSearchCV to find the best classifier and hyperparameters\n", "grid_search = GridSearchCV(pipeline, param_grid, cv=5, scoring='accuracy', n_jobs=-1)\n", "grid_search.fit(X_train, y_train)\n", "\n", "# Print out the best parameters and classifier\n", "best_params = grid_search.best_params_\n", "print(\"Best Parameters:\", best_params)\n", "\n", "# Make predictions on the validation set\n", "y_pred = grid_search.predict(X_val)\n", "\n", "# Evaluate the results\n", "accuracy = accuracy_score(y_val, y_pred)\n", "print(f\"Accuracy: {round(accuracy, 2)*100} %\")\n" ] }, { "cell_type": "code", "execution_count": 22, "id": "bd2d4a3d-061d-4238-8fd9-ab4aaf062a2e", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Accuracy for test data 86.0 %\n" ] } ], "source": [ "## check accuracy score\n", "\n", "y_pred_test = grid_search.predict(X_test)\n", "test_accuracy = accuracy_score(y_test, y_pred_test)\n", "print(f\"Accuracy for test data {round(test_accuracy, 2)*100} %\")" ] }, { "cell_type": "code", "execution_count": 23, "id": "c5cd8314-c444-4bd4-a608-41e8a925ac51", "metadata": { "tags": [] }, "outputs": [ { "data": { "text/plain": [ "['model_train/model_new2.joblib']" ] }, "execution_count": 23, "metadata": {}, "output_type": "execute_result" } ], "source": [ "dir_save_model = \"model_train\"\n", "if not os.path.exists(dir_save_model):\n", " os.mkdir(dir_save_model)\n", "joblib.dump(grid_search, os.path.join(dir_save_model, \"model_new2.joblib\"))" ] }, { "cell_type": "code", "execution_count": 24, "id": "f11593ca-dcb4-4018-8142-2e933812878e", "metadata": { "tags": [] }, "outputs": [], "source": [ "client.close()\n", "cluster.close()" ] } ], "metadata": { "kernelspec": { "display_name": "env_01", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.18" } }, "nbformat": 4, "nbformat_minor": 5 }