{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "72723821",
   "metadata": {},
   "source": [
    "# Prepare Data on Server\n",
    "Chuẩn bị dữ liệu trên server: Tải từ S3, xử lý mây, tính NDVI, lưu thành file\n",
    "\n",
    "**Sau khi chạy xong, tải các file data xuống máy cá nhân để train model**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f502b085",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "%matplotlib inline\n",
    "\n",
    "import importlib\n",
    "import new_import_ODC  \n",
    "\n",
    "importlib.reload(new_import_ODC)\n",
    "\n",
    "from new_import_ODC import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "379dd847",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "# Cấu hình Daskgateway\n",
    "cluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1, 10))\n",
    "# Khai báo 1 Datacube là dc\n",
    "dc = datacube.Datacube()\n",
    "\n",
    "# Cấu hình truy cập dịch vụ S3\n",
    "configure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n",
    "\n",
    "client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da3e2180",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Cấu hình thời gian lấy ảnh và tọa độ\n",
    "date_range = (\"2022-09-01\", \"2023-10-01\")\n",
    "longtitude_range = (105.5, 106.4)\n",
    "latitude_range = (9.2, 10.0)\n",
    "\n",
    "coordinates = (longtitude_range, latitude_range)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3f521c79",
   "metadata": {},
   "outputs": [],
   "source": [
    "## DEBUG: Inspect what datacube wants to load\n",
    "print(\"🔍 DIAGNOSTIC: Checking datacube metadata...\\n\")\n",
    "\n",
    "# Check available products\n",
    "available_products = dc.list_products()\n",
    "s2_products = available_products[available_products['name'].str.contains('s2', case=False)]\n",
    "print(f\"Available S2 products:\\n{s2_products[['name', 'description']].to_string()}\\n\")\n",
    "\n",
    "# Query to check what would be loaded\n",
    "test_query = {\n",
    "    'product': 's2_l2a',\n",
    "    'x': longtitude_range,\n",
    "    'y': latitude_range,\n",
    "    'time': (\"2023-01-01\", \"2023-02-01\"),  # Just 1 month for testing\n",
    "}\n",
    "\n",
    "print(f\"Test query: {test_query}\")\n",
    "\n",
    "try:\n",
    "    # This queries metadata only, doesn't load data\n",
    "    test_datasets = dc.find_datasets(**test_query)\n",
    "    print(f\"\\n📊 Metadata check for Jan 2023:\")\n",
    "    print(f\"   Found {len(test_datasets)} scenes\")\n",
    "    if test_datasets:\n",
    "        first_ds = test_datasets[0]\n",
    "        print(f\"   First scene: {first_ds.center_time}\")\n",
    "        print(f\"   Bounds: {first_ds.bounds}\")\n",
    "        print(f\"   CRS: {first_ds.crs}\")\n",
    "except Exception as e:\n",
    "    print(f\"❌ Error: {e}\")\n",
    "\n",
    "print(\"\\n\" + \"=\"*60 + \"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8cf560b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "## SENTINEL-2 LOADING: Monthly chunks to prevent OOM\n",
    "print(\"📡 Tải dữ liệu Sentinel-2 L2A từ S3...\")\n",
    "print(f\"  AOI: {longtitude_range}, {latitude_range}\")\n",
    "print(f\"  Time range: {date_range}\\n\")\n",
    "\n",
    "data = None\n",
    "\n",
    "# Strategy: Load 13 monthly chunks instead of 396 scenes at once\n",
    "# This keeps memory usage manageable (~5-15 GB per month)\n",
    "\n",
    "date_ranges = [\n",
    "    (\"2022-09-01\", \"2022-10-01\"),\n",
    "    (\"2022-10-01\", \"2022-11-01\"),\n",
    "    (\"2022-11-01\", \"2022-12-01\"),\n",
    "    (\"2022-12-01\", \"2023-01-01\"),\n",
    "    (\"2023-01-01\", \"2023-02-01\"),\n",
    "    (\"2023-02-01\", \"2023-03-01\"),\n",
    "    (\"2023-03-01\", \"2023-04-01\"),\n",
    "    (\"2023-04-01\", \"2023-05-01\"),\n",
    "    (\"2023-05-01\", \"2023-06-01\"),\n",
    "    (\"2023-06-01\", \"2023-07-01\"),\n",
    "    (\"2023-07-01\", \"2023-08-01\"),\n",
    "    (\"2023-08-01\", \"2023-09-01\"),\n",
    "    (\"2023-09-01\", \"2023-10-01\"),\n",
    "]\n",
    "\n",
    "product = 's2_l2a'\n",
    "measurements = ['red', 'nir', 'scl']\n",
    "\n",
    "# Get native CRS once\n",
    "try:\n",
    "    query_crs = {\n",
    "        'product': product,\n",
    "        'x': longtitude_range,\n",
    "        'y': latitude_range,\n",
    "        'time': date_range,\n",
    "    }\n",
    "    native_crs = notebook_utils.mostcommon_crs(dc, query_crs)\n",
    "    print(f\"✅ Native CRS: {native_crs}\\n\")\n",
    "except Exception as e:\n",
    "    print(f\"⚠️  Could not determine CRS: {e}\")\n",
    "    native_crs = 'EPSG:32648'  # Fallback for UTM Zone 48N\n",
    "\n",
    "data_list = []\n",
    "\n",
    "for i, (start_date, end_date) in enumerate(date_ranges):\n",
    "    print(f\"[{i+1:2d}/13] {start_date} → {end_date}  \", end=\"\", flush=True)\n",
    "    \n",
    "    try:\n",
    "        monthly_query = {\n",
    "            'product': product,\n",
    "            'x': longtitude_range,\n",
    "            'y': latitude_range,\n",
    "            'time': (start_date, end_date),\n",
    "        }\n",
    "        \n",
    "        load_params = {\n",
    "            'measurements': measurements,\n",
    "            'output_crs': native_crs,\n",
    "            'resolution': (-10, 10),\n",
    "            'group_by': 'solar_day',\n",
    "            'dask_chunks': {'x': 512, 'y': 512, 'time': 1},\n",
    "            'skip_broken_datasets': True,\n",
    "        }\n",
    "        \n",
    "        monthly_data = load_s2l2a_with_offset(dc, monthly_query | load_params)\n",
    "        \n",
    "        n_scenes = monthly_data.sizes['time']\n",
    "        if n_scenes > 0:\n",
    "            data_list.append(monthly_data)\n",
    "            print(f\"✓ {n_scenes} scenes\")\n",
    "        else:\n",
    "            print(\"⚠️  0 scenes\")\n",
    "    \n",
    "    except MemoryError as e:\n",
    "        print(f\"❌ OOM: {str(e)[:60]}\")\n",
    "        break\n",
    "    except Exception as e:\n",
    "        print(f\"❌ {str(e)[:60]}\")\n",
    "        continue\n",
    "\n",
    "# Combine all monthly chunks\n",
    "if data_list:\n",
    "    print(f\"\\n🔗 Combining {len(data_list)} monthly chunks...\")\n",
    "    data = xr.concat(data_list, dim='time')\n",
    "    print(f\"✅ Success! Shape: {dict(data.dims)}\")\n",
    "    print(f\"   Memory: {notebook_utils.xarray_object_size(data)}\")\n",
    "    display(data)\n",
    "else:\n",
    "    print(\"\\n❌ Failed to load any scenes\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5dc550ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "# Loại bỏ các vị trí bị mây ảnh hưởng\n",
    "print(\"☁️  Xử lý mây...\")\n",
    "result = mask_clean(data)\n",
    "progress(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dd11fab1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tính toán NDVI\n",
    "print(\"🌱 Tính NDVI...\")\n",
    "ds1 = calculate_indices(result, index=\"NDVI\", satellite_mission=\"s2\")\n",
    "ndvi = ds1[\"NDVI\"]\n",
    "print(f\"✅ NDVI shape: {ndvi.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "45bb6665",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Điền mây sử dụng seasonal interpolation\n",
    "print(\"🔧 Điền mây theo mùa vụ...\")\n",
    "time_split = [\n",
    "    slice(\"2022-09-01\", \"2023-01-01\"),\n",
    "    slice(\"2023-01-01\", \"2023-05-01\"),\n",
    "    slice(\"2023-05-01\", \"2023-07-01\"),\n",
    "    slice(\"2023-07-01\", \"2023-10-01\"),\n",
    "]\n",
    "\n",
    "fill_nan_ndvi = fill_nan(ndvi, time_split)\n",
    "print(f\"✅ Mây đã được điền\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6892fb6b",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "# Tính NDVI theo tháng\n",
    "print(\"📊 Tính NDVI trung bình theo tháng...\")\n",
    "average_ndvi = fill_nan_ndvi.resample(time=\"1M\").mean().persist()\n",
    "progress(average_ndvi)\n",
    "average_ndvi = average_ndvi.compute()\n",
    "print(f\"✅ NDVI theo tháng shape: {average_ndvi.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cef198a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tải dữ liệu Sentinel-1 (VH, VV)\n",
    "print(\"📡 Tải dữ liệu Sentinel-1 từ S3...\")\n",
    "dsvh, dsvv = load_data_sen1(dc, date_range, coordinates)\n",
    "print(\"📊 Tính VV, VH trung bình theo tháng...\")\n",
    "average_vv = calculate_average(dsvv, time_pattern='1M')\n",
    "average_vh = calculate_average(dsvh, time_pattern='1M')\n",
    "print(f\"✅ VV shape: {average_vv.shape}\")\n",
    "print(f\"✅ VH shape: {average_vh.shape}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5ba52958",
   "metadata": {},
   "source": [
    "## Lưu dữ liệu đã xử lý thành file NetCDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "06bf3f40",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "# Tạo thư mục lưu data\n",
    "data_dir = \"data_for_training\"\n",
    "if not os.path.exists(data_dir):\n",
    "    os.makedirs(data_dir)\n",
    "    print(f\"✅ Tạo thư mục {data_dir}\")\n",
    "\n",
    "# Lưu NDVI\n",
    "ndvi_path = os.path.join(data_dir, \"average_ndvi.nc\")\n",
    "print(f\"💾 Lưu NDVI vào {ndvi_path}...\")\n",
    "average_ndvi.to_netcdf(ndvi_path)\n",
    "print(f\"✅ NDVI đã lưu ({os.path.getsize(ndvi_path) / 1024**2:.2f} MB)\")\n",
    "\n",
    "# Lưu VV\n",
    "vv_path = os.path.join(data_dir, \"average_vv.nc\")\n",
    "print(f\"💾 Lưu VV vào {vv_path}...\")\n",
    "average_vv.to_netcdf(vv_path)\n",
    "print(f\"✅ VV đã lưu ({os.path.getsize(vv_path) / 1024**2:.2f} MB)\")\n",
    "\n",
    "# Lưu VH\n",
    "vh_path = os.path.join(data_dir, \"average_vh.nc\")\n",
    "print(f\"💾 Lưu VH vào {vh_path}...\")\n",
    "average_vh.to_netcdf(vh_path)\n",
    "print(f\"✅ VH đã lưu ({os.path.getsize(vh_path) / 1024**2:.2f} MB)\")\n",
    "\n",
    "print(f\"\\n✅ Tất cả dữ liệu đã lưu trong thư mục '{data_dir}'\")\n",
    "print(f\"📥 Hãy tải các file này xuống máy cá nhân để train model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "93df03e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lưu training data\n",
    "train_path = \"train/ST_training data_updated_1130points_new.shp\"\n",
    "import shutil\n",
    "\n",
    "print(f\"📋 Copy training data...\")\n",
    "# Copy toàn bộ các file liên quan đến shapefile\n",
    "train_dir = \"train\"\n",
    "train_output = os.path.join(data_dir, \"train_data\")\n",
    "if not os.path.exists(train_output):\n",
    "    os.makedirs(train_output)\n",
    "\n",
    "for file in os.listdir(train_dir):\n",
    "    if \"1130points_new\" in file:\n",
    "        src = os.path.join(train_dir, file)\n",
    "        dst = os.path.join(train_output, file)\n",
    "        shutil.copy2(src, dst)\n",
    "        print(f\"✅ {file}\")\n",
    "\n",
    "print(f\"\\n✅ Training data đã copy vào '{train_output}'\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8bb935d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Đóng client, cluster\n",
    "print(\"\\n🔌 Đóng kết nối...\")\n",
    "client.close()\n",
    "cluster.close()\n",
    "print(\"✅ Xong!\")"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}