CSIROBoeingPhase5-Vietnam/01.prepare_data_on_server.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "72723821",
   "metadata": {},
   "source": [
    "# Prepare Data on Server - Load Raw Data Only\n",
    "Chuẩn bị dữ liệu trên server: **Chỉ tải dữ liệu thô từ S3, không xử lý**\n",
    "\n",
    "**Workflow:**\n",
    "1. Server: Tải S2 (red, nir, scl) + S1 (VH, VV) thô → Lưu NetCDF\n",
    "2. Local: Tải xuống → Tính NDVI → Cloud mask → Fill NaN → Aggregation → Train model\n",
    "\n",
    "**Sau khi chạy xong, tải các file data xuống máy cá nhân để xử lý + train model**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f502b085",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "%matplotlib inline\n",
    "\n",
    "import importlib\n",
    "import new_import_ODC  \n",
    "\n",
    "importlib.reload(new_import_ODC)\n",
    "\n",
    "from new_import_ODC import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "379dd847",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "# Cấu hình Daskgateway\n",
    "cluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1, 10))\n",
    "# Khai báo 1 Datacube là dc\n",
    "dc = datacube.Datacube()\n",
    "\n",
    "# Cấu hình truy cập dịch vụ S3\n",
    "configure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n",
    "\n",
    "client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da3e2180",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Cấu hình thời gian lấy ảnh và tọa độ\n",
    "date_range = (\"2022-09-01\", \"2023-10-01\")\n",
    "longtitude_range = (105.5, 106.4)\n",
    "latitude_range = (9.2, 10.0)\n",
    "\n",
    "coordinates = (longtitude_range, latitude_range)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3f521c79",
   "metadata": {},
   "outputs": [],
   "source": [
    "## DEBUG: Inspect what datacube wants to load\n",
    "print(\"🔍 DIAGNOSTIC: Checking datacube metadata...\\n\")\n",
    "\n",
    "# Check available products\n",
    "available_products = dc.list_products()\n",
    "s2_products = available_products[available_products['name'].str.contains('s2', case=False)]\n",
    "print(f\"Available S2 products:\\n{s2_products[['name', 'description']].to_string()}\\n\")\n",
    "\n",
    "# Query to check what would be loaded\n",
    "test_query = {\n",
    "    'product': 's2_l2a',\n",
    "    'x': longtitude_range,\n",
    "    'y': latitude_range,\n",
    "    'time': (\"2023-01-01\", \"2023-02-01\"),  # Just 1 month for testing\n",
    "}\n",
    "\n",
    "print(f\"Test query: {test_query}\")\n",
    "\n",
    "try:\n",
    "    # This queries metadata only, doesn't load data\n",
    "    test_datasets = dc.find_datasets(**test_query)\n",
    "    print(f\"\\n📊 Metadata check for Jan 2023:\")\n",
    "    print(f\"   Found {len(test_datasets)} scenes\")\n",
    "    if test_datasets:\n",
    "        first_ds = test_datasets[0]\n",
    "        print(f\"   First scene: {first_ds.center_time}\")\n",
    "        print(f\"   Bounds: {first_ds.bounds}\")\n",
    "        print(f\"   CRS: {first_ds.crs}\")\n",
    "except Exception as e:\n",
    "    print(f\"❌ Error: {e}\")\n",
    "\n",
    "print(\"\\n\" + \"=\"*60 + \"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8cf560b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "## SENTINEL-2 LOADING: Monthly chunks to prevent OOM\n",
    "print(\"📡 Tải dữ liệu Sentinel-2 L2A từ S3...\")\n",
    "print(f\"  AOI: {longtitude_range}, {latitude_range}\")\n",
    "print(f\"  Time range: {date_range}\\n\")\n",
    "\n",
    "data = None\n",
    "\n",
    "# Strategy: Load 13 monthly chunks instead of 396 scenes at once\n",
    "# This keeps memory usage manageable (~5-15 GB per month)\n",
    "\n",
    "date_ranges = [\n",
    "    (\"2022-09-01\", \"2022-10-01\"),\n",
    "    (\"2022-10-01\", \"2022-11-01\"),\n",
    "    (\"2022-11-01\", \"2022-12-01\"),\n",
    "    (\"2022-12-01\", \"2023-01-01\"),\n",
    "    (\"2023-01-01\", \"2023-02-01\"),\n",
    "    (\"2023-02-01\", \"2023-03-01\"),\n",
    "    (\"2023-03-01\", \"2023-04-01\"),\n",
    "    (\"2023-04-01\", \"2023-05-01\"),\n",
    "    (\"2023-05-01\", \"2023-06-01\"),\n",
    "    (\"2023-06-01\", \"2023-07-01\"),\n",
    "    (\"2023-07-01\", \"2023-08-01\"),\n",
    "    (\"2023-08-01\", \"2023-09-01\"),\n",
    "    (\"2023-09-01\", \"2023-10-01\"),\n",
    "]\n",
    "\n",
    "product = 's2_l2a'\n",
    "measurements = ['red', 'nir', 'scl']\n",
    "\n",
    "# Get native CRS once\n",
    "try:\n",
    "    query_crs = {\n",
    "        'product': product,\n",
    "        'x': longtitude_range,\n",
    "        'y': latitude_range,\n",
    "        'time': date_range,\n",
    "    }\n",
    "    native_crs = notebook_utils.mostcommon_crs(dc, query_crs)\n",
    "    print(f\"✅ Native CRS: {native_crs}\\n\")\n",
    "except Exception as e:\n",
    "    print(f\"⚠️  Could not determine CRS: {e}\")\n",
    "    native_crs = 'EPSG:32648'  # Fallback for UTM Zone 48N\n",
    "\n",
    "data_list = []\n",
    "\n",
    "for i, (start_date, end_date) in enumerate(date_ranges):\n",
    "    print(f\"[{i+1:2d}/13] {start_date} → {end_date}  \", end=\"\", flush=True)\n",
    "    \n",
    "    try:\n",
    "        monthly_query = {\n",
    "            'product': product,\n",
    "            'x': longtitude_range,\n",
    "            'y': latitude_range,\n",
    "            'time': (start_date, end_date),\n",
    "        }\n",
    "        \n",
    "        load_params = {\n",
    "            'measurements': measurements,\n",
    "            'output_crs': native_crs,\n",
    "            'resolution': (-10, 10),\n",
    "            'group_by': 'solar_day',\n",
    "            'dask_chunks': {'x': 512, 'y': 512, 'time': 1},\n",
    "            'skip_broken_datasets': True,\n",
    "        }\n",
    "        \n",
    "        monthly_data = load_s2l2a_with_offset(dc, monthly_query | load_params)\n",
    "        \n",
    "        n_scenes = monthly_data.sizes['time']\n",
    "        if n_scenes > 0:\n",
    "            data_list.append(monthly_data)\n",
    "            print(f\"✓ {n_scenes} scenes\")\n",
    "        else:\n",
    "            print(\"⚠️  0 scenes\")\n",
    "    \n",
    "    except MemoryError as e:\n",
    "        print(f\"❌ OOM: {str(e)[:60]}\")\n",
    "        break\n",
    "    except Exception as e:\n",
    "        print(f\"❌ {str(e)[:60]}\")\n",
    "        continue\n",
    "\n",
    "# Combine all monthly chunks\n",
    "if data_list:\n",
    "    print(f\"\\n🔗 Combining {len(data_list)} monthly chunks...\")\n",
    "    data = xr.concat(data_list, dim='time')\n",
    "    print(f\"✅ Success! Shape: {dict(data.dims)}\")\n",
    "    print(f\"   Memory: {notebook_utils.xarray_object_size(data)}\")\n",
    "    display(data)\n",
    "else:\n",
    "    print(\"\\n❌ Failed to load any scenes\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cef198a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Tải dữ liệu Sentinel-1 (VH, VV) - Raw data, không xử lý\n",
    "print(\"📡 Tải dữ liệu Sentinel-1 từ S3...\")\n",
    "try:\n",
    "    dsvh, dsvv = load_data_sen1(dc, date_range, coordinates)\n",
    "    print(f\"✅ VH shape: {dsvh.shape}\")\n",
    "    print(f\"✅ VV shape: {dsvv.shape}\")\n",
    "except Exception as e:\n",
    "    print(f\"⚠️  Error loading S1: {e}\")\n",
    "    dsvh = None\n",
    "    dsvv = None"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5ba52958",
   "metadata": {},
   "source": [
    "## Lưu dữ liệu RAW thành file NetCDF (chưa xử lý)\n",
    "Dữ liệu này sẽ được tải xuống máy local để tiếp tục xử lý và training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "06bf3f40",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "# Tạo thư mục lưu data\n",
    "data_dir = \"data_for_training\"\n",
    "if not os.path.exists(data_dir):\n",
    "    os.makedirs(data_dir)\n",
    "    print(f\"✅ Tạo thư mục {data_dir}\\n\")\n",
    "\n",
    "print(\"💾 Lưu dữ liệu RAW từ S2 (red, nir, scl)...\")\n",
    "\n",
    "# Lưu raw Sentinel-2\n",
    "s2_path = os.path.join(data_dir, \"sentinel2_raw.nc\")\n",
    "print(f\"   - Lưu vào {s2_path}...\")\n",
    "data.to_netcdf(s2_path)\n",
    "s2_size = os.path.getsize(s2_path) / 1024**3  # Convert to GB\n",
    "print(f\"   ✅ Sentinel-2 đã lưu ({s2_size:.2f} GB)\")\n",
    "\n",
    "# Lưu raw Sentinel-1 (nếu thành công)\n",
    "if dsvh is not None and dsvv is not None:\n",
    "    print(f\"\\n💾 Lưu dữ liệu RAW từ S1 (VH, VV)...\")\n",
    "    \n",
    "    # Tạo xarray Dataset chứa cả VH và VV\n",
    "    s1_data = xr.Dataset({\n",
    "        'VH': dsvh,\n",
    "        'VV': dsvv,\n",
    "    })\n",
    "    \n",
    "    s1_path = os.path.join(data_dir, \"sentinel1_raw.nc\")\n",
    "    print(f\"   - Lưu vào {s1_path}...\")\n",
    "    s1_data.to_netcdf(s1_path)\n",
    "    s1_size = os.path.getsize(s1_path) / 1024**3  # Convert to GB\n",
    "    print(f\"   ✅ Sentinel-1 đã lưu ({s1_size:.2f} GB)\")\n",
    "else:\n",
    "    print(\"⚠️  Sentinel-1 không được lưu (load thất bại)\")\n",
    "\n",
    "print(f\"\\n✅ Tất cả dữ liệu RAW đã lưu trong thư mục '{data_dir}'\")\n",
    "print(f\"📥 Hãy tải các file này xuống máy cá nhân\")\n",
    "print(f\"🔧 Sẽ tính NDVI, cloud mask, aggregation trên local\")\n",
    "\n",
    "# Hiển thị tổng kích thước\n",
    "import subprocess\n",
    "try:\n",
    "    result = subprocess.run(['du', '-sh', data_dir], capture_output=True, text=True)\n",
    "    print(f\"\\n📊 Tổng dung lượng: {result.stdout.strip()}\")\n",
    "except:\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "93df03e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lưu training data\n",
    "train_path = \"train/ST_training data_updated_1130points_new.shp\"\n",
    "import shutil\n",
    "\n",
    "print(f\"📋 Copy training data...\")\n",
    "# Copy toàn bộ các file liên quan đến shapefile\n",
    "train_dir = \"train\"\n",
    "train_output = os.path.join(data_dir, \"train_data\")\n",
    "if not os.path.exists(train_output):\n",
    "    os.makedirs(train_output)\n",
    "\n",
    "for file in os.listdir(train_dir):\n",
    "    if \"1130points_new\" in file:\n",
    "        src = os.path.join(train_dir, file)\n",
    "        dst = os.path.join(train_output, file)\n",
    "        shutil.copy2(src, dst)\n",
    "        print(f\"✅ {file}\")\n",
    "\n",
    "print(f\"\\n✅ Training data đã copy vào '{train_output}'\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8bb935d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Đóng client, cluster\n",
    "print(\"\\n🔌 Đóng kết nối...\")\n",
    "client.close()\n",
    "cluster.close()\n",
    "print(\"✅ Xong!\")"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}