basketballcantho/remote-sensing
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

thêm chức năng train trên odc predict trên planetary

Browse Source
This commit is contained in:
Victor Phan
2026-03-04 23:03:04 +07:00
parent ebb8e6e4b3
commit 8a1e7bb22e
39 changed files with 15297 additions and 711 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+2
View File
@@ -84,3 +84,5 @@ cloud_removal_model/
# Jupyter checkpoints # Jupyter checkpoints
.ipynb_checkpoints/ .ipynb_checkpoints/
reports/ reports/
easi_tools/
notebooks/
01.train_ODC.py
-634
View File
@@ -1,634 +0,0 @@
"""
01.train_ODC.py
Chuyển đổi từ 01.train_ODC.ipynb
Land Use Classification Training - Sentinel-2 / ODC
"""
import os
import sys
import importlib
import traceback
import numpy as np
import xarray as xr
from datetime import datetime
import new_import_ODC
from new_import_ODC import * # load_train_data, save_model, notebook_utils, ...
# ============================================================
# HYPERPARAMETERS - Chỉnh sửa tại đây
# ============================================================
# --- Dask cluster ---
DASK_N_WORKERS = 4
# --- Dữ liệu Sentinel-2 ---
DATE_RANGE = ("2023-03-01", "2023-12-31")
LONGITUDE_RANGE = (105.5, 106.4)
LATITUDE_RANGE = (9.2, 10.0)
NUM_SCENES = 1 # Số scene cần load (None = tất cả)
# --- Cache ---
CACHE_DIR = "dataset_cache"
CACHE_FILE = f"{CACHE_DIR}/sentinel2_timeseries_40scenes.nc"
# --- Training data ---
TRAIN_PATH = "train/ST_training_data_updated_1130points_new.shp"
# --- Features sử dụng để train ---
AVAILABLE_FEATURES = [
'ndvi_mean', 'ndvi_min', 'ndvi_max', 'ndvi_std', 'ndvi_range',
'ndwi_mean', 'ndbi_mean', 'evi_mean',
]
# --- Train / Test split ---
TEST_SIZE = 0.2
RANDOM_STATE = 42
# --- Random Forest hyperparameters ---
RF_N_ESTIMATORS = 200
RF_MAX_DEPTH = 30
RF_MIN_SAMPLES_SPLIT = 5
RF_N_JOBS = -1
RF_VERBOSE = 1
# --- Output model ---
MODEL_FILENAME = "model_land_use_odc.joblib"
# --- Label mapping ---
LABEL_MAPPING = {
"Lua tom": "0",
"Lua": "1",
"CHN": "2",
"CLN": "3",
"TS": "4",
"Song": "5",
"Dat xay dung": "6",
"Rung": "7",
}
# ============================================================
# END HYPERPARAMETERS
# ============================================================
def setup_imports():
"""Load custom modules."""
import new_import_ODC
importlib.reload(new_import_ODC)
print("✅ All modules loaded successfully")
return new_import_ODC
def setup_dask_and_datacube():
"""Khởi động Dask cluster và kết nối Datacube."""
from dask.distributed import Client, LocalCluster
import datacube
print("✅ AWS credentials loaded from environment variables")
cluster = LocalCluster(n_workers=DASK_N_WORKERS)
client = Client(cluster)
print("✅ Dask cluster initialized")
print(f" Cluster: {cluster}")
# Cấu hình S3 access cho rasterio/GDAL (bắt buộc để đọc COGs từ S3)
configure_s3_access(aws_unsigned=False, requester_pays=True, client=client)
print("✅ S3 access configured (requester_pays=True)")
try:
dc = datacube.Datacube()
print("✅ Datacube connected (metadata only)")
except Exception as e:
print(f"⚠️ Datacube connection not critical: {e}")
dc = None
print("\n" + "=" * 70)
return client, cluster, dc
def get_scene_metadata(dc):
"""Lấy metadata Sentinel-2 từ datacube."""
print("=" * 70)
print("GETTING SENTINEL-2 SCENE METADATA")
print("=" * 70)
try:
print(f"\n[1] Loading metadata from datacube...")
datasets = list(dc.find_datasets(product='s2_l2a', time=DATE_RANGE))
print(f" ✅ Found {len(datasets)} scenes")
if datasets:
selected = datasets[0]
print(f"\n[2] Selected scene: {selected.metadata.label}")
scene_datetime = selected.time.begin if hasattr(selected.time, 'begin') else selected.time
print(f" Date: {scene_datetime}")
print(f"\n[3] Available bands:")
for name, measurement in selected.measurements.items():
print(f" - {name}: {measurement['path'][:80]}")
except Exception as e:
print(f"❌ Error: {e}")
traceback.print_exc()
print("=" * 70)
def check_cache():
"""Kiểm tra dataset cache. Trả về (data, use_cache)."""
print("=" * 70)
print("CHECKING FOR CACHED DATASET")
print("=" * 70)
use_cache = False
data = None
if os.path.exists(CACHE_FILE):
print(f"\n✅ Cache file found: {CACHE_FILE}")
file_size_gb = os.path.getsize(CACHE_FILE) / (1024 ** 3)
print(f" File size: {file_size_gb:.2f} GB")
try:
print(f"\n🔄 Loading dataset from cache...")
data = xr.open_dataset(CACHE_FILE)
print(f"✅ Dataset loaded from cache!")
print(f" Total scenes: {len(data['time'])}")
print(f" Variables: {len(data.data_vars)}")
print(f" Dimensions: {dict(data.dims)}")
print(f"\n ⏭️ Skipping S3 download (using cached data)")
use_cache = True
except Exception as e:
print(f"❌ Error loading cache: {e}")
print(f" Will download fresh data from S3")
else:
print(f"\n⏳ Cache file not found: {CACHE_FILE}")
print(f" Will download from S3 and save cache")
print(f" (Next run will use cache automatically)")
print("=" * 70)
return data, use_cache
def load_satellite_data(dc, data, use_cache):
"""Download / load Sentinel-2 data và tính spectral indices."""
import rasterio
from scipy import ndimage
print("=" * 70)
print("LOADING SENTINEL-2 DATA FROM S3 COGs (RASTERIO) - OPTIMAL ACCURACY")
print("=" * 70)
ndvi = None
try:
if use_cache and data is not None:
print(f"\n✅ Using cached dataset - skipping download!")
print(f" Variables: {len(data.data_vars)}")
print(f" Shape: {data.dims}")
else:
# --- Download from S3 ---
print(f"\n📥 Downloading from S3...")
datasets = list(dc.find_datasets(product='s2_l2a', time=DATE_RANGE))
if not datasets:
raise ValueError("No datasets found for date range")
print(f"\n📦 Found {len(datasets)} available scenes")
print(f" Date range: {DATE_RANGE[0]} to {DATE_RANGE[1]}")
num_scenes = NUM_SCENES if NUM_SCENES is not None else len(datasets)
print(f"\n[LOADING] Loading {num_scenes} scenes with ALL available bands...")
print(f" (Keeping NATIVE resolution - NO upsampling/magnification)")
all_data_dict = {}
failed_scenes = []
scene_dates = []
first_scene = datasets[0]
all_available_bands = list(first_scene.measurements.keys())
print(f" Available bands: {all_available_bands}")
for scene_idx in range(num_scenes):
selected = datasets[scene_idx]
scene_label = selected.metadata.label
scene_datetime = selected.time.begin if hasattr(selected.time, 'begin') else selected.time
scene_dates.append(scene_datetime)
if scene_idx % 5 == 0 or scene_idx == 0 or scene_idx == num_scenes - 1:
print(f"\n [{scene_idx + 1:2d}/{num_scenes}] {scene_label} ({scene_datetime.date()})")
scene_data_dict = {}
for band_name in all_available_bands:
if band_name in selected.measurements:
band_path = selected.measurements[band_name]['path']
try:
with rasterio.open(band_path) as src:
scene_data_dict[band_name] = src.read(1)
except Exception as e:
if scene_idx % 5 == 0:
print(f" ⚠️ Error loading {band_name}: {str(e)[:30]}")
failed_scenes.append((scene_idx, scene_label, band_name, str(e)))
if scene_data_dict:
all_data_dict[scene_idx] = scene_data_dict
if scene_idx % 5 == 0 or scene_idx == num_scenes - 1:
print(f" ✅ {len(scene_data_dict)} bands loaded")
else:
failed_scenes.append((scene_idx, scene_label, "all", "No bands loaded"))
if not all_data_dict:
raise ValueError("Could not load any bands from any scene")
print(f"\n✅ Successfully loaded {len(all_data_dict)} scenes!")
if failed_scenes:
print(f"⚠️ Failed to load {len(failed_scenes)} band instances (will be skipped)")
# --- Resolution normalization ---
print(f"\n[RESOLUTION NORMALIZATION] Aligning all bands to native resolution...")
ref_resolution = max_size = 0
max_band = None
for s_idx in all_data_dict:
for b_name, b_data in all_data_dict[s_idx].items():
sz = b_data.shape[0]
if sz > max_size:
max_size = sz
ref_resolution = sz
max_band = b_name
print(f" Reference resolution: {max_size}×{max_size} pixels (native {max_band})")
resampled_count = 0
for s_idx in all_data_dict:
for b_name in list(all_data_dict[s_idx]):
arr = all_data_dict[s_idx][b_name]
if arr.shape[0] != ref_resolution:
scale = ref_resolution / arr.shape[0]
order = 0 if b_name == 'scl' else 1
resampled = ndimage.zoom(arr, scale, order=order)
all_data_dict[s_idx][b_name] = resampled
if s_idx == 0:
print(f" Resampling {b_name}: {arr.shape[0]}×{arr.shape[0]} "
f"→ {resampled.shape[0]}×{resampled.shape[0]}")
resampled_count += 1
print(f"✅ Resolution normalization complete! ({resampled_count} bands resampled)")
# --- Spectral indices ---
print(f"\n[SPECTRAL INDICES] Calculating spectral indices for each scene...")
indices_count = 0
for s_idx in all_data_dict:
sd = all_data_dict[s_idx]
try:
if 'nir' in sd and 'red' in sd:
nir, red = sd['nir'].astype(float), sd['red'].astype(float)
sd['ndvi'] = ((nir - red) / (nir + red + 1e-8)).astype(np.float32)
indices_count += 1
if 'b11' in sd and 'nir' in sd:
swir, nir = sd['b11'].astype(float), sd['nir'].astype(float)
sd['ndbi'] = ((swir - nir) / (swir + nir + 1e-8)).astype(np.float32)
indices_count += 1
if 'nir' in sd and 'b11' in sd:
nir, swir = sd['nir'].astype(float), sd['b11'].astype(float)
sd['ndwi'] = ((nir - swir) / (nir + swir + 1e-8)).astype(np.float32)
indices_count += 1
if 'nir' in sd and 'red' in sd and 'blue' in sd:
nir = sd['nir'].astype(float)
red = sd['red'].astype(float)
blue = sd['blue'].astype(float)
sd['evi'] = (2.5 * (nir - red) / (nir + 6 * red - 7.5 * blue + 1)).astype(np.float32)
indices_count += 1
except Exception:
pass
print(f"✅ Calculated {indices_count} spectral indices per scene")
# --- Stack along time ---
print(f"\n[STACKING] Stacking {len(all_data_dict)} scenes to time-series...")
data_vars = {}
band_names = list(all_data_dict[0].keys())
for b_name in band_names:
arr_list = [
all_data_dict[si][b_name]
for si in sorted(all_data_dict)
if b_name in all_data_dict[si]
]
if arr_list:
data_vars[b_name] = (['time', 'y', 'x'], np.stack(arr_list, axis=0))
first_arr = list(all_data_dict[0].values())[0]
y_size, x_size = first_arr.shape
data = xr.Dataset(
data_vars,
coords={
'time': np.arange(len(all_data_dict)),
'x': np.arange(x_size),
'y': np.arange(y_size),
}
)
# --- Temporal features ---
print(f"\n[TEMPORAL FEATURES] Computing temporal features...")
tf_added = 0
if 'ndvi' in data.data_vars:
ts = data['ndvi']
data['ndvi_min'] = ts.min(dim='time'); tf_added += 1
data['ndvi_max'] = ts.max(dim='time'); tf_added += 1
data['ndvi_mean'] = ts.mean(dim='time'); tf_added += 1
data['ndvi_range'] = data['ndvi_max'] - data['ndvi_min']; tf_added += 1
data['ndvi_std'] = ts.std(dim='time'); tf_added += 1
for b_name in ['ndbi', 'ndwi', 'evi']:
if b_name in data.data_vars:
data[f'{b_name}_mean'] = data[b_name].mean(dim='time')
tf_added += 1
print(f"✅ Added {tf_added} temporal/aggregate features")
# --- Save cache ---
print(f"\n[CACHE] Saving dataset to cache...")
try:
os.makedirs(CACHE_DIR, exist_ok=True)
data.to_netcdf(CACHE_FILE, engine='netcdf4')
cache_size = os.path.getsize(CACHE_FILE) / (1024 ** 3)
print(f"✅ Dataset saved to cache: {CACHE_FILE}")
print(f" Cache size: {cache_size:.2f} GB")
except Exception as e:
print(f"⚠️ Error saving cache: {e}")
print(f"\n✅ Dataset created!")
print(f" 🎬 Scenes: {len(all_data_dict)}")
print(f" 📊 Variables: {len(data.data_vars)}")
print(f" 🖼️ Size: {x_size} × {y_size} px")
print(f" ⏰ {scene_dates[0].date()} → {scene_dates[-1].date()}")
# --- Extract NDVI ---
print(f"\n[NDVI EXTRACTION] Extracting NDVI for model training...")
if 'ndvi_mean' in data.data_vars:
ndvi = data['ndvi_mean']
print(f"✅ NDVI extracted (mean across time) - shape: {ndvi.shape}")
elif 'ndvi' in data.data_vars:
ndvi = data['ndvi'].isel(time=0)
print(f"✅ NDVI extracted (first time step) - shape: {ndvi.shape}")
else:
print(f"❌ NDVI not found in dataset")
except Exception as e:
print(f"❌ Error: {e}")
traceback.print_exc()
data = None
print("=" * 70)
return data, ndvi
def load_training_data():
"""Load training shapefile và trả về GeoDataFrame."""
print("=" * 70)
print("TRAINING DATA SETUP")
print("=" * 70)
print(f"\n[1] Loading training data: {TRAIN_PATH}")
try:
train = load_train_data(TRAIN_PATH)
print(f" ✅ Loaded {len(train)} training points")
print(f" Columns: {list(train.columns)}")
except Exception as e:
print(f" ❌ Error: {e}")
train = None
print(f"\n[2] Label mapping:")
for label, code in LABEL_MAPPING.items():
print(f" {code}: {label}")
print("\n" + "=" * 70)
return train
def train_model(train, data):
"""Extract features, split, train RandomForest, evaluate."""
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score, classification_report
print("=" * 70)
print("LAND USE CLASSIFICATION TRAINING")
print("=" * 70)
model = X_train = X_test = y_train = y_test = y_pred = accuracy = None
features_to_use = class_names = []
if train is None or data is None:
print("❌ Missing training data or satellite data")
return model, None, None, None, None, None, None, features_to_use, class_names
print("\n[1] Extracting features from satellite data...")
try:
features_to_use = [f for f in AVAILABLE_FEATURES if f in data.data_vars]
if not features_to_use:
print(" ❌ No spectral features found in dataset!")
print(" Available variables:", list(data.data_vars))
return model, None, None, None, None, None, None, features_to_use, class_names
print(f" Using {len(features_to_use)} features: {features_to_use}")
X, y = [], []
for _, point in train.iterrows():
try:
feature_vec = [
float(data[fn].sel(x=point.geometry.x, y=point.geometry.y, method='nearest').values)
for fn in features_to_use
]
label = LABEL_MAPPING[point.Hientrang]
if not np.isnan(feature_vec).any():
X.append(feature_vec)
y.append(int(label))
except Exception:
continue
if not X:
print(" ❌ No samples extracted")
return model, None, None, None, None, None, None, features_to_use, class_names
X = np.array(X)
y = np.array(y)
print(f" ✅ {len(X)} samples × {X.shape[1]} features")
print(f"\n Feature statistics:")
for i, fn in enumerate(features_to_use):
print(f" {fn:15s}: mean={X[:, i].mean():.3f}, std={X[:, i].std():.3f}")
# Split
print(f"\n[2] Splitting data ({int((1-TEST_SIZE)*100)}-{int(TEST_SIZE*100)})...")
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=TEST_SIZE, random_state=RANDOM_STATE, stratify=y
)
print(f" Train: {len(X_train)}, Test: {len(X_test)}")
unique, counts = np.unique(y_train, return_counts=True)
print(f"\n Class distribution in training set:")
for cls, count in zip(unique, counts):
cls_name = [k for k, v in LABEL_MAPPING.items() if v == str(cls)][0]
print(f" {cls}: {cls_name:15s} - {count:4d} ({count/len(y_train)*100:.1f}%)")
# Train
print(f"\n[3] Training Random Forest...")
model = RandomForestClassifier(
n_estimators=RF_N_ESTIMATORS,
max_depth=RF_MAX_DEPTH,
min_samples_split=RF_MIN_SAMPLES_SPLIT,
random_state=RANDOM_STATE,
n_jobs=RF_N_JOBS,
verbose=RF_VERBOSE,
)
model.fit(X_train, y_train)
y_pred = model.predict(X_test)
accuracy = accuracy_score(y_test, y_pred)
print(f"\n ✅ Training accuracy: {model.score(X_train, y_train)*100:.2f}%")
print(f" ✅ Testing accuracy: {accuracy*100:.2f}%")
# Feature importance
print(f"\n Feature importance:")
importances = model.feature_importances_
for rank, idx in enumerate(np.argsort(importances)[::-1]):
print(f" {rank+1}. {features_to_use[idx]:15s}: {importances[idx]:.4f}")
# Classification report
class_names = [k for k, v in sorted(LABEL_MAPPING.items(), key=lambda x: x[1])]
print(f"\n[4] Classification Report:")
print(classification_report(y_test, y_pred, target_names=class_names, zero_division=0))
except Exception as e:
print(f" ❌ Error: {e}")
traceback.print_exc()
model = None
print("\n" + "=" * 70)
return model, X_train, X_test, y_train, y_test, y_pred, accuracy, features_to_use, class_names
def save_trained_model(model, X_train, X_test, y_train, y_test,
y_pred, accuracy, features_to_use, class_names):
"""Lưu model + metadata bằng ModelManager."""
from sklearn.metrics import classification_report
print("=" * 70)
print("MODEL SAVING")
print("=" * 70)
if model is None:
print("❌ No model to save")
return
print("\n🔄 Saving model with metadata...")
try:
y = np.concatenate([y_train, y_test])
metadata = {
"timestamp": datetime.now().isoformat(),
"data_source": "Local S3 ODC (Open Data Cube)",
"collections": ["sentinel-2-l2a"],
"features": features_to_use,
"feature_mode": "extended",
"training_samples": len(X_train),
"testing_samples": len(X_test),
"test_size": TEST_SIZE,
"train_accuracy": float(model.score(X_train, y_train)),
"test_accuracy": float(accuracy),
"model_type": "random_forest",
"device": "cpu",
"n_estimators": RF_N_ESTIMATORS,
"max_depth": RF_MAX_DEPTH,
"min_samples_split": RF_MIN_SAMPLES_SPLIT,
"learning_rate": None,
"cnn_epochs": None,
"n_features": X_train.shape[1],
"n_classes": len(np.unique(y)),
"class_names": list(LABEL_MAPPING.keys()),
"classification_report": classification_report(
y_test, y_pred, target_names=class_names,
output_dict=True, zero_division=0
),
"bbox": None,
"time_range": f"{DATE_RANGE[0]}/{DATE_RANGE[1]}",
"resolution": 10,
"notes": "Land Use Classification (8 classes) trained from 01.train_ODC.py",
}
save_model(MODEL_FILENAME, model, metadata=metadata, label_encoder=None)
print(f"✅ Model saved → model_train/{MODEL_FILENAME}")
print(f" Train Accuracy: {metadata['train_accuracy']*100:.2f}%")
print(f" Test Accuracy: {metadata['test_accuracy']*100:.2f}%")
print(f" Classes: {metadata['n_classes']}")
except Exception as e:
print(f"❌ Error saving model: {e}")
traceback.print_exc()
print("=" * 70)
def cleanup(client, cluster):
"""Đóng Dask client/cluster."""
print("=" * 70)
print("CLEANUP")
print("=" * 70)
try:
client.close()
cluster.close()
print("✅ Cleanup complete")
except Exception as e:
print(f"⚠️ Error during cleanup: {e}")
print("\n" + "=" * 70)
print("✅ PIPELINE COMPLETE")
print("=" * 70)
# ============================================================
# MAIN
# ============================================================
if __name__ == "__main__":
# 1. Reload modules
setup_imports()
# 2. Dask + Datacube
client, cluster, dc = setup_dask_and_datacube()
# 3. Scene metadata
if dc is not None:
get_scene_metadata(dc)
# 4. Check cache
data, use_cache = check_cache()
# 5. Load / download satellite data
data, ndvi = load_satellite_data(dc, data, use_cache)
# 6. Training data
train = load_training_data()
# 7. Train model
(model, X_train, X_test, y_train, y_test,
y_pred, accuracy, features_to_use, class_names) = train_model(train, data)
# 8. Save model
if model is not None:
save_trained_model(model, X_train, X_test, y_train, y_test,
y_pred, accuracy, features_to_use, class_names)
# 9. Cleanup
cleanup(client, cluster)
01.train_ODC_DecisionTree.ipynb
+891
View File
File diff suppressed because one or more lines are too long
COGNITO_GUIDE.md
+316
View File
@@ -0,0 +1,316 @@
# Cognito Authentication Guide
# Hướng Dẫn Xác Thực Cognito
## ✅ Kết quả Test
### Authentication Flow thành công:
```
Cognito Tokens → AWS Credentials → S3 Access
✓ ✓ ✓
```
### Thông tin User từ Cognito:
- **Username**: hienm2523001
- **Name**: Hien Phan
- **Email**: hienm2523001@gstudent.ctu.edu.vn
- **Groups**:
- default-group
- allocation:R-19244:CSIRO and Vietnam partners
- **Token Expiry**: ~8 giờ từ khi login
### S3 Buckets có thể truy cập:
✅ sentinel-cogs (us-west-2)
✅ sentinel-s2-l2a (eu-central-1)
---
## 📁 Files đã tạo
### 1. `cognito_auth.py`
Module Python để xác thực với Cognito tokens
**Tính năng:**
- Load Cognito tokens từ file
- Decode và hiển thị thông tin user
- Load AWS credentials (đã được EASI exchange từ Cognito)
- Set credentials vào environment
- Test S3 access
### 2. `test_cognito_s3.py`
Script test đầy đủ flow: Cognito → AWS → S3
---
## 🚀 Cách sử dụng
### Quick Test
```bash
python test_cognito_s3.py
```
### Sử dụng trong code
#### 1. Load Cognito authentication:
```python
from cognito_auth import CognitoAuthenticator
# Initialize
auth = CognitoAuthenticator(region='ap-southeast-1')
# Load tokens và credentials
auth.load_tokens_from_file('train_files/crediential.txt')
# Xem thông tin user
auth.print_token_info()
# Get AWS credentials
auth.get_credentials_from_cognito()
# Set vào environment
auth.set_environment_credentials()
```
#### 2. Truy cập S3:
```python
# Test S3 access
auth.test_s3_access('sentinel-cogs', 'us-west-2')
```
#### 3. Sử dụng với datacube:
```python
from datacube.utils.rio import configure_s3_access
# Configure S3 access
configure_s3_access(
aws_unsigned=False,
region_name='us-west-2',
cloud_defaults=True
)
# Load dữ liệu như bình thường
```
---
## 🔄 Cấu trúc File Credentials
File `train_files/crediential.txt` chứa:
```bash
# AWS Credentials (đã được exchange từ Cognito bởi EASI)
export AWS_ACCESS_KEY_ID="ASIA..."
export AWS_SECRET_ACCESS_KEY="..."
export AWS_SESSION_TOKEN="..."
# Cognito Tokens
Cognito: eyJraWQi... # Access Token
ID: eyJraWQi... # ID Token
```
---
## 📊 So sánh 2 phương pháp
| Feature | Direct Credentials | Cognito Tokens |
|---------|-------------------|----------------|
| **Authentication** | Không có | ✅ User info, groups |
| **S3 Access** | ✅ | ✅ |
| **User Identity** | Chỉ có role ARN | ✅ Username, email, groups |
| **Token Info** | Không | ✅ Expiry time, claims |
| **Security** | Basic | ✅ Better (identity-based) |
---
## 💡 Flow hoạt động
### Trong hệ thống EASI:
```
1. User login vào EASI Hub
↓
2. AWS Cognito xác thực
↓
3. Cognito trả về:
- Access Token (authentication)
- ID Token (user info)
↓
4. EASI Backend exchange tokens → AWS Credentials
↓
5. User nhận cả Cognito tokens + AWS credentials
↓
6. Sử dụng credentials để truy cập S3
```
### Trong code của bạn:
```python
# Load từ file
auth.load_tokens_from_file()
↓
# Parse user info
auth.print_token_info()
↓
# Get AWS credentials (đã có sẵn trong file)
auth.get_credentials_from_cognito()
↓
# Set environment
auth.set_environment_credentials()
↓
# Access S3
auth.test_s3_access()
```
---
## ⚙️ Advanced Usage
### Decode token để lấy thông tin:
```python
import jwt
decoded = jwt.decode(id_token, options={"verify_signature": False})
print(decoded)
# {
# 'cognito:username': 'hienm2523001',
# 'email': 'hienm2523001@gstudent.ctu.edu.vn',
# 'cognito:groups': ['default-group', 'allocation:R-19244:...'],
# 'exp': 1772661158,
# ...
# }
```
### Check token expiration:
```python
from datetime import datetime
exp = decoded['exp']
exp_time = datetime.fromtimestamp(exp)
now = datetime.now()
if exp_time > now:
print(f"Token còn hiệu lực đến: {exp_time}")
else:
print("Token đã hết hạn!")
```
### Sử dụng với boto3:
```python
import boto3
s3 = boto3.client(
's3',
aws_access_key_id=auth.aws_credentials['AccessKeyId'],
aws_secret_access_key=auth.aws_credentials['SecretAccessKey'],
aws_session_token=auth.aws_credentials['SessionToken']
)
# List objects
response = s3.list_objects_v2(Bucket='sentinel-cogs', MaxKeys=10)
```
---
## 🔒 Security Notes
### ✅ Best Practices:
- Token có thời hạn (tự động expire sau ~8 giờ)
- Sử dụng HTTPS cho mọi API calls
- KHÔNG commit tokens vào Git
- KHÔNG share tokens công khai
- Refresh tokens khi hết hạn
### ⚠️ Lưu ý:
- Cognito tokens và AWS credentials **ĐỀU CÓ THỜI HẠN**
- Khi hết hạn, cần login lại vào EASI hub
- File `.gitignore` nên bao gồm `train_files/crediential.txt`
---
## 🐛 Troubleshooting
### Error: "Token đã hết hạn"
```
✗ Token EXPIRED at: 2026-03-05 04:52:38
```
**Giải pháp:** Login lại vào EASI hub để lấy tokens mới
### Error: "ModuleNotFoundError: No module named 'jwt'"
```bash
pip install PyJWT
```
### Error: "No AWS credentials available"
**Giải pháp:**
- Check file `train_files/crediential.txt` có đầy đủ không
- Đảm bảo có cả AWS credentials VÀ Cognito tokens
### Error: "AccessDenied" khi truy cập S3
**Giải pháp:**
- Token có thể đã hết hạn
- Bucket có thể yêu cầu quyền cao hơn
- Thử bucket khác (public bucket)
---
## 📚 Dependencies
```bash
pip install boto3 botocore PyJWT datacube rasterio
```
Hoặc:
```bash
pip install -r requirements_api.txt
```
---
## 📞 Contact
- **EASI Asia Support**: CSIRO EASI Hub
- **Project**: R-19244: CSIRO and Vietnam partners
- **Region**: ap-southeast-1
---
## 📝 Example Output
```
======================================================================
Test S3 Access Using Cognito Tokens
======================================================================
[1/5] Loading Cognito tokens from file...
✓ AWS credentials loaded from file
✓ Cognito tokens loaded successfully
[2/5] Displaying token information...
User Information:
Username: hienm2523001
Name: Hien Phan
Email: hienm2523001@gstudent.ctu.edu.vn
Groups: default-group, allocation:R-19244:CSIRO and Vietnam partners
Token expires: 2026-03-05 04:52:38
Time remaining: 7h 45m
[3/5] Getting AWS credentials...
✓ Using AWS credentials loaded from file
[4/5] Configuring environment...
✓ AWS credentials set in environment
[5/5] Configuring datacube S3 access...
✓ Datacube S3 access configured
S3 Access Results:
✓ sentinel-cogs
✓ sentinel-s2-l2a
✓ SUCCESS: Cognito authentication working!
======================================================================
```
---
**Last updated:** March 4, 2026
**Status:** ✅ Working perfectly!
ODC_COGNITO_GUIDE.md
+405
View File
@@ -0,0 +1,405 @@
# ODC with Cognito Authentication Guide
# Hướng Dẫn Sử Dụng ODC với Cognito Authentication
## 📦 Files Đã Tạo
### 1. Core Modules
- **`new_import_ODC_cognito.py`** - ODC module tích hợp Cognito authentication
- **`cognito_auth.py`** - Cognito authentication core module
### 2. Test Scripts
- **`test_cognito_s3.py`** - Test Cognito authentication + S3 access
- **`test_s3_datacube_access.py`** - Test S3 access với datacube pattern
- **`test_s3_list_all.py`** - Demo list nhiều objects từ S3
### 3. Notebooks
- **`train_files/01.train_ODC_DecisionTree.ipynb`** - ✨ Updated với Cognito auth
- **`train_files/test_cognito_odc.ipynb`** - Demo notebook test Cognito + ODC
### 4. Documentation
- **`COGNITO_GUIDE.md`** - Hướng dẫn chi tiết về Cognito
- **`S3_ACCESS_GUIDE.md`** - Hướng dẫn truy cập S3
- **`ODC_COGNITO_GUIDE.md`** - File này
---
## 🚀 Quick Start
### Option 1: Sử dụng trong Notebook
```python
# 1. Import module
import sys
sys.path.insert(0, '/media/x79/2A7D-FAA0/remote-sensing')
import new_import_ODC_cognito
from new_import_ODC_cognito import *
# 2. Setup Cognito authentication
auth = setup_cognito_auth('train_files/crediential.txt')
# 3. Initialize datacube (S3 đã được config)
dc = datacube.Datacube()
# 4. Load data như bình thường
data = load_data(
dc=dc,
date_range=("2023-01-01", "2023-01-31"),
longtitude_range=(105.5, 106.0),
latitude_range=(9.5, 10.0)
)
```
### Option 2: Test độc lập
```bash
# Test Cognito authentication + S3
python test_cognito_s3.py
# Test list nhiều objects
python test_s3_list_all.py
```
---
## 📚 So Sánh: Trước vs Sau
### ❌ Trước (Không có Cognito):
```python
import new_import_ODC
from new_import_ODC import *
# Khởi tạo Dask + Datacube
cluster, client = notebook_utils.initialize_dask(use_gateway=True)
dc = datacube.Datacube()
# Configure S3 (unsigned - public access only)
configure_s3_access(aws_unsigned=True)
```
**Hạn chế:**
- Chỉ truy cập public buckets
- Không có authentication
- Không biết ai đang truy cập
- Không có audit trail
### ✅ Sau (Có Cognito):
```python
import new_import_ODC_cognito
from new_import_ODC_cognito import *
# Setup Cognito authentication
auth = setup_cognito_auth('train_files/crediential.txt')
# Thông tin user tự động hiển thị:
# Username: hienm2523001
# Email: hienm2523001@gstudent.ctu.edu.vn
# Groups: CSIRO and Vietnam partners
# Khởi tạo Datacube (S3 đã được authenticated)
cluster, client = notebook_utils.initialize_dask(use_gateway=True)
dc = datacube.Datacube()
```
**Lợi ích:**
- ✅ Truy cập cả private buckets
- ✅ Identity-based authentication
- ✅ Biết user identity (name, email, groups)
- ✅ Token tự động expire (security)
- ✅ Audit trail đầy đủ
- ✅ Group-based permissions
---
## 🔐 Authentication Flow
```
┌─────────────────────────────────────────────────────────────┐
│ 1. User Login → EASI Hub │
└───────────────────────┬─────────────────────────────────────┘
↓
┌─────────────────────────────────────────────────────────────┐
│ 2. AWS Cognito Authentication │
│ - Verify username/password │
│ - Check group membership │
└───────────────────────┬─────────────────────────────────────┘
↓
┌─────────────────────────────────────────────────────────────┐
│ 3. Cognito Returns Tokens │
│ - Access Token (for API authentication) │
│ - ID Token (user info: name, email, groups) │
└───────────────────────┬─────────────────────────────────────┘
↓
┌─────────────────────────────────────────────────────────────┐
│ 4. EASI Backend Exchanges Tokens → AWS Credentials │
│ - Access Key ID │
│ - Secret Access Key │
│ - Session Token │
└───────────────────────┬─────────────────────────────────────┘
↓
┌─────────────────────────────────────────────────────────────┐
│ 5. User Receives: │
│ ✓ Cognito Tokens (in crediential.txt) │
│ ✓ AWS Credentials (in crediential.txt) │
└───────────────────────┬─────────────────────────────────────┘
↓
┌─────────────────────────────────────────────────────────────┐
│ 6. In Your Code: │
│ setup_cognito_auth('crediential.txt') │
│ → Loads both tokens + credentials │
│ → Configures S3 access for datacube │
│ → Ready to use! │
└─────────────────────────────────────────────────────────────┘
```
---
## 📝 File Structure
```
remote-sensing/
├── cognito_auth.py # Core Cognito authentication
├── new_import_ODC_cognito.py # ODC module with Cognito
├── test_cognito_s3.py # Test script
├── test_s3_list_all.py # List S3 objects demo
├── COGNITO_GUIDE.md # Cognito documentation
├── S3_ACCESS_GUIDE.md # S3 access documentation
├── ODC_COGNITO_GUIDE.md # This file
│
└── train_files/
├── crediential.txt # ⚠️ PRIVATE - Credentials
├── 01.train_ODC_DecisionTree.ipynb # ✨ Updated notebook
└── test_cognito_odc.ipynb # Demo notebook
```
---
## 🔧 API Reference
### Core Functions
#### `setup_cognito_auth(credential_file, region='ap-southeast-1')`
Setup Cognito authentication cho S3/ODC access.
**Parameters:**
- `credential_file` (str): Path to credential file
- `region` (str): AWS region
**Returns:**
- `CognitoAuthenticator` instance hoặc `None` nếu failed
**Example:**
```python
auth = setup_cognito_auth('train_files/crediential.txt')
```
#### `get_cognito_auth()`
Lấy Cognito authenticator instance hiện tại.
**Returns:**
- Current `CognitoAuthenticator` instance
#### `print_auth_status()`
In trạng thái authentication hiện tại.
**Example:**
```python
print_auth_status()
# Output:
# ══════════════════════════════════════════════════════════
# AUTHENTICATION STATUS
# ══════════════════════════════════════════════════════════
# ✅ Cognito authentication is active
# ✅ AWS credentials loaded
# Access Key: ASIA4YF43ZWIXQ6HJIAY...
# ✅ Cognito tokens loaded
# User: hienm2523001
# Email: hienm2523001@gstudent.ctu.edu.vn
```
#### `auto_setup(credential_file='train_files/crediential.txt')`
Tự động setup nếu credential file tồn tại.
**Returns:**
- `CognitoAuthenticator` instance hoặc `None`
### Data Loading Functions
#### `load_data(dc, date_range, longtitude_range, latitude_range, measurements=None)`
Load Sentinel-2 L2A data từ datacube.
**Parameters:**
- `dc`: Datacube instance
- `date_range`: Tuple of (start_date, end_date)
- `longtitude_range`: Tuple of (min_lon, max_lon)
- `latitude_range`: Tuple of (min_lat, max_lat)
- `measurements`: List of bands (default: ['red', 'nir', 'scl'])
**Returns:**
- `xarray.Dataset`
#### `load_data_sen1(dc, date_range, longtitude_range, latitude_range)`
Load Sentinel-1 SAR data (VV, VH).
**Returns:**
- `xarray.Dataset` with VV, VH bands
#### `mask_clean(data)`
Apply cloud mask sử dụng SCL band.
**Returns:**
- Cleaned `xarray.Dataset`
#### `calculate_average(data, variables, resample='1MS')`
Calculate temporal average và resample.
**Returns:**
- Resampled `xarray.Dataset`
---
## ⚙️ Configuration
### Credential File Format
File `train_files/crediential.txt`:
```bash
export AWS_ACCESS_KEY_ID="ASIA4YF43ZWIXQ6HJIAY"
export AWS_SECRET_ACCESS_KEY="3N8KoV2ZBqQcFqRUVxQXW8K9sm90CNDV9aHUkNw0"
export AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEO7//////////..."
Cognito: eyJraWQiOiIzejR4V0txYmd5Mlo4NXR3TFVvRGFSNmp4...
ID: eyJraWQiOiJOMmdRc1c0S3o1YUltR3hGZEVJVmUx...
```
### Environment Variables
Sau khi setup, các environment variables được set:
```bash
AWS_ACCESS_KEY_ID
AWS_SECRET_ACCESS_KEY
AWS_SESSION_TOKEN
```
---
## 🐛 Troubleshooting
### Error: "Token đã hết hạn"
**Nguyên nhân:** Cognito tokens expire sau ~8 giờ
**Giải pháp:**
1. Login lại vào EASI Hub
2. Copy credentials mới
3. Update file `crediential.txt`
4. Restart notebook kernel
### Error: "cognito_auth module not found"
**Nguyên nhân:** Module chưa được import đúng path
**Giải pháp:**
```python
import sys
sys.path.insert(0, '/media/x79/2A7D-FAA0/remote-sensing')
import new_import_ODC_cognito
```
### Error: "No AWS credentials available"
**Nguyên nhân:** File credentials chưa đúng format hoặc thiếu
**Giải pháp:**
1. Check file `train_files/crediential.txt` exists
2. Verify format (có cả AWS credentials VÀ Cognito tokens)
3. Re-run `setup_cognito_auth()`
### Warning: "EASI tools not available"
**Tác động:** Module vẫn chạy nhưng dùng standard datacube functions
**Giải pháp:** (Optional)
```bash
pip install easi-tools
```
---
## 📊 Performance Notes
### Token Expiration
- **Cognito tokens**: ~8 hours
- **AWS session tokens**: ~12 hours
- **Best practice**: Refresh mỗi session
### S3 Access
- **Authenticated access**: Nhanh hơn (cached credentials)
- **Pagination**: Support listing unlimited objects
- **Concurrent requests**: Thread-safe
---
## 🔒 Security Best Practices
### DO ✅
- Store credentials trong file riêng biệt
- Add `crediential.txt` vào `.gitignore`
- Refresh tokens thường xuyên
- Use HTTPS cho mọi API calls
- Check token expiration trước khi dùng
### DON'T ❌
- KHÔNG commit credentials vào Git
- KHÔNG share credentials publicly
- KHÔNG hardcode credentials trong code
- KHÔNG dùng credentials đã expire
- KHÔNG skip authentication checks
---
## 📞 Support
### Documentation
- `COGNITO_GUIDE.md` - Chi tiết về Cognito
- `S3_ACCESS_GUIDE.md` - Chi tiết về S3
- `test_cognito_odc.ipynb` - Demo notebook
### Test Scripts
```bash
# Test full flow
python test_cognito_s3.py
# Test list objects
python test_s3_list_all.py
# Test trong notebook
jupyter notebook train_files/test_cognito_odc.ipynb
```
### Contact
- **Project**: R-19244: CSIRO and Vietnam partners
- **EASI Hub**: easi-asia-csiro
- **Region**: ap-southeast-1
---
## 📈 Migration Checklist
Nếu đang migrate từ code cũ (không có Cognito):
- [ ] Copy `cognito_auth.py` vào project
- [ ] Copy `new_import_ODC_cognito.py` vào project
- [ ] Update imports: `new_import_ODC` → `new_import_ODC_cognito`
- [ ] Thêm `setup_cognito_auth()` trước datacube initialization
- [ ] Remove `configure_s3_access(aws_unsigned=True)`
- [ ] Test với `test_cognito_odc.ipynb`
- [ ] Update notebooks khác tương tự
---
**Version:** 1.0
**Last Updated:** March 4, 2026
**Status:** ✅ Production Ready
S3_ACCESS_GUIDE.md
+205
View File
@@ -0,0 +1,205 @@
# AWS S3 Direct Access Test Guide
# Hướng Dẫn Test Truy Cập Trực Tiếp AWS S3
## Overview / Tổng quan
Dự án này bao gồm các file test để kiểm tra kết nối và truy cập AWS S3 với credentials có quyền hạn chế.
## Files / Các File
### 1. `test_s3_direct_access.py`
File test cơ bản cho AWS S3, yêu cầu quyền `ListAllMyBuckets`.
**Sử dụng:**
```bash
python test_s3_direct_access.py
```
**Lưu ý:** File này sẽ báo lỗi `AccessDenied` nếu credentials không có quyền list tất cả buckets.
### 2. `test_s3_datacube_access.py` ⭐ (Recommended / Khuyên dùng)
File test nâng cao, truy cập trực tiếp các bucket cụ thể mà không cần quyền `ListAllMyBuckets`.
**Sử dụng:**
```bash
python test_s3_datacube_access.py
```
**Tính năng:**
- ✓ Load credentials từ `train_files/crediential.txt`
- ✓ Cấu hình datacube S3 access
- ✓ Test truy cập nhiều bucket phổ biến
- ✓ Hiển thị danh sách file trong bucket
- ✓ Thống kê kết quả test
## Kết quả Test / Test Results
### Các bucket có thể truy cập:
- ✅ **sentinel-cogs** (us-west-2) - Sentinel-2 L2A COGs data
- Public bucket chứa dữ liệu Sentinel-2
- Prefix: `sentinel-s2-l2a-cogs/`
### Credentials hiện tại:
```
Role: arn:aws:sts::876569415057:assumed-role/easi-asia-csiro-easihub-client/hienm2523001
Region: ap-southeast-1
Expiration: Token có thời hạn (session token)
```
### Quyền hạn (Permissions):
- ✅ Read objects từ public buckets
- ✅ List objects trong bucket cụ thể
- ✅ Head bucket (check bucket existence)
- ❌ ListAllMyBuckets (list tất cả buckets)
- ❌ GetBucketLocation (một số bucket)
## Cách sử dụng trong code / How to use in code
### 1. Load credentials và cấu hình S3:
```python
import os
from datacube.utils.rio import configure_s3_access
# Load credentials
os.environ['AWS_ACCESS_KEY_ID'] = "YOUR_ACCESS_KEY"
os.environ['AWS_SECRET_ACCESS_KEY'] = "YOUR_SECRET_KEY"
os.environ['AWS_SESSION_TOKEN'] = "YOUR_SESSION_TOKEN"
# Configure S3 access for datacube/rasterio
configure_s3_access(
aws_unsigned=False,
region_name='ap-southeast-1',
cloud_defaults=True
)
```
### 2. Truy cập S3 objects với boto3:
```python
import boto3
# Create S3 client
s3_client = boto3.client('s3', region_name='us-west-2')
# List objects in bucket
response = s3_client.list_objects_v2(
Bucket='sentinel-cogs',
Prefix='sentinel-s2-l2a-cogs/',
MaxKeys=10
)
for obj in response.get('Contents', []):
print(f"File: {obj['Key']}, Size: {obj['Size']} bytes")
```
### 3. Đọc dữ liệu từ S3 với rasterio:
```python
import rasterio
# Read raster file directly from S3
s3_path = 's3://sentinel-cogs/sentinel-s2-l2a-cogs/1/C/CV/2018/10/S2B_1CCV_20181004_0_L2A/B02.tif'
with rasterio.open(s3_path) as src:
data = src.read(1)
print(f"Shape: {data.shape}")
print(f"CRS: {src.crs}")
```
### 4. Load dữ liệu với xarray:
```python
import xarray as xr
import rioxarray
# Open S3 raster with rioxarray
s3_path = 's3://sentinel-cogs/sentinel-s2-l2a-cogs/1/C/CV/2018/10/S2B_1CCV_20181004_0_L2A/B02.tif'
ds = rioxarray.open_rasterio(s3_path)
print(ds)
```
## Refresh Credentials / Làm mới Credentials
AWS session tokens có thời hạn. Khi token hết hạn, bạn sẽ thấy lỗi:
```
ExpiredToken: The security token included in the request is expired
```
**Cách làm mới:**
1. Login lại vào AWS console hoặc EASI hub
2. Copy credentials mới
3. Update file `train_files/crediential.txt`
4. Chạy lại test
## Troubleshooting / Xử lý lỗi
### Lỗi: `ModuleNotFoundError: No module named 'boto3'`
```bash
pip install boto3 botocore
```
### Lỗi: `ModuleNotFoundError: No module named 'datacube'`
```bash
pip install datacube
```
### Lỗi: `AccessDenied`
- Kiểm tra credentials có đúng không
- Kiểm tra token còn hạn không
- Thử bucket khác (có thể bucket đó yêu cầu quyền cao hơn)
### Lỗi: `ExpiredToken`
- Token AWS đã hết hạn
- Cần refresh credentials mới
### Lỗi: `404 Not Found`
- Bucket không tồn tại
- Bucket name có thể sai
- Region có thể sai
## Dependencies / Thư viện cần thiết
```bash
pip install boto3 botocore datacube rasterio rioxarray xarray
```
Hoặc sử dụng file requirements:
```bash
pip install -r requirements_api.txt
```
## Security / Bảo mật
⚠️ **QUAN TRỌNG:**
- **KHÔNG** commit file `crediential.txt` lên Git
- **KHÔNG** share credentials công khai
- Session tokens có thời hạn ngắn (thường vài giờ)
- Luôn sử dụng IAM roles với quyền tối thiểu cần thiết
## Thông tin thêm / Additional Information
### Public Sentinel-2 Buckets:
- `sentinel-cogs` (us-west-2) - ✅ Accessible
- `sentinel-s2-l2a` (eu-central-1) - COGs format
- `sentinel-s2-l1c` (eu-central-1) - Level 1C
### EASI/Datacube Buckets:
- Thường là private buckets
- Cần credentials với quyền cụ thể
- Contact admin để được cấp quyền
## Examples / Ví dụ
Xem các file trong thư mục `backup_S3_download_Amazon/`:
- `new_import_S3.py` - Load dữ liệu S3 với datacube
## Contact / Liên hệ
Nếu cần thêm quyền truy cập hoặc gặp vấn đề:
- Contact: EASI Asia CSIRO admin
- Role: easi-asia-csiro-easihub-client
---
**Last updated:** March 4, 2026
cognito_auth.py
+348
View File
@@ -0,0 +1,348 @@
#!/usr/bin/env python3
"""
Cognito Authentication Module
Module xác thực sử dụng AWS Cognito Tokens
"""
import os
import json
import base64
import boto3
from botocore.exceptions import ClientError
from datetime import datetime
import jwt
class CognitoAuthenticator:
"""
Class để xác thực và lấy AWS credentials từ Cognito tokens
"""
def __init__(self, region='ap-southeast-1'):
self.region = region
self.cognito_identity = None
self.access_token = None
self.id_token = None
self.aws_credentials = None
def load_tokens_from_file(self, credential_file='train_files/crediential.txt'):
"""
Load Cognito tokens và AWS credentials từ file
"""
try:
with open(credential_file, 'r') as f:
content = f.read()
aws_creds = {}
# Parse all credentials
lines = content.split('\n')
for line in lines:
line = line.strip()
# Parse AWS credentials
if line.startswith('export '):
line = line[7:]
if '=' in line:
key, value = line.split('=', 1)
value = value.strip('"')
aws_creds[key] = value
# Parse Cognito tokens
elif line.startswith('Cognito:'):
self.access_token = line.split('Cognito:')[1].strip()
elif line.startswith('ID:'):
self.id_token = line.split('ID:')[1].strip()
# Set AWS credentials if found
if aws_creds:
self.aws_credentials = {
'AccessKeyId': aws_creds.get('AWS_ACCESS_KEY_ID', ''),
'SecretAccessKey': aws_creds.get('AWS_SECRET_ACCESS_KEY', ''),
'SessionToken': aws_creds.get('AWS_SESSION_TOKEN', ''),
}
print("✓ AWS credentials loaded from file")
if self.access_token and self.id_token:
print("✓ Cognito tokens loaded successfully")
return True
else:
print("✗ Cognito tokens not found in file")
return False
except FileNotFoundError:
print(f"✗ Error: Credential file not found: {credential_file}")
return False
except Exception as e:
print(f"✗ Error loading tokens: {e}")
return False
def decode_token(self, token, verify=False):
"""
Decode JWT token để xem thông tin
"""
try:
# Decode without verification (for inspection only)
decoded = jwt.decode(token, options={"verify_signature": False})
return decoded
except Exception as e:
print(f"✗ Error decoding token: {e}")
return None
def print_token_info(self):
"""
In thông tin từ Cognito tokens
"""
if not self.id_token:
print("✗ No ID token available")
return
print("\n=== Cognito Token Information ===")
try:
decoded_id = self.decode_token(self.id_token)
decoded_access = self.decode_token(self.access_token)
if decoded_id:
print("\nUser Information:")
print(f" Username: {decoded_id.get('cognito:username', 'N/A')}")
print(f" Name: {decoded_id.get('name', 'N/A')}")
print(f" Email: {decoded_id.get('email', 'N/A')}")
print(f" Groups: {', '.join(decoded_id.get('cognito:groups', []))}")
# Check expiration
exp = decoded_id.get('exp')
if exp:
exp_time = datetime.fromtimestamp(exp)
now = datetime.now()
if exp_time > now:
time_left = exp_time - now
hours = time_left.seconds // 3600
minutes = (time_left.seconds % 3600) // 60
print(f" Token expires: {exp_time}")
print(f" Time remaining: {hours}h {minutes}m")
else:
print(f" ✗ Token EXPIRED at: {exp_time}")
return decoded_id, decoded_access
except Exception as e:
print(f"✗ Error parsing token info: {e}")
return None, None
def get_credentials_from_cognito(self, identity_pool_id=None, cognito_provider=None):
"""
Exchange Cognito ID token để lấy AWS temporary credentials
Args:
identity_pool_id: Cognito Identity Pool ID (nếu có)
cognito_provider: Cognito provider URL
"""
if not self.id_token:
print("✗ No ID token available")
return False
try:
print("\n=== Getting AWS Credentials from Cognito ===")
# Nếu không có identity pool ID, thử tự động detect
if not identity_pool_id:
print("⚠ No Identity Pool ID provided")
print("⚠ Using existing AWS credentials (already exchanged from Cognito)...")
# Kiểm tra xem có credentials đã load không
if self.aws_credentials and self.aws_credentials.get('AccessKeyId'):
print("✓ Using AWS credentials loaded from file")
return True
# Kiểm tra trong environment
elif os.environ.get('AWS_ACCESS_KEY_ID'):
print("✓ Using existing AWS credentials from environment")
self.aws_credentials = {
'AccessKeyId': os.environ.get('AWS_ACCESS_KEY_ID'),
'SecretAccessKey': os.environ.get('AWS_SECRET_ACCESS_KEY'),
'SessionToken': os.environ.get('AWS_SESSION_TOKEN'),
}
return True
else:
print("✗ No AWS credentials available")
return False
# Create Cognito Identity client
cognito_identity = boto3.client('cognito-identity', region_name=self.region)
# Default provider URL nếu không có
if not cognito_provider:
decoded = self.decode_token(self.id_token)
if decoded and 'iss' in decoded:
iss = decoded['iss']
# Extract provider from issuer URL
# Example: https://cognito-idp.ap-southeast-1.amazonaws.com/ap-southeast-1_C4GCbYaOa
cognito_provider = iss.replace('https://', '')
print(f"Identity Pool ID: {identity_pool_id}")
print(f"Cognito Provider: {cognito_provider}")
# Get identity ID
logins = {cognito_provider: self.id_token}
identity_response = cognito_identity.get_id(
IdentityPoolId=identity_pool_id,
Logins=logins
)
identity_id = identity_response['IdentityId']
print(f"✓ Got Identity ID: {identity_id}")
# Get credentials for identity
credentials_response = cognito_identity.get_credentials_for_identity(
IdentityId=identity_id,
Logins=logins
)
self.aws_credentials = credentials_response['Credentials']
print("✓ Successfully obtained AWS credentials from Cognito!")
print(f" Access Key: {self.aws_credentials['AccessKeyId'][:20]}...")
print(f" Expiration: {self.aws_credentials['Expiration']}")
return True
except ClientError as e:
error_code = e.response.get('Error', {}).get('Code', 'Unknown')
print(f"✗ AWS Error: {error_code}")
print(f"✗ Message: {e.response.get('Error', {}).get('Message', 'Unknown')}")
if error_code == 'NotAuthorizedException':
print("✗ Token may be expired or invalid")
return False
except Exception as e:
print(f"✗ Error getting credentials: {e}")
return False
def set_environment_credentials(self):
"""
Set AWS credentials vào environment variables
"""
if not self.aws_credentials:
print("✗ No AWS credentials available")
return False
try:
os.environ['AWS_ACCESS_KEY_ID'] = self.aws_credentials['AccessKeyId']
os.environ['AWS_SECRET_ACCESS_KEY'] = self.aws_credentials['SecretAccessKey']
os.environ['AWS_SESSION_TOKEN'] = self.aws_credentials['SessionToken']
print("✓ AWS credentials set in environment")
return True
except Exception as e:
print(f"✗ Error setting credentials: {e}")
return False
def test_s3_access(self, bucket_name='sentinel-cogs', region='us-west-2'):
"""
Test S3 access với credentials từ Cognito
"""
try:
print(f"\n=== Testing S3 Access ===")
print(f"Bucket: {bucket_name}")
# Create S3 client với credentials
if self.aws_credentials:
s3_client = boto3.client(
's3',
region_name=region,
aws_access_key_id=self.aws_credentials['AccessKeyId'],
aws_secret_access_key=self.aws_credentials['SecretAccessKey'],
aws_session_token=self.aws_credentials['SessionToken']
)
else:
# Use credentials from environment
s3_client = boto3.client('s3', region_name=region)
# Test bucket access
s3_client.head_bucket(Bucket=bucket_name)
print(f"✓ Successfully accessed bucket: {bucket_name}")
# List some objects
response = s3_client.list_objects_v2(
Bucket=bucket_name,
MaxKeys=100
)
if 'Contents' in response:
print(f"✓ Listed {len(response['Contents'])} objects:")
for obj in response['Contents']:
size_mb = obj['Size'] / (1024 * 1024)
print(f" - {obj['Key']} ({size_mb:.2f} MB)")
return True
except ClientError as e:
error_code = e.response.get('Error', {}).get('Code', 'Unknown')
print(f"✗ S3 Error: {error_code}")
return False
except Exception as e:
print(f"✗ Error: {e}")
return False
def main():
"""
Test function
"""
print("=" * 60)
print("Cognito Authentication Test")
print("Test Xác Thực Cognito")
print("=" * 60)
# Initialize authenticator
auth = CognitoAuthenticator(region='ap-southeast-1')
# Step 1: Load tokens
print("\n[Step 1] Loading Cognito tokens...")
if not auth.load_tokens_from_file():
print("\n✗ Failed to load tokens")
return False
# Step 2: Display token info
print("\n[Step 2] Parsing token information...")
auth.print_token_info()
# Step 3: Get AWS credentials
print("\n[Step 3] Getting AWS credentials...")
# Option A: Nếu có Identity Pool ID (uncomment nếu biết)
# identity_pool_id = 'ap-southeast-1:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx'
# auth.get_credentials_from_cognito(identity_pool_id=identity_pool_id)
# Option B: Sử dụng credentials có sẵn trong file
auth.get_credentials_from_cognito()
# Step 4: Set environment
print("\n[Step 4] Setting environment credentials...")
auth.set_environment_credentials()
# Step 5: Test S3 access
print("\n[Step 5] Testing S3 access...")
auth.test_s3_access()
print("\n" + "=" * 60)
print("✓ Test completed!")
print("=" * 60)
return True
if __name__ == "__main__":
try:
success = main()
exit(0 if success else 1)
except KeyboardInterrupt:
print("\n\n✗ Test interrupted by user")
exit(1)
except Exception as e:
print(f"\n✗ Fatal error: {e}")
import traceback
traceback.print_exc()
exit(1)
deafrica_tools/__pycache__/__init__.cpython-310.pyc
BIN
View File
Binary file not shown.
deafrica_tools/__pycache__/bandindices.cpython-310.pyc
BIN
View File
Binary file not shown.
easi_tools/__init__.py
+9
View File
@@ -0,0 +1,9 @@
#!python 3
from .deployments import EasiDefaults
from .notebook_utils import \
heading, \
initialize_dask, \
mostcommon_crs, \
unset_cachingproxy, \
xarray_object_size
easi_tools/deployments.py
+354
View File
@@ -0,0 +1,354 @@
#!python3
import sys
import os
import logging
import collections
# A class that provides notebook variables for each of the EASI deployments
# Map an internal deployment name to deployment variables and search parameters.
# Update to ensure that the product/space/time parameters are available in the respective databases
deployment_map = {
'adias': {
'domain': 'adias.aquawatchaus.space',
'db_database': 'adias_prod_db',
'training_shapefile': '',
'scratch': 'adias-prod-user-scratch',
'productmap': {'landsat': 'landsat8_c2l2_sr', 'sentinel-2': 's2_l2a', 'sar': 'asf_s1_grd_gamma0', 'dem': 'copernicus_dem_30'},
'location': 'Lake Tahoe, California',
'latitude': (39.0, 39.3),
'longitude': (-120.2, -119.9),
'time': ('2022-02-01', '2022-05-01'),
'target': {
'landsat': {'crs': 'epsg:26911', 'resolution': (-30,30)},
'sentinel-2': {'crs': 'epsg:26911', 'resolution': (-10,10)}
},
'aliases': {
'landsat': {'qa_band': 'qa_pixel', 'nir': 'nir08', 'swir1': 'swir16', 'swir2': 'swir22'}
},
'qa_mask': {
'landsat': {'nodata': False, 'water': 'land_or_cloud',
'cloud': 'not_high_confidence', 'cloud_shadow': 'not_high_confidence'}
}
},
'asia': {
'domain': 'asia.easi-eo.solutions',
'db_database': 'easi_asia_db',
'training_shapefile': '',
'scratch': 'easi-asia-user-scratch',
'productmap': {'landsat': 'landsat8_c2l2_sr', 'sentinel-2': 'sentinel_2_c1_l2a', 'sentinel-1': 'sentinel1_grd_gamma0_20m', 'dem': 'copernicus_dem_30'},
'location': 'Lake Tempe, Indonesia',
'latitude': (-4.2, -3.9),
'longitude': (119.8, 120.1),
'time': ('2020-02-01', '2020-04-01'),
'proxy': True,
'target': {
'landsat': {'crs': 'epsg:32650', 'resolution': (-30,30)},
'sentinel-2': {'crs': 'epsg:32650', 'resolution': (-10,10)}
},
'aliases': {
'landsat': {'qa_band': 'qa_pixel', 'nir': 'nir08', 'swir1': 'swir16', 'swir2': 'swir22'}
},
'qa_mask': {
'landsat': {'nodata': False, 'water': 'land_or_cloud',
'cloud': 'not_high_confidence', 'cloud_shadow': 'not_high_confidence'}
}
},
'chile': {
'domain': 'datacubechile.cl',
'db_database': 'easido_prod_db',
'training_shapefile': '',
'scratch': 'easido-prod-user-scratch',
'productmap': {'landsat': 'landsat8_c2l2_sr', 'sentinel-2': 'sentinel_2_c1_l2a', 'sar': 'asf_s1_grd_gamma0', 'dem': 'copernicus_dem_30'},
'location': 'La Serena, Chile',
'latitude': (-29.95, -29.85),
'longitude': (-71.3, -71.2),
'latitude_big': (-29.95, -27.95),
'longitude_big': (-71.3, -69.3),
'time': ('2022-02-01', '2022-05-01'),
'target': {
'landsat': {'crs': 'epsg:32718', 'resolution': (-30,30)},
'sentinel-2': {'crs': 'epsg:32718', 'resolution': (-10,10)}
},
'aliases': {
'landsat': {'qa_band': 'qa_pixel', 'nir': 'nir08', 'swir1': 'swir16', 'swir2': 'swir22'}
},
'qa_mask': {
'landsat': {'nodata': False, 'water': 'land_or_cloud',
'cloud': 'not_high_confidence', 'cloud_shadow': 'not_high_confidence'}
}
},
'cal': {
'domain': 'cal.ceos.org',
'db_database': 'easi_cal_db',
'training_shapefile': './ancillary_data/VA_Counties_Newport_News.shp',
'scratch': 'easi-cal-user-scratch',
'ows': False,
'map': False,
'productmap': {'landsat': 'landsat8_c2l2_sr', 'sentinel-2': 's2_l2a', 'sentinel-1': 's1_rtc', 'dem': 'copernicus_dem_30'},
'location': 'Newport News, Virginia',
'latitude': (37.02, 37.12),
'longitude': (-76.55, -76.45),
'time': ('2022-01-01', '2022-04-01'),
'target': {
'landsat': {'crs': 'epsg:32618', 'resolution': (-30,30)},
'sentinel-2': {'crs': 'epsg:32618', 'resolution': (-10,10)}
},
'aliases': {
'landsat': {'qa_band': 'qa_pixel', 'nir': 'nir08', 'swir1': 'swir16', 'swir2': 'swir22'}
},
'qa_mask': {
'landsat': {'nodata': False, 'water': 'land_or_cloud',
'cloud': 'not_high_confidence', 'cloud_shadow': 'not_high_confidence'}
}
},
'csiro': {
'domain': 'csiro.easi-eo.solutions',
'db_database': 'easihub_csiro_db',
'training_shapefile': '',
'scratch': 'easihub-csiro-user-scratch',
'productmap': {'landsat': 'ga_ls8c_ard_3', 'sentinel-2': 'ga_s2am_ard_3', 'sentinel-1': 'sentinel1_grd_gamma0_20m', 'dem': 'copernicus_dem_30'},
'location': 'Lake Hume, Australia',
'latitude': (-36.3, -35.8),
'longitude': (146.8, 147.3),
'time': ('2020-02-01', '2020-04-01'),
'aliases': {
'landsat': {'red': 'nbart_red', 'green': 'nbart_green', 'blue': 'nbart_blue',
'nir': 'nbart_nir', 'swir1': 'nbart_swir_1', 'swir2': 'nbart_swir_2',
'qa_band': 'oa_fmask'}
},
'qa_mask': {
'landsat': {'fmask':'valid'}
}
},
'dcceew': {
'domain': 'dcceew.easi-eo.solutions',
'db_database': 'dcceew_prod_db',
'training_shapefile': '',
'scratch': 'dcceew-prod-user-scratch',
'productmap': {'landsat': 'ga_ls8c_ard_3', 'sentinel-2': 'ga_s2am_ard_3', 'sentinel-1': 'sentinel1_grd_gamma0_20m', 'dem': 'copernicus_dem_30'},
'location': 'Lake Hume, Australia',
'latitude': (-36.3, -35.8),
'longitude': (146.8, 147.3),
'time': ('2020-02-01', '2020-04-01'),
'aliases': {
'landsat': {'red': 'nbart_red', 'green': 'nbart_green', 'blue': 'nbart_blue',
'nir': 'nbart_nir', 'swir1': 'nbart_swir_1', 'swir2': 'nbart_swir_2',
'qa_band': 'oa_fmask'}
},
'qa_mask': {
'landsat': {'fmask':'valid'}
}
},
'sub-apse2': {
'domain': 'sub-apse2.easi-eo.solutions',
'db_database': '',
'training_shapefile': '',
'scratch': '',
'ows': False,
'map': False,
'productmap': {'landsat': 'ga_ls8c_ard_3', 'sentinel-2': 'ga_s2am_ard_3', 'dem': 'copernicus_dem_30'},
'location': 'Lake Hume, Australia',
'latitude': (-36.3, -35.8),
'longitude': (146.8, 147.3),
'time': ('2020-02-01', '2020-04-01'),
'aliases': {
'landsat': {'red': 'nbart_red', 'green': 'nbart_green', 'blue': 'nbart_blue',
'nir': 'nbart_nir', 'swir1': 'nbart_swir_1', 'swir2': 'nbart_swir_2',
'qa_band': 'oa_fmask'}
},
'qa_mask': {
'landsat': {'fmask':'valid'}
}
},
}
class EasiDefaults():
"""Provide deployment-specific default variables for EASI notebooks"""
def __init__(self, deployment=None):
"""Initialise"""
self._log = _getlogger(self.__class__.__name__)
self.name = deployment if deployment else self._find_deployment()
self.deployment = self._validate(self.name)
self.proxy = None
self._aliases = {}
if self.deployment and self.deployment.get('proxy', None):
self.proxy = EasiCachingProxy()
if self.deployment:
self._log.info(f'Successfully found configuration for deployment "{self.name}"')
def _validate(self, deployment) -> dict:
"""Return the dict associated with the deployment name"""
names = deployment_map.keys()
if deployment is None or deployment not in names:
self._log.error(f'Deployment name not recognised: {deployment}')
self._log.error(f'Select one of: {", ".join(names)}')
return None
return deployment_map[deployment]
def _find_deployment(self) -> str:
"""Use the deployment's database environment variable as a lookup into the deployment_map dict"""
db_database = os.environ['DB_DATABASE']
deployment_name = [item for item in deployment_map if deployment_map[item]["db_database"] == db_database]
msg = 'Try specifying one using EasiDefaults(deployment="deployment_name").'
if len(deployment_name) == 0:
self._log.error(f'Deployment could not be found automatically. {msg}')
return None
elif len(deployment_name) > 1:
self._log.error(f'More than one deployment found. {msg}')
return None
return deployment_name[0]
@property
def domain(self):
"""Deployment domain"""
return self.deployment['domain']
@property
def db_database(self):
"""Database name"""
return self.deployment['db_database']
@property
def training_shapefile(self):
"""A local shapefile"""
return self.deployment['training_shapefile']
@property
def hub(self):
"""JupyterLab URL"""
return f'https://hub.{self.domain}'
@property
def explorer(self):
"""Explorer URL"""
return f'https://explorer.{self.domain}'
@property
def ows(self):
"""OWS URL"""
if not self.deployment.get('ows', True):
self._log.warning(f'Deployment does not have an OWS service: {self.name}')
return None
return f'https://ows.{self.domain}'
@property
def terria(self):
"""Terria Map URL"""
if not self.deployment.get('map', True):
self._log.warning(f'Deployment does not have a Map service: {self.name}')
return None
return f'https://map.{self._domain()}'
@property
def scratch(self):
"""Scratch bucket"""
return self.deployment['scratch']
@property
def location(self):
"""Default location name"""
return self.deployment['location']
@property
def latitude(self):
"""Default latitude range"""
return self.deployment['latitude']
@property
def longitude(self):
"""Default longitude range"""
return self.deployment['longitude']
@property
def latitude_big(self):
"""Default big latitude range"""
if 'latitude_big' in self.deployment:
return self.deployment['latitude_big']
self._log.warning(f'Default big latitude range not defined for "{self.deployment}". Using default latitude range')
return self.latitude
@property
def longitude_big(self):
"""Default big longitude range"""
if 'longitude_big' in self.deployment:
return self.deployment['longitude_big']
self._log.warning(f'Default big longitude range not defined for "{self.deployment}". Using default longitude range')
return self.latitude
@property
def time(self):
"""Default time range"""
return self.deployment['time']
def product(self, family='landsat'):
"""Product name. Family loosely describes products from a satellite series or product type."""
p = self.deployment['productmap'].get(family, None)
if p is None:
self._log.warning(f'Product family not defined for "{self.name}": {family}')
out = ', '.join([f'{k} > {v}' for k,v in self.deployment['productmap'].items()])
self._log.warning(f'{self.name}: {out}')
return None
return p
def crs(self, family='landsat'):
"""Default resolution. Family loosely describes products from a satellite series or product type."""
return self.deployment.get('target', {}).get(family, {}).get('crs', None)
def resolution(self, family='landsat'):
"""Default resolution. Family loosely describes products from a satellite series or product type."""
return self.deployment.get('target', {}).get(family, {}).get('resolution', None)
def aliases(self, family='landsat') -> collections.UserDict:
"""Return a dict-like object that maps a common name to a specific measurement/alias name.
Family loosely describes products from a satellite series or product type.
The common name is returned if there is no specific measurement/alias name defined.
That is, the common name should work as a measurement/alias name for the family in this deployment.
Else, provide a specific measurement/alias name in the defaults above.
"""
if family not in self._aliases:
self._aliases[family] = EasiAlias(self.deployment.get('aliases', {}).get(family, {}))
return self._aliases[family]
def qa_mask(self, family='landsat') -> dict:
"""Default QA mask values. Family loosely describes products from a satellite series or product type."""
return self.deployment.get('qa_mask', {}).get(family, {})
class EasiAlias(collections.UserDict):
"""Custom UserDict that returns a default measurement name for a given key if defined.
Else returns the key as the value. Items can not be set."""
def __init__(self, default:dict = {}):
self.data = default
self._log = _getlogger(self.__class__.__name__)
def __getitem__(self, key):
if key in self.data:
return self.data[key]
return key
def __setitem__(self, key, val):
self._log.error(f'Error <{self.__class__.__name__}>: Can not set items')
class EasiCachingProxy():
"""Set, unset and return information about the user's caching-proxy configuration"""
def __init__(self):
pass
def _getlogger(name):
"""Return a logger. Define here to limit external dependencies"""
# Default logger
# log.hasHandlers() = False
# log.getEffectiveLevel() = 30 = warning
# log.propagate = True
logger = logging.getLogger(name)
logger.setLevel(logging.INFO)
if not len(logger.handlers):
logger.addHandler(logging.StreamHandler(sys.stdout))
logger.propagate = False # Do not propagate up to root logger, which may have other handlers
return logger
easi_tools/load_s2l2a.py
+293
View File
@@ -0,0 +1,293 @@
#!python
# Sentinel-2 L2A Collection 0 scaling and offset corrections.
# - Applies to data indexed from https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a
# - The newer https://earth-search.aws.element84.com/v1/collections/sentinel-2-c1-l2a (Collection 1) may not be affected in the same way
#
# TL;DR:
# DN values in COG files have different definitions depending on the processing baseline version
# and whether the offset change has been pre-applied by the cloud data custodian.
#
# Background:
#
# ESA has undertaken a reprocessing of the Sentinel-2 L2A product that includes
# a change to the offset value used to convert digital numbers (in file) to
# scientific values (reflectances).
#
# https://sentinels.copernicus.eu/web/sentinel/technical-guides/sentinel-2-msi/level-2a-algorithms-products
#
# L2A algorithm and products: Starting with the PB 04.00 (25th January 2022), the dynamic
# range of the Level-2A products is shifted by a band-dependent constant: BOA_ADD_OFFSET.
# This offset will allow encoding negative surface reflectances that may occur over very
# dark surfaces.
#
# L2A_SRi = (L2A_DNi + BOA_ADD_OFFSETi) / QUANTIFICATION_VALUEi
#
# QUANTIFICATION_VALUEi = 10000
# BOA_ADD_OFFSETi = -1000
#
# refl = (dn -1000) / 10000
# refl = dn/10000 - 1000/10000
# refl = dn * 0.0001 - 0.1
#
# These are the values in the EASI product definition, e.g.
# https://explorer.asia.easi-eo.solutions/products/s2_l2a.odc-product.yaml
#
# Example workflow:
#
# ESA's reprocessing is flowing through to the AWS open data repository of S2 L2A but
# while this stabilises we may see inconsistencies in time series queries due to:
# - More than one processed version of a dataset (scene) in the AWS bucket and indexed in an EASI database
# - Datasets (scenes) that indicate they have an offset applied by ESA but the offset correction
# has been not been applied to the COG
#
# Element-84 discussion:
# https://github.com/Element84/earth-search/issues/23#issuecomment-1834674853
import xarray as xr
import pandas as pd
import logging
from pathlib import Path
import sys, re
import datacube
from datacube.api.core import output_geobox
from datacube.api.query import SPATIAL_KEYS, CRS_KEYS, OTHER_KEYS
from datacube.utils import masking
# Set logger
log = logging.getLogger(Path(__file__).stem)
log.setLevel(logging.INFO)
log.addHandler(logging.StreamHandler(sys.stdout))
# Constants
search_keys = (
'product',
'time',
'geopolygon',
'like',
'limit',
'ensure_location',
'dataset_predicate',
) + SPATIAL_KEYS + CRS_KEYS + OTHER_KEYS
# TODO: get measurement aliases from the ODC product record
refl_bands = {
'coastal','band_01','B01','coastal_aerosol',
'blue','band_02','B02',
'green','band_03','B03',
'red','band_04','B04',
'rededge1','band_05','B05','red_edge_1',
'rededge2','band_06','B06','red_edge_2',
'rededge3','band_07','B07','red_edge_3',
'nir','band_08','B08','nir_1',
'nir08','band_8a','B8A','nir_2',
'nir09','band_09','B09','nir_3',
'swir16','band_11','B11','swir_1','swir_16',
'swir22','band_12','B12','swir_2','swir_22',
}
scale_factor = 0.0001
add_offset = -0.1
def highest_sequence_number(matches: list) -> dict:
"""Filter for the highest element84 processing sequence number per scene (scene label excluding the sequence number)
: return : { scene_id_excluding_sequence_number : { highest_sequence_number : datacube.model.Dataset }}
"""
p = re.compile(r'(S2.+)_([0-9]+)_(L2A)')
sorter = {}
for ds in matches:
# Separate the scene label from the sequence number
label = ds.metadata_doc['label']
m = p.match(label)
if not m:
log.warning(f'Dataset label does not match expected pattern: {label}')
continue
key = f'{m.group(1)}_{m.group(3)}'
seq = int(m.group(2))
# Retain the highest sequence number
if key in sorter:
if list(sorter[key])[0] < seq:
sorter[key] = {seq: ds}
else:
sorter[key] = {seq: ds}
return sorter
def ds_requires_offset(ds: datacube.model.Dataset) -> bool:
"""Return True if a dataset's metadata indicates that the offset correction should be applied"""
props = ds.metadata_doc['properties']
# If baseline is less than '04.00' then offset correction does not apply
baseline = props.get('s2:processing_baseline', '0.0')
p = re.compile(r'(\d+)\.(\d+)')
m = p.match(baseline)
if not m:
log.warning(f'Dataset processing_baseline does not match expected pattern: {baseline}')
return None
if int(m.group(1)) < 4:
return False
# If the boa_offset_applied has been applied then offset correction is not required
boa_offset_applied = props.get('earthsearch:boa_offset_applied', False)
return not boa_offset_applied
def apply_correction_to_data(ds: xr.Dataset, offset: float = 0) -> xr.Dataset:
"""Apply the scale and offset correction to each reflectance band where there is valid data (not nodata)"""
refl_vars = [x for x in ds.data_vars if x in refl_bands]
mask = masking.valid_data_mask(ds[refl_vars])
# Save on a dask step?
if offset == 0:
ds[refl_vars] = ds[refl_vars].where(mask) * scale_factor
else:
ds[refl_vars] = ds[refl_vars].where(mask) * scale_factor + offset
return ds
def load_s2l2a_with_offset(
dc: datacube.Datacube,
query: dict,
) -> xr.Dataset:
"""
Replaces datacube.load(**query) for s2_l2a products.
Method:
- Find all datasets matching the query (dc.find_datasets)
- Filter for the highest element84 processing sequence number per scene (scene label excluding the sequence number)
- Filter into two lists for datasets that have
- "s2:processing_baseline" >= "04.00" and "earthsearch:boa_offset_applied" == False (offset correction required)
- everything else (no correction required)
- If either list is empty then load the non-empty list, apply scale (and offset if required), and return the xarray Dataset
- Load and combine the two lists of datasets
- Load each list, apply scale (and offset if required)
- Concat on time dimension and sort by time
- Return the combined xarray Dataset
Notes:
- Any 'groupby' function is applied to each of the xarray Datasets prior to them being combined.
This could create "extra" (non-grouped) time layers in the combined Dataset if the groupby function
would have grouped datasets (scenes) from both lists.
- Scale and offset are applied to the reflectance bands where there is valid data (not `nodata`).
This includes applying the "scale_factor" even if no datasets require the offset correction.
Other masks can be applied by the user (e.g. pixel quality or cloud masking).
"""
product = query.get('product', '<all products>')
if product != 's2_l2a':
log.error(f'This function only applies to the "s2_l2a" product, not: {product}')
return None
# Find all datasets matching the query
matches = None
if 'datasets' in query:
matches = query['datasets']
del query['datasets']
if matches is None:
search_params = {k:v for k,v in query.items() if k in search_keys}
matches = dc.find_datasets(**search_params)
if 'skip_broken_datasets' not in query:
# This helps to avoid data loading error messages
query['skip_broken_datasets'] = True
# Filter for the highest element84 processing sequence number
sorter = highest_sequence_number(matches)
# Filter into two lists
offset_applied, offset_required = [], []
for key in sorter.keys():
ds = list(sorter[key].values())[0]
isrequired = ds_requires_offset(ds)
if isrequired is None:
continue
elif isrequired:
offset_required.append(ds)
else:
offset_applied.append(ds)
matches_combined = offset_applied + offset_required
# If either list is empty then no separation and merge is required
this_offset = None
if len(offset_applied) == 0:
log.info('All datasets require offset correction')
msg = 'The valid_data_mask, scale and offset have been applied to the reflectance bands'
this_offset = add_offset
if len(offset_required) == 0:
log.info('No datasets require offset correction')
msg = 'The valid_data_mask and scale (no offset) have been applied to the reflectance bands'
this_offset = 0
if this_offset is not None:
data = dc.load(
datasets = matches_combined,
**query
)
xx = apply_correction_to_data(data, this_offset)
log.info(msg)
return xx
# DEBUG: What do we have
# def func(s):
# p = re.compile('(\d{8})')
# m = p.search(s[0])
# if m:
# return m.group(1)
# log.info(f'Number of datasets in initial query: {len(matches)}')
# log.info(f'{sorted([(x.metadata_doc["label"],x.id) for x in matches], key=func)}')
# log.info(f'Number of datasets with offset applied: {len(offset_applied)}')
# log.info(f'{sorted([(x.metadata_doc["label"],x.id) for x in offset_applied], key=func)}')
# log.info(f'Number of datasets without offset applied: {len(offset_required)}')
# log.info(f'{sorted( [(x.metadata_doc["label"],x.id) for x in offset_required], key=func)}')
# return
# Else, load data into two Datasets
log.info('Mix of datasets found with either offset required or not.')
log.info('We will load two xarrays, apply offset where required, and merge into one xarray.')
# 1. Ensure the target geobox covers all datasets
target_geobox = output_geobox(
datasets = matches_combined,
**query,
)
# 2. Edit the query for our needs
# Ensure that dask time chunking = 1
dask_input = None
if 'dask_chunks' in query:
dask_input = query['dask_chunks'] # Save
if dask_input.get('time', 1) != 1:
query['dask_chunks'].update({'time': 1})
# Remove keys that are not compatible with 'like'
for x in ('output_crs', 'resolution', 'align'):
if x in query:
del query[x]
# 3. Load two xarrays
data_offset_applied = dc.load(
datasets = offset_applied,
like = target_geobox,
**query
)
data_offset_required = dc.load(
datasets = offset_required,
like = target_geobox,
**query
)
# 4. Apply respective scale and offsets
data_offset_applied = apply_correction_to_data(data_offset_applied)
data_offset_required = apply_correction_to_data(data_offset_required, add_offset)
# 5. Combine the two xarrays
combined = xr.concat([data_offset_applied, data_offset_required], dim='time')
combined = combined.sortby('time')
# 6. Reapply any time > 1 chunking
if dask_input is not None:
if dask_input.get('time', 1) != 1:
combined = combined.chunk(dask_input)
log.info('The valid_data_mask, scale and offset have been applied to the reflectance bands')
return combined
easi_tools/notebook_utils.py
+178
View File
@@ -0,0 +1,178 @@
#!python3
# A collection of utilities that can be used in Python notebooks.
#
# License: Apache 2.0
# Created for EASI Hub training notebooks, https://dev.azure.com/csiro-easi/easi-hub-public/_git/hub-notebooks
# Data tools
import numpy as np
import xarray as xr
import pandas as pd
import geopandas as gpd
import datacube
from datacube.utils import masking
from datetime import datetime
# hvPlot, Holoviews, Datashader and Bokeh
import hvplot.pandas
import hvplot.xarray
import panel as pn
import holoviews as hv
# hv.extension("bokeh", logo=False) # Its likely set from in the notebooks
# Jupyter Lab
from IPython.display import HTML
# Python
import sys, os, re
import logging
from pathlib import Path
from collections import Counter
import contextlib
# Dask
import dask
from dask.distributed import Client, LocalCluster
from dask_gateway import Gateway
# EASIDefaults
from . import EasiDefaults
# Set logger
logger = logging.getLogger(Path(__file__).stem)
logger.setLevel(logging.INFO)
if not len(logger.handlers):
logger.addHandler(logging.StreamHandler(sys.stdout))
def display_table(
df: pd.DataFrame,
panel: bool = False,
):
"""Display the full pandas dataframe. If panel is True use a panel object"""
table = None
if panel:
# Dicts are rendered as "[object Object]". Need to set a formatter, I guess.
table = pn.widgets.DataFrame(df,
# sizing_mode='stretch_width', # equal column widths, full screen
autosize_mode='fit_viewport', # fitted columns, about 90-95% width
# reorderable=True, # didn't work first try
)
else:
with pd.option_context("display.max_rows", None,
"display.max_columns", None,
"display.max_colwidth", -1):
table = HTML( df.to_html().replace(r"\n", "<br>") )
display(table)
def heading(txt: str):
"""Print a simple HTML heading"""
display(HTML( f"<h4>{txt}</h4>" ))
def hv_table_hook(plot, element):
"""Selected options for hv.table() formatting
Use: df.hv.table().opts(hooks=[hv_table_hook])
"""
plot.handles["table"].autosize_mode="fit_viewport"
# Other examples
# plot.handles['table'].row_height = 40
# from bokeh.models.widgets import DateFormatter
# plot.handles['table'].columns[6].formatter = DateFormatter(format='%Y-%m-%d')
def xarray_object_size(data):
"""Return a formatted string"""
val, unit = data.nbytes / (1024 ** 2), "MB"
if val > 1024:
val, unit = data.nbytes / (1024 ** 3), "GB"
return f"Dataset size: {val:.2f} {unit}"
def mostcommon_crs(dc, query):
"""Adapted from https://github.com/GeoscienceAustralia/dea-notebooks/blob/develop/Tools/dea_tools/datahandling.py"""
matching_datasets = dc.find_datasets(**query)
crs_list = [str(i.crs) for i in matching_datasets]
crs_mostcommon = None
if len(crs_list) > 0:
# Identify most common CRS
crs_counts = Counter(crs_list)
crs_mostcommon = crs_counts.most_common(1)[0][0]
else:
logger.warning("No data was found for the supplied product query")
return crs_mostcommon
def initialize_dask(use_gateway=False, workers=(1,2), wait=False, local_port=8786, **kwargs):
"""Initialize a Dask Gateway or Local cluster"""
# Check inputs
if isinstance(workers, (int, float)):
workers = (int(workers), int(workers))
if len(workers) != 2:
logger.error("Require workers to be a single integer or a 2-element tuple/list")
return None, None
if isinstance(local_port, (str, float)):
local_port = int(local_port)
# Dask gateway
if use_gateway:
gateway = Gateway()
clusters = gateway.list_clusters()
if not clusters:
logger.info("Starting new cluster")
cluster = gateway.new_cluster(**kwargs)
else:
logger.info(f"An existing cluster was found. Connecting to: {clusters[0].name}")
cluster = gateway.connect(clusters[0].name)
client = cluster.get_client()
cluster.adapt(minimum=workers[0], maximum=workers[1])
if wait:
logger.info("Waiting for at least one cluster worker")
# client.wait_for_workers(n_workers=1) # Before release 2023.10.0
client.sync(client._wait_for_workers,n_workers=1) # Since release 2023.10.0
# Local cluster
else:
cluster = LocalCluster(n_workers=max(workers))
client = Client(cluster)
# Try to set custom dashboard link for JupyterHub (only works on EASI Hub)
try:
server = f'https://hub.{EasiDefaults().domain}'
user = os.environ.get('JUPYTERHUB_SERVICE_PREFIX')
if user: # Only set if running in JupyterHub
dask.config.set({"distributed.dashboard.link": f'{server}{user}' + "proxy/{port}/status"})
except (KeyError, Exception):
# Not running on EASI Hub - LocalCluster will use default dashboard link
pass
return cluster, client
def localcluster_dashboard(client, server="https://hub.csiro.easi-eo.solutions"):
"""Return a dashboard link using jupyter proxy"""
dashboard_link = client.dashboard_link
for host in ("127.0.0.1", "localhost"):
if host in dashboard_link:
port = re.search(r":(\d+)\/status", dashboard_link).group(1)
dashboard_link = f'{server}{os.environ["JUPYTERHUB_SERVICE_PREFIX"]}proxy/{port}/status'
break
return dashboard_link
@contextlib.contextmanager
def unset_cachingproxy():
"""Unset the EASI caching proxy with a context manager"""
# Inspired by https://stackoverflow.com/a/34333710
env = os.environ
remove = ("AWS_HTTPS", "GDAL_HTTP_PROXY")
update_after = {k: env[k] for k in remove}
try:
[env.pop(k, None) for k in remove]
yield
finally:
env.update(update_after)
load_data_no_odc.py
+306
View File
@@ -0,0 +1,306 @@
"""
Load Sentinel data without ODC Database
Sử dụng Microsoft Planetary Computer STAC API
Usage:
from load_data_no_odc import load_sentinel2_stac, load_sentinel1_stac
"""
import xarray as xr
import numpy as np
import pandas as pd
from pystac_client import Client
import planetary_computer
import odc.stac
from datetime import datetime
def load_sentinel2_stac(
bbox, # (lon_min, lat_min, lon_max, lat_max)
date_range, # ("2022-09-01", "2023-10-01")
bands=None,
resolution=10,
chunks={'time': 1, 'x': 2048, 'y': 2048}
):
"""
Load Sentinel-2 L2A data from Microsoft Planetary Computer
Args:
bbox: Bounding box (lon_min, lat_min, lon_max, lat_max)
date_range: Tuple of start and end dates
bands: List of bands to load (default: ['red', 'nir', 'scl'])
resolution: Spatial resolution in meters
chunks: Dask chunk sizes
Returns:
xarray.Dataset with Sentinel-2 data
"""
if bands is None:
bands = ['red', 'nir', 'blue', 'green', 'nir08', 'swir16', 'swir22', 'SCL']
print(f"🔍 Searching Sentinel-2 data...")
print(f" Bbox: {bbox}")
print(f" Date: {date_range[0]} to {date_range[1]}")
print(f" Bands: {bands}")
# Connect to Planetary Computer STAC
catalog = Client.open(
"https://planetarycomputer.microsoft.com/api/stac/v1",
modifier=planetary_computer.sign_inplace,
)
# Search for Sentinel-2 items
search = catalog.search(
collections=["sentinel-2-l2a"],
bbox=bbox,
datetime=f"{date_range[0]}/{date_range[1]}",
)
items = list(search.items())
print(f"✅ Found {len(items)} Sentinel-2 scenes")
if len(items) == 0:
print("⚠ No data found for the given parameters")
return None
# Load data using odc-stac
print("📥 Loading data...")
try:
data = odc.stac.load(
items,
bands=bands,
bbox=bbox,
resolution=resolution,
chunks=chunks,
groupby="solar_day",
)
print(f"✅ Loaded Sentinel-2 data: {data.dims}")
print(f" Shape: {dict(data.dims)}")
print(f" Bands: {list(data.data_vars)}")
return data
except Exception as e:
print(f"❌ Error loading data: {e}")
import traceback
traceback.print_exc()
return None
def load_sentinel1_stac(
bbox, # (lon_min, lat_min, lon_max, lat_max)
date_range, # ("2022-09-01", "2023-10-01")
bands=None,
resolution=10,
chunks={'time': 1, 'x': 2048, 'y': 2048}
):
"""
Load Sentinel-1 RTC data from Microsoft Planetary Computer
Args:
bbox: Bounding box (lon_min, lat_min, lon_max, lat_max)
date_range: Tuple of start and end dates
bands: List of bands to load (default: ['vv', 'vh'])
resolution: Spatial resolution in meters
chunks: Dask chunk sizes
Returns:
xarray.Dataset with Sentinel-1 data
"""
if bands is None:
bands = ['vv', 'vh']
print(f"🔍 Searching Sentinel-1 data...")
print(f" Bbox: {bbox}")
print(f" Date: {date_range[0]} to {date_range[1]}")
print(f" Bands: {bands}")
# Connect to Planetary Computer STAC
catalog = Client.open(
"https://planetarycomputer.microsoft.com/api/stac/v1",
modifier=planetary_computer.sign_inplace,
)
# Search for Sentinel-1 RTC items
search = catalog.search(
collections=["sentinel-1-rtc"],
bbox=bbox,
datetime=f"{date_range[0]}/{date_range[1]}",
)
items = list(search.items())
print(f"✅ Found {len(items)} Sentinel-1 scenes")
if len(items) == 0:
print("⚠ No data found for the given parameters")
return None
# Load data using odc-stac
print("📥 Loading data...")
try:
data = odc.stac.load(
items,
bands=bands,
bbox=bbox,
resolution=resolution,
chunks=chunks,
groupby="solar_day",
)
# Rename bands to uppercase for consistency
if 'vv' in data.data_vars:
data = data.rename({'vv': 'VV', 'vh': 'VH'})
print(f"✅ Loaded Sentinel-1 data: {data.dims}")
print(f" Shape: {dict(data.dims)}")
print(f" Bands: {list(data.data_vars)}")
return data
except Exception as e:
print(f"❌ Error loading data: {e}")
import traceback
traceback.print_exc()
return None
def mask_clean_s2(data, scl_band='SCL'):
"""
Apply cloud mask to Sentinel-2 data using SCL band
Good pixels: 2=dark, 4=vegetation, 5=not-vegetated, 6=water
"""
print("🧹 Applying cloud mask...")
if scl_band not in data.data_vars:
print(f"⚠ Warning: {scl_band} band not found, skipping cloud mask")
return data
# Good pixel values
good_pixels = [2, 4, 5, 6]
# Create mask
mask = xr.zeros_like(data[scl_band], dtype=bool)
for pixel_val in good_pixels:
mask = mask | (data[scl_band] == pixel_val)
# Apply mask to all bands except SCL
data_vars = [v for v in data.data_vars if v != scl_band]
result = data[data_vars].where(mask)
print(f"✅ Cloud mask applied")
return result
def calculate_ndvi(data, nir_band='nir08', red_band='red'):
"""
Calculate NDVI from Sentinel-2 data
"""
print(f"📊 Calculating NDVI using {nir_band} and {red_band}...")
if nir_band not in data.data_vars or red_band not in data.data_vars:
print(f"⚠ Warning: Required bands not found")
print(f" Available bands: {list(data.data_vars)}")
return data
nir = data[nir_band]
red = data[red_band]
ndvi = (nir - red) / (nir + red)
ndvi = ndvi.rename('NDVI')
# Add NDVI to dataset
result = xr.merge([data, ndvi])
print(f"✅ NDVI calculated")
return result
def fill_nan_temporal(data, dim='time'):
"""
Fill NaN values using forward/backward fill along time dimension
"""
print(f"🔧 Filling NaN values using temporal interpolation...")
filled = data.bfill(dim=dim).ffill(dim=dim)
print(f"✅ NaN values filled")
return filled
def resample_monthly(data, method='mean'):
"""
Resample data to monthly frequency
"""
print(f"📅 Resampling to monthly using {method}...")
if method == 'mean':
result = data.resample(time='1MS').mean()
elif method == 'median':
result = data.resample(time='1MS').median()
else:
raise ValueError(f"Unknown method: {method}")
print(f"✅ Resampled to monthly: {result.dims}")
return result
# ══════════════════════════════════════════════════════════════════════════════
# CONVENIENCE WRAPPER
# ══════════════════════════════════════════════════════════════════════════════
def load_and_process_s2(bbox, date_range, apply_cloud_mask=True, calculate_indices=True):
"""
Load and process Sentinel-2 data in one step
"""
# Load data
data = load_sentinel2_stac(bbox, date_range)
if data is None:
return None
# Apply cloud mask
if apply_cloud_mask:
data = mask_clean_s2(data)
# Calculate NDVI
if calculate_indices:
data = calculate_ndvi(data)
# Fill NaN
data = fill_nan_temporal(data)
# Resample to monthly
data_monthly = resample_monthly(data)
return data_monthly
def load_and_process_s1(bbox, date_range):
"""
Load and process Sentinel-1 data in one step
"""
# Load data
data = load_sentinel1_stac(bbox, date_range)
if data is None:
return None
# Resample to monthly
data_monthly = resample_monthly(data)
return data_monthly
# Print module info
print("=" * 70)
print("📦 Data Loading Module (No ODC Database Required)")
print("=" * 70)
print("\n💡 Usage:")
print(" from load_data_no_odc import load_and_process_s2, load_and_process_s1")
print("\n bbox = (lon_min, lat_min, lon_max, lat_max)")
print(" date_range = ('2022-09-01', '2023-10-01')")
print("\n data_s2 = load_and_process_s2(bbox, date_range)")
print(" data_s1 = load_and_process_s1(bbox, date_range)")
print("=" * 70)
print()
new_import_ODC_cognito.py
+514
View File
@@ -0,0 +1,514 @@
"""
ODC Import Module with Cognito Authentication
Module ODC tích hợp xác thực Cognito
Usage:
import new_import_ODC_cognito
from new_import_ODC_cognito import *
# Setup Cognito authentication
setup_cognito_auth('train_files/crediential.txt')
# Then use datacube normally
"""
import matplotlib.pyplot as plt
# Common imports and settings
import os, sys
os.environ['USE_PYGEOS'] = '0'
from IPython.display import Markdown
import pandas as pd
pd.set_option("display.max_rows", None)
import xarray as xr
# Datacube
import datacube
from datacube.utils.rio import configure_s3_access
from datacube.utils import masking
from datacube.utils.cog import write_cog
# DEA Tools
from dea_tools.plotting import display_map, rgb
from dea_tools.datahandling import mostcommon_crs
# EASI defaults - Update path to local installation
easi_tools_path = '/media/x79/2A7D-FAA0/remote-sensing'
if easi_tools_path not in sys.path:
sys.path.insert(0, easi_tools_path)
# Try to import EASI tools
EASI_AVAILABLE = False
notebook_utils = None
load_s2l2a_with_offset = None
try:
# Check if dask_gateway is available first
try:
import dask_gateway
dask_gateway_available = True
except ImportError:
dask_gateway_available = False
print("⚠ dask_gateway not installed - will use LocalCluster instead")
# Import EASI tools
if dask_gateway_available:
from easi_tools import notebook_utils
from easi_tools.load_s2l2a import load_s2l2a_with_offset
EASI_AVAILABLE = True
print("✅ EASI tools loaded successfully (with Gateway support)")
else:
# Import what we can without dask_gateway
print("⚠ Loading EASI tools without Gateway support...")
# Don't import notebook_utils if it requires dask_gateway
from easi_tools.load_s2l2a import load_s2l2a_with_offset
print("✅ EASI load_s2l2a loaded (without notebook_utils)")
except ImportError as e:
print(f"⚠ EASI tools not available: {e}")
print("⚠ Using standard datacube functions")
EASI_AVAILABLE = False
# Create fallback notebook_utils if not available
if notebook_utils is None:
class FallbackNotebookUtils:
"""Fallback implementation when EASI tools not available"""
@staticmethod
def initialize_dask(use_gateway=False, workers=(1, 10)):
"""Initialize Dask cluster"""
from dask.distributed import Client, LocalCluster
if use_gateway:
print("⚠ Dask Gateway not available, using LocalCluster")
n_workers = workers[0] if isinstance(workers, tuple) else workers
cluster = LocalCluster(
n_workers=n_workers,
threads_per_worker=1,
memory_limit='4GB'
)
client = Client(cluster)
print(f"✅ Dask LocalCluster started")
print(f" Workers: {n_workers}")
print(f" Dashboard: {client.dashboard_link}")
return cluster, client
@staticmethod
def mostcommon_crs(dc, query):
"""Get most common CRS - fallback to Vietnam default"""
try:
# Try to get CRS from datacube
datasets = list(dc.find_datasets(**query))
if datasets and len(datasets) > 0:
return str(datasets[0].crs)
except Exception as e:
print(f"⚠ Could not determine CRS from datacube: {e}")
# Default CRS for Vietnam
print(" Using default CRS: EPSG:32648 (Vietnam)")
return 'EPSG:32648'
notebook_utils = FallbackNotebookUtils()
print("✅ Fallback notebook_utils created")
from dask.distributed import progress
# Data tools
import numpy as np
from datetime import datetime
# ODC algo
from odc.algo import enum_to_bool
from odc.algo import xr_reproject
from datacube.utils.geometry import GeoBox, box
# Holoviews, Datashader and Bokeh
import hvplot.pandas
import hvplot.xarray
import holoviews as hv
import panel as pn
import colorcet as cc
import cartopy.crs as ccrs
from datashader import reductions
from holoviews import opts
hv.extension('bokeh', logo=False)
# ML and Geo tools
from deafrica_tools.bandindices import calculate_indices
from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.metrics import accuracy_score, classification_report, mean_squared_error, r2_score
from sklearn.preprocessing import LabelEncoder, StandardScaler, PolynomialFeatures
from sklearn.pipeline import Pipeline
from sklearn.impute import SimpleImputer
from sklearn.linear_model import LinearRegression
from shapely.geometry import Point, Polygon
import geopandas as gpd
from pyproj import CRS
from matplotlib.colors import ListedColormap
from bokeh.models.tickers import FixedTicker
from rioxarray.merge import merge_arrays
import rasterio
import rioxarray
import joblib
# Import utils
try:
from utils import load_data_geo
except ImportError:
print("⚠ utils.py not found, load_data_geo() may not work")
# ══════════════════════════════════════════════════════════════════════════════
# COGNITO AUTHENTICATION
# ══════════════════════════════════════════════════════════════════════════════
# Global authenticator instance
_cognito_auth = None
def setup_cognito_auth(credential_file='train_files/crediential.txt', region='ap-southeast-1'):
"""
Setup Cognito authentication for S3/ODC access
Thiết lập xác thực Cognito cho truy cập S3/ODC
Args:
credential_file: Path to credential file containing AWS + Cognito tokens
region: AWS region
Returns:
CognitoAuthenticator instance
"""
global _cognito_auth
try:
# Import cognito_auth module
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from cognito_auth import CognitoAuthenticator
print("🔐 Setting up Cognito authentication...")
# Initialize authenticator
_cognito_auth = CognitoAuthenticator(region=region)
# Load tokens and credentials
if not _cognito_auth.load_tokens_from_file(credential_file):
print("✗ Failed to load Cognito tokens")
return None
# Display token info
print("\n📋 Token Information:")
decoded_id, _ = _cognito_auth.print_token_info()
# Get AWS credentials
if not _cognito_auth.get_credentials_from_cognito():
print("✗ Failed to get AWS credentials")
return None
# Set environment credentials
if not _cognito_auth.set_environment_credentials():
print("✗ Failed to set environment credentials")
return None
# Configure S3 access for datacube
print("\n🌐 Configuring datacube S3 access...")
configure_s3_access(
aws_unsigned=False, # Using credentials
region_name=region,
cloud_defaults=True
)
print("\n✅ Cognito authentication setup complete!")
print("✅ Ready to use datacube with S3 access\n")
return _cognito_auth
except ImportError as e:
print(f"✗ Error: cognito_auth module not found: {e}")
print(" Make sure cognito_auth.py is in the same directory")
return None
except Exception as e:
print(f"✗ Error setting up Cognito auth: {e}")
import traceback
traceback.print_exc()
return None
def get_cognito_auth():
"""
Get the current Cognito authenticator instance
Lấy instance Cognito authenticator hiện tại
"""
return _cognito_auth
# ══════════════════════════════════════════════════════════════════════════════
# DATA LOADING FUNCTIONS
# ══════════════════════════════════════════════════════════════════════════════
def load_data(dc, date_range, longtitude_range, latitude_range, measurements=None):
"""
Load Sentinel-2 L2A data from datacube
"""
product = 's2_l2a'
query = {
'product': product,
'x': longtitude_range,
'y': latitude_range,
'time': date_range,
}
if EASI_AVAILABLE:
native_crs = notebook_utils.mostcommon_crs(dc, query)
else:
native_crs = 'EPSG:32648' # Default for Vietnam
print(f'Most common native CRS: {native_crs}')
if measurements is None:
measurements = ['red', 'nir', 'scl']
load_params = {
'measurements': measurements,
'output_crs': native_crs,
'resolution': (-10, 10),
'group_by': 'solar_day',
'dask_chunks': {'x': 2048, 'y': 2048},
}
if EASI_AVAILABLE:
data = load_s2l2a_with_offset(dc, query | load_params)
else:
data = dc.load(**{**query, **load_params})
return data
def load_data_sen1(dc, date_range, longtitude_range, latitude_range):
"""
Load Sentinel-1 SAR data (VV, VH bands)
"""
product = 's1_rtc'
query = {
'product': product,
'x': longtitude_range,
'y': latitude_range,
'time': date_range,
'measurements': ['VV', 'VH'],
'output_crs': 'EPSG:32648',
'resolution': (-10, 10),
'group_by': 'solar_day',
'dask_chunks': {'x': 2048, 'y': 2048},
}
data = dc.load(**query)
return data
def mask_clean(data):
"""
Apply cloud mask to Sentinel-2 data using SCL band
"""
flag_name = 'scl'
flag_desc = masking.describe_variable_flags(data[flag_name])
display(flag_desc)
display(flag_desc.loc['qa'].values[1])
# Good pixel flags: 2=dark, 4=vegetation, 5=not-vegetated, 6=water
flags_def = flag_desc.loc['qa'].values[1]
good_pixel_flags = [flags_def[str(i)] for i in [2, 4, 5, 6]]
good_pixel_mask = enum_to_bool(data[flag_name], good_pixel_flags)
data_layer_names = [x for x in data.data_vars if x != 'scl']
result = data[data_layer_names].where(good_pixel_mask).persist()
return result
def fill_nan(ndvi, time_split=None):
"""
Fill NaN values in NDVI using forward/backward fill
"""
if time_split is None:
# Simple fill without time splits
fill_m = ndvi.bfill(dim='time').ffill(dim='time')
return fill_m
# Fill with time splits
rs = []
for times in time_split:
tmp = ndvi.sel(time=times)
fill_ds = tmp.bfill(dim='time').ffill(dim='time')
rs.append(fill_ds)
merged_ndvi = xr.concat(rs, dim="time")
fill_m = merged_ndvi.bfill(dim="time").ffill(dim="time")
return fill_m
def calculate_average(data, variables, resample='1MS'):
"""
Calculate temporal average and resample
"""
result = data[variables].resample(time=resample).mean()
return result
def load_train_data(train_path=None, label_mapping=None):
"""
Load training data from shapefile or GeoJSON
"""
if train_path is None:
print("⚠ No train_path provided")
return None
train = load_data_geo(train_path)
if label_mapping is not None:
# Apply label mapping
train['label_id'] = train['label'].map(label_mapping).astype(int)
return train
def get_data_sen1_and_sen2(train_data, data_sen2, data_sen1):
"""
Extract Sentinel-1 and Sentinel-2 data for training points
"""
X_list = []
y_list = []
for idx, point in train_data.iterrows():
try:
lon, lat = point.geometry.x, point.geometry.y
# Extract S2 data
s2_values = data_sen2.sel(x=lon, y=lat, method='nearest').values.flatten()
# Extract S1 data
s1_values = data_sen1.sel(x=lon, y=lat, method='nearest').values.flatten()
# Combine features
features = np.concatenate([s2_values, s1_values])
# Skip if contains NaN
if not np.isnan(features).any():
X_list.append(features)
y_list.append(point['label_id'])
except Exception as e:
print(f"⚠ Skip point {idx}: {e}")
continue
X = np.array(X_list)
y = np.array(y_list)
print(f"✅ Extracted {len(X)} training samples")
print(f" Features: {X.shape[1]}")
print(f" Classes: {sorted(set(y.tolist()))}")
return X, y
def split_train_data(X, y, test_size=0.2, val_size=0.1, random_state=42):
"""
Split data into train/val/test sets
"""
# First split: train+val vs test
X_temp, X_test, y_temp, y_test = train_test_split(
X, y, test_size=test_size, random_state=random_state, stratify=y
)
# Second split: train vs val
val_ratio = val_size / (1 - test_size)
X_train, X_val, y_train, y_val = train_test_split(
X_temp, y_temp, test_size=val_ratio, random_state=random_state, stratify=y_temp
)
print(f"✅ Data split:")
print(f" Train: {len(X_train)} samples")
print(f" Val: {len(X_val)} samples")
print(f" Test: {len(X_test)} samples")
return X_train, X_val, X_test, y_train, y_val, y_test
# ══════════════════════════════════════════════════════════════════════════════
# HELPER FUNCTIONS
# ══════════════════════════════════════════════════════════════════════════════
def load_sen1(name_vh, name_vv):
"""Load Sentinel-1 from local files"""
dsvv = rioxarray.open_rasterio(name_vv)
dsvh = rioxarray.open_rasterio(name_vh)
return dsvh, dsvv
def print_auth_status():
"""Print current authentication status"""
global _cognito_auth
print("=" * 60)
print("AUTHENTICATION STATUS")
print("=" * 60)
if _cognito_auth:
print("✅ Cognito authentication is active")
# Check credentials
if _cognito_auth.aws_credentials:
print("✅ AWS credentials loaded")
print(f" Access Key: {_cognito_auth.aws_credentials['AccessKeyId'][:20]}...")
else:
print("⚠ No AWS credentials")
# Check tokens
if _cognito_auth.id_token:
print("✅ Cognito tokens loaded")
decoded = _cognito_auth.decode_token(_cognito_auth.id_token)
if decoded:
print(f" User: {decoded.get('cognito:username', 'N/A')}")
print(f" Email: {decoded.get('email', 'N/A')}")
else:
print("⚠ No Cognito tokens")
else:
print("⚠ Cognito authentication not setup")
print(" Run: setup_cognito_auth('train_files/crediential.txt')")
print("=" * 60)
# ══════════════════════════════════════════════════════════════════════════════
# AUTO SETUP
# ══════════════════════════════════════════════════════════════════════════════
def auto_setup(credential_file='train_files/crediential.txt'):
"""
Automatically setup Cognito authentication if credential file exists
"""
if os.path.exists(credential_file):
print(f"🔍 Found credential file: {credential_file}")
print("🚀 Auto-setting up Cognito authentication...\n")
return setup_cognito_auth(credential_file)
else:
print(f"⚠ Credential file not found: {credential_file}")
print(" Using default S3 configuration (unsigned)")
configure_s3_access(aws_unsigned=True)
return None
# Print module info on import
print("=" * 70)
print("📦 ODC Module with Cognito Authentication Loaded")
print("=" * 70)
print("\n💡 Quick Start:")
print(" 1. setup_cognito_auth('train_files/crediential.txt')")
print(" 2. Use datacube normally with authenticated S3 access")
print("\n📚 Functions:")
print(" - setup_cognito_auth() : Setup Cognito authentication")
print(" - get_cognito_auth() : Get authenticator instance")
print(" - print_auth_status() : Show auth status")
print(" - auto_setup() : Auto-setup if credentials exist")
print("=" * 70)
print()
notebooks/01-welcome-to-easi.ipynb
+625
View File
@@ -0,0 +1,625 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {
"tags": []
},
"source": [
"# Welcome to EASI <img align=\"right\" src=\"../resources/csiro_easi_logo.png\">\n",
"\n",
"This notebook introduces new users to working with EASI notebooks and the Open Data Cube (ODC).\n",
"\n",
"It will demonstrate the following basic functionality:\n",
"- [Notebook setup](#Notebook-setup)\n",
"- [Select an EASI environment](#Select-an-EASI-environment)\n",
"- [Connect to the OpenDataCube](#Connect-to-the-OpenDataCube)\n",
" - [List products](#List-products)\n",
" - [List measurements and attributes](#List-measurements-and-attributes)\n",
" - [Choose an area of interest](#Choose-an-area-of-interest)\n",
" - [Load data](#Load-data)\n",
" - [Plot the data](#Plot-the-data)\n",
" - [Masking and scaling](#Masking-and-scaling)\n",
" - [Perform a calculation on the data](#Perform-a-calculation-on-the-data)\n",
" - [Save the results to file](#Save-the-results-to-file)\n",
"- [Summary](#Summary)\n",
"- [Be a good cloud citizen](#Be-a-good-cloud-citizen)\n",
"- [Further reading](#Further-reading)"
]
},
{
"cell_type": "markdown",
"metadata": {
"tags": []
},
"source": [
"## Notebook setup\n",
"\n",
"A notebook consists of cells that contain either text descriptions or python code for performing operations on data.\n",
"\n",
"1. Start by clicking on the cell below to select it.\n",
"1. Execute a selected cell, or each cell in sequence, by clicking the &#9654; button (in the notebook toolbar above) or pressing `Shift`+`Enter`.\n",
"1. Each cell will show an asterisk icon <font color='#999'>[*]:</font> when it is running. Once this changes to a number, the cell has finished.\n",
"1. The cell below imports packages to use and sets some formatting options."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Common imports and settings\n",
"import os, sys, re\n",
"from pathlib import Path\n",
"from IPython.display import Markdown\n",
"import pandas as pd\n",
"pd.set_option(\"display.max_rows\", None)\n",
"import xarray as xr\n",
"\n",
"# Datacube\n",
"import datacube\n",
"from datacube.utils.aws import configure_s3_access\n",
"import odc.geo.xr\n",
"from datacube.utils import masking\n",
"# https://github.com/GeoscienceAustralia/dea-notebooks/tree/develop/Tools\n",
"from dea_tools.plotting import display_map, rgb\n",
"\n",
"# Basic plots\n",
"%matplotlib inline\n",
"# import matplotlib.pyplot as plt\n",
"# plt.rcParams['figure.figsize'] = [12, 8]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### EASI defaults\n",
"\n",
"Each EASI deployment has a different set of products in its opendatacube database. We introduce a set of defaults to allow these training notebooks to be used between EASI deployments.\n",
"\n",
"There are also some convenience or helper functions that we use in most notebooks. The cell below can be copied and adapted into your own noteboks."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# EASI defaults\n",
"# These are convenience functions so that the notebooks in this repository work in all EASI deployments\n",
"\n",
"# The `git.Repo()` part returns the local directory that easi-notebooks has been cloned into\n",
"# If using the `easi-tools` functions from another path, replace `repo` with your local path to `easi-notebooks` directory\n",
"try:\n",
" import git\n",
" repo = git.Repo('.', search_parent_directories=True).working_tree_dir # Path to this cloned local directory\n",
"except (ImportError, git.InvalidGitRepositoryError):\n",
" repo = Path.home() / 'easi-notebooks' # Reasonable default\n",
" if not repo.is_dir():\n",
" raise RuntimeError('To use `easi-tools` please provide the local path to `https://github.com/csiro-easi/easi-notebooks`')\n",
"if repo not in sys.path:\n",
" sys.path.append(str(repo)) # Add the local path to `easi-notebooks` to python\n",
"\n",
"from easi_tools import EasiDefaults\n",
"from easi_tools import initialize_dask, xarray_object_size"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"For this notebook we select the default **Landsat** product."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"easi = EasiDefaults()\n",
"\n",
"family = 'landsat'\n",
"product = easi.product(family)\n",
"display(Markdown(f'Default {family} product for \"{easi.name}\": [{product}]({easi.explorer}/products/{product})'))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Connect to the OpenDataCube\n",
"\n",
"The `Datacube()` API provides search, load and information functions for data products *indexed* in an ODC database. More information on the Open Data Cube software:\n",
"\n",
"- https://datacube-core.readthedocs.io/en/latest/\n",
"- https://github.com/opendatacube/datacube-core"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"dc = datacube.Datacube()\n",
"\n",
"# Access AWS \"requester-pays\" buckets\n",
"# This is necessary for reading data from most third-party AWS S3 buckets such as for Landsat and Sentinel-2\n",
"configure_s3_access(aws_unsigned=False, requester_pays=True);"
]
},
{
"cell_type": "markdown",
"metadata": {
"tags": []
},
"source": [
"### List products\n",
"Show all available products in the ODC database and list them along with selected properties.\n",
"\n",
"The **ODC Explorer** also has this information and more: view available products, data coverage, product definitions, dimensions, metadata and paths to the files.\n",
"\n",
"The product definitions include details about the *measurements* (or bands) in each product and, usually, the spatial resolution and CRS (if common to all member *datasets*).\n",
"\n",
"> **Exercise**: Browse the ODC Explorer link and find the information described above."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"display(Markdown(f'#### ODC Explorer: {easi.explorer}'))\n",
"\n",
"products = dc.list_products() # Pandas DataFrame\n",
"products"
]
},
{
"cell_type": "markdown",
"metadata": {
"tags": []
},
"source": [
"### List measurements and attributes\n",
"\n",
"The data arrays for each product are called **measurements**. In different data science domains these might also be called the \"bands\", \"variables\" or \"parameters\" of a product.\n",
"\n",
"List the measurements of a product. The columns are selected attributes or metadata for each measurement.\n",
"\n",
"> **Hint**: Measurements often have **aliases** defined. Any of the available alias names can be used in place of the measurement name when loading (reading) data. The *xarray* variable name will then be the alias name. Use this feature to help make your loaded data more consistent between products. We do this below."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"measurements = dc.list_measurements() # Pandas DataFrame, all products\n",
"measurements.loc[[product]]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Choose an area of interest\n",
"\n",
"Choose an area of interest with `latitude`/`longitude` bounds. The `display_map` function will draw a map with the bounding box highlighted. See also the ODC Explorer website for the available *latitude*, *longitude* and *time* ranges for each product.\n",
"\n",
"> **Exercise**: Feel free to change the `latitude`/`longitude` or `time` ranges of the query below. "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Default area of interest\n",
"\n",
"display(Markdown(f\"#### Location: {easi.location}\"))\n",
"display(Markdown(f\"See: {easi.explorer}/products/{product}\"))\n",
"\n",
"latitude = easi.latitude\n",
"longitude = easi.longitude\n",
"\n",
"# Or set your own latitude / longitude\n",
"# latitude = (-36.3, -35.8)\n",
"# longitude = (146.8, 147.3)\n",
"\n",
"display_map(longitude, latitude)"
]
},
{
"cell_type": "markdown",
"metadata": {
"tags": []
},
"source": [
"### Load data \n",
"Here we load product data for a given latitude, longitude and time range. The `datacube.load()` function returns an **xarray.Dataset** object.\n",
"\n",
"Once you have an xarray object this can be used with many Python packages. Further information on **xarray**:\n",
"\n",
"- https://tutorial.xarray.dev/overview/xarray-in-45-min.html\n",
"- https://xarray.pydata.org/en/stable/user-guide/data-structures.html\n",
"\n",
"**What is the size of my dataset?**\n",
"\n",
"The `display(data)` view is a convenient way to check the data request size, shape and attributes.\n",
"\n",
"> **Exercise**: Click the various arrows and icons in the *xarray.Dataset* output from the previous cell to reveal information about your data.\n",
"\n",
"The `data.nbytes` property returns the number of bytes in the xarray Dataset of DataArray. We have a function that formats this value for convenience.\n",
"\n",
"**Datacube.load() notes**\n",
"\n",
"- Use `measurements=[measurement or alias names]` to only load the measurements you will use, and label them accordingly.\n",
"- The `output_crs` and `resolution` parameters allow for remapping to a new grid. These will be required if default values are not defined for the product (see *measurement* attributes).\n",
"- The `datacube.load()` function does not apply missing values or scaling attributes. These are left to the user's discretion and requirements.\n",
"- `dask_chunks` will return a **dask** array. See the [EASI tutorial dask notebooks](dask/01_-_Introduction_to_Dask.ipynb) for information and examples.\n",
"\n",
"> **Exercise**: The default is to load all available measurements. Load a selected set of measurements or alias names and consider the result.<br>\n",
"> **Exercise**: The default is to load the data arrays onto a default grid that closely matches the source data. Change the target resolution or CRS and consider the result."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# A standard datacube.load() call.\n",
"# This may take a few minutes while the data are loaded into JupyterLab (so choose a small area and time range).\n",
"# Using a dask cluster will make this step seem quicker (see other notebooks for examples).\n",
"\n",
"target_crs = easi.crs(family) # If defined, else None\n",
"target_res = easi.resolution(family) # If defined, else None\n",
"\n",
"data = dc.load(\n",
" product = product,\n",
" latitude = latitude,\n",
" longitude = longitude,\n",
" time = easi.time,\n",
" measurements = ['red', 'green', 'blue', 'nir'], # List of selected measurement names or aliases\n",
"\n",
" output_crs = target_crs, # Target CRS\n",
" resolution = target_res, # Target resolution\n",
" # dask_chunks = {'x':2048, 'y':2048), # Dask chunk size. Requires a dask cluster (see the \"dask\" notebooks)\n",
" group_by = 'solar_day', # Group by day method\n",
")\n",
"\n",
"display(data)\n",
"\n",
"display(f'Number of bytes: {data.nbytes}')\n",
"display(xarray_object_size(data))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Plot the data\n",
"Plot the measurement data for a set of timesteps. The `xarray.plot()` function can simplify the rendering of plots by using the labelled dimensions and data ranges automatically. \n",
"\n",
"See the [EASI tutorial visualisation notebook](03-visualisation.ipynb) for information and examples.\n",
"\n",
"- The [robust](https://docs.xarray.dev/en/stable/user-guide/plotting.html#robust) option excludes outliers when calculating the colour limts for a more consistent result across subplots (time layers in this case).\n",
"\n",
"> **Exercise**: Change the data variable and perhaps the selection of time layers."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Select a data variable (measurement) name\n",
"band = 'nir'\n",
"\n",
"# Xarray simple array plotting\n",
"display(Markdown(f'#### Measurement: {band}'))\n",
"data[band].plot(col=\"time\", robust=True, col_wrap=4);"
]
},
{
"cell_type": "markdown",
"metadata": {
"tags": []
},
"source": [
"### Masking and scaling\n",
"\n",
"Most data products include a no-data value and/or a *data quality* array that can be used to mask (filter) the measurement arrays. For example, remote sensing quality arrays often include a \"cloud\" confidence flag that can be used to remove pixels affected by clouds from further analysis. Measurement arrays can also include *scale and offset factors* to transform the array values to scientific values.\n",
"\n",
"This step is common to most data analysis problems so we encourage users to find and understand the relevant quality, scale and offset metadata for each product used and apply these in their applications. For example, here are the relevant product metadata pages for Landsat and Sentinel-2:\n",
"- https://www.usgs.gov/landsat-missions/landsat-science-products\n",
"- https://sentinels.copernicus.eu/web/sentinel/technical-guides/sentinel-2-msi/level-2a/algorithm\n",
"\n",
"The opendatacube provides functions for creating mask arrays from quality measurements defined in the *product definition*. These are covered in various product-specific and example notebooks.\n",
"\n",
"Here we use a simple function to create a mask using only the *no data* value of each measurement array.\n",
"\n",
"> **Exercise**: What is the effect of applying, or not applying, the `valid_mask` to these data (hint: see the NDVI plot below)."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Mask by nodata\n",
"\n",
"# Under the hood: data != data.nodata -> bool\n",
"# Applies to each variable in an xarray.Dataset (including any bit-masks)\n",
"valid_mask = masking.valid_data_mask(data)\n",
"\n",
"# Use numpy.where() to apply a mask array to measurement arrays\n",
"valid_data = data.where(valid_mask) # Default: Where False replace with NaN -> convert dtype to float64\n",
"\n",
"# Or provide a no-data value and retain the dtype\n",
"# nodata = -9999 # A new nodata value\n",
"# valid_data = data.where(valid_mask, nodata) # Where False replace with nodata -> retain dtype if compatible"
]
},
{
"cell_type": "markdown",
"metadata": {
"tags": []
},
"source": [
"### Perform a calculation on the data\n",
"As a simple example, we calculate the [Normalized Difference Vegetation Index (NDVI)](https://en.wikipedia.org/wiki/Normalized_difference_vegetation_index) using the *near infra-red (NIR)* and *red* measurements of the product.\n",
"\n",
"- **Note**: This may not be a realistic *NDVI* example if the measurements have not been scaled to science values.\n",
"\n",
"See this DEA notebook for a set of other remote sensing band indices: https://github.com/GeoscienceAustralia/dea-notebooks/blob/develop/Tools/dea_tools/bandindices.py\n",
"\n",
"> **Exercise**: Calculate a different remote sensing band index, possibly with different measurements loaded into `data`."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Get measurement or alias names corresponding to near-infra read (NIR) and Red bands.\n",
"\n",
"# Calculate the NDVI\n",
"varname = 'ndvi'\n",
"band_diff = valid_data.nir - valid_data.red\n",
"band_sum = valid_data.nir + valid_data.red\n",
"calculation = band_diff / band_sum # xarray.DataArray\n",
"\n",
"# Convert to an xarray.Dataset\n",
"calculation = calculation.to_dataset(name=varname, promote_attrs=True)\n",
"\n",
"# Plot the NDVI\n",
"display(Markdown(f'#### Calculation: {varname.upper()}'))\n",
"calculation[varname].plot(col=\"time\", robust=True, col_wrap=4, vmin=0, vmax=0.7, cmap='RdYlGn');"
]
},
{
"cell_type": "markdown",
"metadata": {
"tags": []
},
"source": [
"### Save the results to file\n",
"We can save an `xarray.Dataset` to a file(s) that can then be imported into other applications for further analysis or publication if required.\n",
"\n",
"In the code below, the file(s) will be saved to your home directory and appear in the File Browser panel to the left. You may need to select the `folder` icon to go to the top level (`$HOME`) and then `output/`.\n",
"\n",
"Download a file by `'right-click' Download`.\n",
"\n",
"> **Exercise**: Use the Terminal to also list the files in the output directory."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### To netCDF"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Xarray can save the data to a netCDF file\n",
"# See also: https://github.com/GeoscienceAustralia/dea-notebooks/blob/develop/Frequently_used_code/Exporting_NetCDFs.ipynb\n",
"\n",
"target = f'{os.environ[\"HOME\"]}/output'\n",
"if not os.path.isdir(target):\n",
" os.mkdir(target)\n",
"\n",
"calculation.time.attrs.pop('units', None) # Xarray re-applies this\n",
"calculation.to_netcdf(f'{target}/example_landsat_{varname}.nc')\n",
"calculation.close()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### To Cloud-optimised Geotiff files"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Single-layer time slices can be written to Geotiff files\n",
"# See also: https://github.com/GeoscienceAustralia/dea-notebooks/blob/develop/Frequently_used_code/Exporting_GeoTIFFs.ipynb\n",
"\n",
"target = f'{os.environ[\"HOME\"]}/output'\n",
"if not os.path.isdir(target):\n",
" os.mkdir(target)\n",
"\n",
"for i in range(len(calculation.time)):\n",
" date = calculation[varname].isel(time=i).time.dt.strftime('%Y%m%d').data\n",
" single = calculation[varname].isel(time=i)\n",
" single.odc.write_cog(\n",
" fname=f'{target}/example_landsat_{varname}_{date}.tif',\n",
" overwrite=True,\n",
" )"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Summary\n",
"\n",
"This notebook introduced the main steps for querying data (with OpenDataCube), and filtering, plotting, calculating and saving a \"cube\" of data (with **Xarray**).\n",
"\n",
"There is plenty of detail and options to explore so please work through the other notebooks to learn more and refer back to these notebooks when required. We encourage you to create or bring your own notebooks, and adapt notebooks from other [open-license repositories](https://docs.asia.easi-eo.solutions/user-guide/users-guide/03-using-notebooks/#other-available-odc-notebooks)."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"rgb(valid_data.isel(time=2), ['red', 'green', 'blue'])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Be a good cloud citizen\n",
"\n",
"It is good practice to close your JupyterLab session when you have finished with it. Your home directory will be retained and in most cases your workspace of open notebooks will also be retained. These will be available when you return to EASI JupyterLab.\n",
"\n",
"Select `File` menu and `Hub Control Panel` from the JupyterLab menu. Then `Stop My Server`.\n",
"- Stop My Server: Your JupyterLab resources will be safely shutdown.\n",
"- Log Out: Log out of your JupyterLab *browser* session. Your JupyterLab resources will remain active until the system cleans up.\n",
"\n",
"![image](../resources/stop-my-server.png)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Further reading \n",
"\n",
"#### JupyterLab\n",
"The JupyterLab website has excellent documentation and video instructions. We recommend users take a few minutes to orientate themselves with the use and features of JupyterLab.\n",
"\n",
"> *Recommended level: Familiarity with notebooks.*\n",
"\n",
"- Getting started: [https://jupyterlab.readthedocs.io/en/stable/getting_started/overview.html](https://jupyterlab.readthedocs.io/en/stable/getting_started/overview.html)\n",
"- Drag and drop upload of files: [https://jupyterlab.readthedocs.io/en/stable/user/files.html](https://jupyterlab.readthedocs.io/en/stable/user/files.html)\n",
"\n",
"#### Python3\n",
"There are many options for learning Python from online resources or facilitated training. Some examples are offered here with no suggestion that EASI endorses any of them.\n",
"\n",
"> *Recommended level: Basic Python knowledge and familiarity with array manipulations, __numpy__ and __xarray__. Familiarity with some plotting libraries (e.g., __matplotlib__) would also help.*\n",
"\n",
"- Get started: [https://www.python.org/about/gettingstarted](https://www.python.org/about/gettingstarted/)\n",
"- Learn Python tutorials: [https://www.learnpython.org](https://www.learnpython.org/)\n",
"- Data Camp: [https://www.datacamp.com](https://www.datacamp.com/)\n",
"- David Beazley courses: [https://dabeaz-course.github.io/practical-python](https://dabeaz-course.github.io/practical-python/)\n",
"- Numpy: [https://numpy.org/doc/stable/user/quickstart.html](https://numpy.org/doc/stable/user/quickstart.html)\n",
"- Xarray: [http://xarray.pydata.org/en/stable/user-guide/data-structures.html](http://xarray.pydata.org/en/stable/user-guide/data-structures.html)\n",
"- Pandas: [https://pandas.pydata.org/docs/getting_started/index.html](https://pandas.pydata.org/docs/getting_started/index.html)\n",
"\n",
"#### Git\n",
"Git is a document version control system. It retains a full history of changes to all files (including deleted ones) by tracking incremental changes and recording a history timeline of changes. The best way to learn Git is by practice and incrementally: start with simple, common actions and gain more knowledge as required. \n",
"\n",
"> *Recommended level: Basic understanding of Git repositories (e.g., github.com) and practices such as __clone__, __pull__/__push__ and __merging__ changes.*\n",
"\n",
"- Getting started: [https://git-scm.com/doc](https://git-scm.com/doc)\n",
"- JupyterLab Git extension: [https://github.com/jupyterlab/jupyterlab-git#readme](https://github.com/jupyterlab/jupyterlab-git#readme)\n",
"- DEA Git guide: [https://github.com/GeoscienceAustralia/dea-notebooks/wiki/Guide-to-using-DEA-Notebooks-with-git](https://github.com/GeoscienceAustralia/dea-notebooks/wiki/Guide-to-using-DEA-Notebooks-with-git)\n",
"- Undoing things guide: [https://git-scm.com/book/en/v2/Git-Basics-Undoing-Things](https://git-scm.com/book/en/v2/Git-Basics-Undoing-Things)\n",
"- Understanding branches: [https://nvie.com/posts/a-successful-git-branching-model](https://nvie.com/posts/a-successful-git-branching-model)\n",
"\n",
"#### Open Data Cube\n",
"The ODC is a Python library that allows the user to search for datasets in its database and return an **xarray** data array. There are convenience functions and methods for resampling, reprojecting and masking the data.\n",
"\n",
"> *Recommended level: Overview of the design and intent of ODC or other datacubes*\n",
"\n",
"- ODC website: [https://www.opendatacube.org](https://www.opendatacube.org)\n",
"- ODC API reference: [https://datacube-core.readthedocs.io](https://datacube-core.readthedocs.io)\n",
"- ODC Github code: [https://github.com/opendatacube](https://github.com/opendatacube)\n",
"\n",
"#### Notebooks for EO data analysis \n",
"There are growing collections of notebooks available from many organizations, most of which can be adapted to use with ODC and EASI.\n",
"\n",
"> *Recommended level: Overview of available notebooks and selected EO applications*\n",
"\n",
"- CSIRO EASI notebooks: [https://github.com/csiro-easi/easi-notebooks](https://github.com/csiro-easi/easi-notebooks)\n",
"- Digital Earth Australia: [https://github.com/GeoscienceAustralia/dea-notebooks](https://github.com/GeoscienceAustralia/dea-notebooks)\n",
"- Digital Earth Africa: [https://github.com/digitalearthafrica/deafrica-sandbox-notebooks](https://github.com/digitalearthafrica/deafrica-sandbox-notebooks)\n",
"- CEOS SEO (NASA): [https://github.com/ceos-seo/data_cube_notebooks](https://github.com/ceos-seo/data_cube_notebooks)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.3"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
notebooks/02-data-storage.ipynb
+458
View File
@@ -0,0 +1,458 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "1f47c2fa-32af-4ff1-8563-35605ee42edb",
"metadata": {},
"source": [
"# Data storage and locality <img align=\"right\" src=\"../resources/csiro_easi_logo.png\">\n",
"\n",
"We have the potential to use and generate huge volumes of data, in different formats, for different purposes and across multiple projects and collaborations.\n",
"\n",
"How we store this data can have significant impacts on:\n",
"\n",
"- Our compute efficiency and science productivity\n",
"- Our ability to connect with other researchers\n",
"- Our budget\n",
"- Our cybersecurity and\n",
"- Our commitment to Research Data Management policies, procedures, and practices\n",
"\n",
"So, how do we choose the right storage solution?\n",
"\n",
"This module will help you explore the different storage options on EASI, their use-cases, their limitations and how you can get started using them. By the end of this module, you will be able to:\n",
"\n",
"- Identify the different available storage options in EASI and recognize:\n",
" - When to use one or more storage options\n",
" - Considerations associated with these storage options (cost, security, latency etc)\n",
"- Upload data to use with your Jupyter Notebook and where to store data outputs for download or re-use by yourself or others.\n",
"- Access resources to help build skills and knowledge to further your learning around data storage options\n",
"\n",
"**Contents**:\n",
"- [How do we normally store data?](#How-do-we-normally-store-data?)\n",
"- [Different storage types in EASI](#Different-storage-types-in-EASI)\n",
"- [Decision making matrix](#Decision-making-matrix)\n",
"- [Scenarios](#Scenarios)"
]
},
{
"cell_type": "markdown",
"id": "448eaa6b-f22c-43c5-9bb1-b682180a5a92",
"metadata": {},
"source": [
"## How do we normally store data? \n",
"\n",
"If you have been using a computer, chances are that you are familiar with storing data on your home drive, or on an external hard drive, or maybe even on the cloud. This is typically referred to as **File Storage**.\n",
"\n",
"With file storage, all data is stored together (usually in one file) and the file extension type helps determine the applications that can read or open the file and access the data (e.g., .jpg, .docx, or .txt).\n",
"\n",
"File storage systems also make it easier for users to find and manage files using a hierarchical structure that organizes files into folders and subfolders. To access the file, select or enter the path of the file, including subdirectories and file name. Most users manage their file storage using a simple file system such as a file manager.\n",
"\n",
"There are other alternatives to the everyday local file storage, like network file storage (connect to the network server first then access by path/file) and *Object Storage*, which is most relevant to cloud-computing.\n",
"\n",
"**Object storage** is like a giant virtual bucket where you can store your belongings in labelled boxes. Each box has a unique ID and can hold different types of objects. You can easily add or remove boxes as needed, and the bucket can expand to accommodate more and more boxes. Plus, you can access any box from anywhere in the world, if you have the ID and access permission. Object storage is ideal for storing vast amounts of unstructured data, such as videos, images, and documents, and is commonly used in cloud storage services. You can have many Buckets where you can store vast amounts of your data and they can all have different access permissions and life cycles. \n",
"\n",
"EASI supports many types of data storage, local and network file storage and Object Storage, and each method has certain advantages, limitations and requirements. EASI also manages data for all users in the Open Data Cube, and there are third-party datasets accessible directly in the Cloud. \n",
"\n",
"Some additional terms that may be helpful when talking about Data Storage:\n",
"\n",
"**Latency**\n",
"\n",
"Data latency is the time it takes for data packets to be stored or retrieved. \n",
"\n",
"**Data Retrieval**\n",
"\n",
"The process of identifying and extracting data from a database or storage system, based on a query provided by the user or application. It enables the fetching of data from storage in order to display it on a monitor and/or use within an application. \n",
"\n",
"**Cache** \n",
"\n",
"A high-speed memory or storage device that helps reduce the time required to read and write data to a slower device, such as a hard drive or a remote server (or bucket)."
]
},
{
"cell_type": "markdown",
"id": "d0b692c6-e989-418b-86de-908818af7891",
"metadata": {
"tags": []
},
"source": [
"## Different storage types in EASI\n",
"\n",
"- [Type 1: Home Directory](#Home-Directory)\n",
"- [Type 2: DataCube](#DataCube)\n",
"- [Type 3: User Scratch](#User-Scratch)\n",
"- [Type 4: Project Data](#Project-Data)\n",
"- [Type 5: Cloud Data](#Cloud-Data)"
]
},
{
"cell_type": "markdown",
"id": "981c0d96-5f23-443e-8c17-88a8f7ab6ca7",
"metadata": {},
"source": [
"### Home Directory\n",
"\n",
"**What is the Home Directory?**\n",
"\n",
"The Home Directory uses the Amazon Web Service (AWS) Elastic File System (EFS) storage architecture. This is similar to a normal network file system in that it has folders, sub folders and files and it is easy and familiar to navigate and use.\n",
"\n",
"**When would I use the Home Directory to store data?**\n",
"\n",
"The Home Directory is best used for small files including source code and notebooks and smaller sets of data outputs.\n",
"\n",
"You can also use the Home Directory to store files that cannot readily work from other storage options (i.e., files or programs that require a normal file system). Again, if these are large, then you may wish to find an alternative and your EASI Administrator will be happy to advise.\n",
"\n",
"**What are the advantages of the Home Directory?**\n",
"\n",
"The data housed in your Home Directory is persistent through log-in/log-out cycles (your home directory will be as you left it). The home directory is also automatically backed-up nightly and retained for 90 days. \n",
"\n",
"The EASI Admin team can restore a file. However this is not always guaranteed. If you are looking for more stable and long-term storage, there are better solutions out there! \n",
"\n",
"**What are the limitations of the Home Directory?**\n",
"\n",
"This is the most expensive form of storage available on EASI. It is ok to store a Gigabyte or so of data, code and outputs. For larger datasets and outputs there are other EASI storage types available. \n",
"\n",
"Note that your home directory is not visible to *Dask* workers, as they run in a different part of EASI. We will learn about the Python *Dask* library later in the module.\n",
"\n",
"**How do I upload and use data?**\n",
"\n",
"There’s a video of this and information in the [Jupyter labs documentation](https://jupyterlab.readthedocs.io/en/stable/user/files.html).\n",
"\n",
"- For small files, it is simplest to drag-and-drop your files into the JupyterLab file interface directly from your desktop. This will upload the file via your browser into your home directory.\n",
"- For larger files, it's best to use the AWS CLI running locally along with your EASI credentials and upload directly to your [EASI S3 User Scratch space](#User-Scratch) space.\n",
"\n",
"Example usage and EASI documentation: https://docs.csiro.easi-eo.solutions/user-guide/users-guide/07-uploading-data/ \n",
"\n",
"**Tasks**\n",
"\n",
"Open an EASI JupyterLab session and a separate browser tab to: \n",
"\n",
"- [ ] Familiarise yourself with the JupyterLabs documentation.\n",
"- [ ] Upload some of your own data or file to your JupyterLab home directory."
]
},
{
"cell_type": "markdown",
"id": "5dc7e716-27ab-4b53-ab4c-ae9cd7e4697e",
"metadata": {},
"source": [
"### DataCube\n",
"\n",
"**What is DataCube?**\n",
"\n",
"The [Open Data Cube (ODC)](https://github.com/opendatacube) is an Open-Source Geospatial Data Management and Analysis Software project that helps you harness the power of Satellite data. At its core, the ODC is a set of Python libraries and PostgreSQL database that helps you work with geospatial raster data.\n",
"\n",
"The broad goal of the ODC is to make it easier to access and use large data holdings, without requiring data to be stored in a specific way or in a specific place. What this means is that you can point it at your data repository and index the data where it sits, abstracting the complexity of managing large, distributed data holdings.\n",
"\n",
"**EASI managed data**\n",
"\n",
"EASI manages an amount of data in its datacube databases. These can be viewed in each EASI's Explorer website and with the *Datacube()* API.\n",
"\n",
"- This is the CSIRO Explorer site: https://explorer.csiro.easi-eo.solutions/\n",
"- The EASI documentation [lists the URLs for other EASIs](https://docs.csiro.easi-eo.solutions/user-guide/developers/easi-platform-overview/#easi-deployments-services-and-support).\n",
"\n",
"There are many commonalities in the data processing workflows (into a datacube databases) across the ODC community. We do this so that its easier to move your work between ODC systems (including third-parties). The EASI team are working to make our data processing workflows transparent and open to contributions.\n",
"\n",
"**When would I use the DataCube**\n",
"\n",
"If you’re using data that’s available as EASI managed data, then it’s much easier to use via the ODC API and you don’t have to have your own copy! See the [easi-notebooks](https://github.com/csiro-easi/easi-notebooks) for getting started tutorials.\n",
"\n",
"**What are the advantages of the DataCube?**\n",
"\n",
"DataCube is managed for you! So, you don’t need to worry about updates, maintenance, storage costs etc. Many EO and large public data collections are available, or becoming available, via the cloud. EASI and ODC connects directly to the current best collection endpoints and API services, and utilises cloud efficiencies and capabilities where possible.\n",
"\n",
"Many of the data collections and processing routines are open source, and are contributed to by the ODC, research and domain-coordination communities. EASI leverages and contributes to these open resources to enure our users stay up to date. The EASI team welcome suggestions and contributions to the development of current or new data processing workflows.\n",
"\n",
"**What are the limitations of the DataCube?**\n",
"\n",
"It may not have the specific dataset you are looking for. However datasets are always being added and updated, and you can build your own and share it!\n",
"\n",
"The Datacube API may not suit your overall workflow. Just remember that the datacube shows “one way to do it”. Talk to the EASI team about your workflow and we can advise what others options are available or could be combined. For example, you can access the list of files (rather than the data) that the datacube API would return for your query, and you can likely access the same sources that EASI’s workflows do.\n",
"\n",
"**How do I upload and use data?**\n",
"\n",
"Getting start with using data from, and uploading to, the Open Data Cube is straightforward. You can explore the tutorial notebooks in the git repository: https://github.com/csiro-easi/easi-notebooks.\n",
"\n",
"**Tasks**\n",
"\n",
"Open an EASI JupyterLab session to:\n",
"\n",
"- [ ] Upload a shapefile (copy from some resource or choose your own)\n",
"- [ ] Load EO datacube into an xarray\n",
"- [ ] Mask with an uploaded shapefile"
]
},
{
"cell_type": "markdown",
"id": "f13ce0ac-7d78-44bf-930d-2beaf66a4add",
"metadata": {},
"source": [
"### User Scratch\n",
"\n",
"**What is User Scratch?**\n",
"\n",
"EASI has a \"scratch\" bucket available for all EASI users to write to. Scratch storage serves as a temporary location for large data sets that are more appropriate to save and access from AWS S3 storage. These could be intermediate datasets in your workflow or interim results for testing and exploration.\n",
"\n",
"**When would I use the User Scratch?**\n",
"\n",
"The User Scratch bucket is helpful to save files between processing runs or share files between your projects. You can use Scratch as large, efficient temporary storage for your workflows and projects, e.g. for saving intermediate results from your workflows.\n",
"\n",
"Scratch can be accessed by *Dask* workers for passing data and saving snapshots of results, which is an advantage over Home Directory storage. Don’t know what *Dask* is? Don’t worry, we’ll cover Dask in detail.\n",
"\n",
"**What are the advantages of the User Scratch?**\n",
"\n",
"It’s a dedicated S3 Bucket in EASI that provides:\n",
"\n",
"- Lots of storage and scale (efficient)\n",
"- Works with Dask workers (fast)\n",
"- Has a managed lifetime, in case you forget it will (eventually) clean up after you\n",
"- You can use the AWS CLI and library functions to read and write files to this bucket\n",
"\n",
"**What are the limitations of the User Scratch?**\n",
"\n",
"User Scratch is *Object Storage*, which is a bit different to file storage, so it requires a different method to access and manage the data. Many tools support efficient read/write to cloud object storage directly but some older tools that you use may not. If so, use the AWS CLI to make a temporary copy of the files to your JupyterLab home directory or to dask workers.\n",
"\n",
"Secondly, objects in the user scratch bucket have a 30-day (since date of creation) lifecycle rule. It is not intended for long-term data storage. Similarly, it is also not a practical solution if you need to share files and resources between projects, particularly beyond 30 days. If sharing data or a different lifecycle is a project requirement, then consider provisioning a \"Project\" bucket tailored to your needs.\n",
"\n",
"**How do I upload and use data?**\n",
"\n",
"User guide / additional information:\n",
"\n",
"- https://docs.csiro.easi-eo.solutions/user-guide/users-guide/07-uploading-data/\n",
"- https://github.com/csiro-easi/easi-notebooks\n",
"\n",
"**Tasks**\n",
"\n",
"- [ ] Save your xarray masked result (from previous step) to User Scratch"
]
},
{
"cell_type": "markdown",
"id": "129ac062-fc83-46b5-9ac0-a244dfb10da1",
"metadata": {},
"source": [
"### Project Data\n",
"\n",
"**What is Project Data?**\n",
"\n",
"Like the “User Scratch bucket”, which all EASI users can access, each project can also make additional storage resources available for their users in EASI. This is known as a **Project** bucket, and it can be shared across all project members and have different levels of membership (read/write, read-only).\n",
"\n",
"Create and manage your own AWS account, and then create a bucket that can be shared with EASI. The EASI team will coordinate with you to enable cross-account authorisation between EASI and your shared project bucket. You retain full control of your AWS resources. Contact EASI for more information.\n",
"\n",
"**When would I use Project Data?**\n",
"\n",
"Project Data is best used when you need to share data and results with collaborators in a project, or when you want to manage your own large data in your own project account with your own lifecycle rules.\n",
"\n",
"**What are the advantages?**\n",
"\n",
"From EASI, a Project bucket is only accessible by users who have been nominated as a member of the *project*. This allows sharing with colleagues who are part of the same project.\n",
"\n",
"The project's AWS account administrator can create custom lifecycle rules and can enable other services that might be appropriate for the project.\n",
"\n",
"**What are the limitations?**\n",
"\n",
"Only accessible to people that have access to this storage, i.e. colleagues as part of the same project, which is the intent.\n",
"\n",
"**How do I upload and use data?**\n",
"\n",
"The methods for uploading or reading data from a project bucket are the same as for the User Scratch bucket. Just change the bucket name!\n",
"\n",
"**User guide / additional information:**\n",
"\n",
"In CSIRO, contact the Cloud Platforms Team to set up your project AWS account. Then contact the EASI team to enable cross-account authorisation and to nominate users and their read/write permissions.\n",
"\n",
"**Tasks**\n",
"\n",
"- [ ] If you have a project bucket then use the same approach as for User Scratch to upload your xarray result to the project bucket.\n",
"- [ ] If not, then the only task is to remember that this option exists!"
]
},
{
"cell_type": "markdown",
"id": "cee6dfc5-3405-4262-9d3f-1cdbc99e27a8",
"metadata": {},
"source": [
"### Cloud Data\n",
"\n",
"**What is cloud-available data and what cloud data services / technologies does EASI support?\n",
"\n",
"The “cloud” in this context refers to the main global cloud data companies – Amazon Web Services, Google Earth Engine and Microsoft Planetary Computer. However, the ideas mentioned here are also reasonably applicable to many internet-available datasets, although the specific way to access these may differ.\n",
"\n",
"Many Earth observation and other public data collection providers are making their data available directly in the cloud. In many cases the cloud copy is their ‘authoritative’ public copy of the data. The same data collection may also be available in more than one cloud system.\n",
"\n",
"The amount and variety of data and providers is vast, and we can’t cover them all here but will highlight that they can in most cases be accessed directly from EASI.\n",
"\n",
"**When would I use cloud-based data assets?**\n",
"\n",
"When a provider has data you want and its just as efficient to access it directly as it is to download and store (why bother curating it yourself!)\n",
"\n",
"**What are the advantages of cloud-based data assets?**\n",
"\n",
"- You don’t pay for the long-term storage\n",
"- Maintained by the data provider\n",
"- There’s a lot of it\n",
"\n",
"Many of the EASI-managed datacube products are indexed directly or indirectly from the cloud.\n",
"\n",
"**What are the limitations of cloud-based data assets?**\n",
"\n",
"Cost and efficiency mostly. While these cloud data collections are publicly available they are not necessarily free to access. The cloud companies typically charge network data transfer and related costs associated for moving data within, into and out of their systems.\n",
"\n",
"Additionally, accessing data over the internet can slow workflow speed and repeated use in workflows will result in more network costs (possibly more than the cost of downloading the data and temporarily storing it). All these need to be managed.\n",
"\n",
"For small, infrequent access (like one off grabs of data) you can safely just use them. For large data use or frequent re-use then if you are unsure contact your EASI Admin and we’ll help you navigate. It’s not difficult, just awkward at the start to understand what to watch for.\n",
"\n",
"**How do I access and use cloud-based data?**\n",
"\n",
"This varies depending on the data source and we can’t cover them all. However, there are common considerations highlighted below to help you think about how to use cloud-based data. Each cloud data company provides a list or summary of their available public data collections. It is important to be able to locate these and to identify and interpret some of the key points.\n",
"\n",
"- [Registry of Open Data on AWS](https://registry.opendata.aws/)\n",
"- [Earth Engine Data Catalog](https://developers.google.com/earth-engine/datasets)\n",
"- [Planetary Computer Data Catalog](https://planetarycomputer.microsoft.com/catalog)\n",
"\n",
"The OpenDataCube provides a tool that can read from the cloud data collections and return an xarray object as if the data had been read from a datacube database. Examples are provided here: https://odc-stac.readthedocs.io/en/latest/examples.html\n",
"\n",
"**Tasks**\n",
"\n",
"- [ ] Let's look at a data catalog example from AWS: https://registry.opendata.aws/sentinel-2-l2a-cogs/\n",
"\n",
"<img align=\"center\" src=\"../resources/aws-opendata-markup.png\">"
]
},
{
"cell_type": "markdown",
"id": "ee4e2480-5702-4326-a15a-2276d17c1965",
"metadata": {},
"source": [
"## Decision making matrix\n",
"\n",
"This table summarises the key attributes of the different storage types.\n",
"\n",
"<table>\n",
"<tr align=\"center\">\n",
" <th>&nbsp;\n",
" <th colspan=2>User access rights\n",
" <th colspan=2>Dask workers access rights\n",
" <th colspan=2>Collaborator access rights\n",
" <th colspan=2>Suitable file size\n",
" <th colspan=2>Network costs\n",
" <th colspan=2>Storage costs\n",
" <th colspan=2>Shelf life\n",
"<tr align=\"center\">\n",
" <td>&nbsp;\n",
" <td><em>Read<td><em>Write\n",
" <td><em>Read<td><em>Write\n",
" <td><em>Read<td><em>Write\n",
" <td><em>Large<td><em>Small\n",
" <td><em>High<td><em>Low\n",
" <td><em>High<td><em>Low\n",
" <td><em>Short term<td><em>Long term\n",
"<tr align=\"center\">\n",
" <td><a href=\"#Home-directory\">Home directory</a><td>Yes<td>Yes<td>No<td>No<td>No<td>No<td>&nbsp;<td>X<td>&nbsp;<td>X<td>X<td>&nbsp;<td>X<td>X\n",
"<tr align=\"center\">\n",
" <td><a href=\"#User-scratch\">User scratch</a><td>Yes<td>Yes<td>Yes<td>Yes<td>No<td>No<td>X<td>X<td>&nbsp;<td>X<td>&nbsp;<td>X<td>X<td>&nbsp;\n",
"<tr align=\"center\">\n",
" <td><a href=\"#Project-data\">Project data</a><td>Yes<td>Yes<td>Yes<td>Yes<td>Yes<td>Yes<td>X<td>X<td>&nbsp;<td>X<td>&nbsp;<td>X<td>X<td>X\n",
"<tr align=\"center\">\n",
" <td><a href=\"#Data-cube\">Data cube</a><td>Yes<td>No<td>Yes<td>No<td>Yes<td>No<td>X<td>X<td>&nbsp;<td>X<td>&nbsp;<td>X<td>&nbsp;<td>X\n",
"<tr align=\"center\">\n",
" <td><a href=\"#Cloud-data\">Cloud data</a><td>Yes<td>No<td>Yes<td>No<td>Yes<td>No<td>X<td>X<td>X<td>X<td>&nbsp;<td>X<td>X<td>X\n",
"</table>\n",
"\n",
"Note that the specifics for any cloud data sources can vary depending on the provider, where the data is stored, where it is being accessed and more. For an in-depth conversation, reach out to the EASI team.\n",
"\n",
"<!--\n",
"Markup version - could not find a way to do colspan\n",
"| | User access rights | | Dask workers access rights | | Collaborator access rights | | Suitable file size | | Network costs | | Storage cost | | Shelf life | |\n",
"|--|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|\n",
"| | *Read* | *Write* | *Read* | *Write* | *Read* | *Write* | *Large* | *Small* | *High* | *Low* | *High* | *Low* | *Short term* | *Long term* |\n",
"| [User scratch](#User-scratch) | Yes | Yes | No | No | No | No | | X | | X | X | | X | X |\n",
"-->"
]
},
{
"cell_type": "markdown",
"id": "fdd2703e-24b1-4d56-8c48-7d7accf8b41d",
"metadata": {},
"source": [
"## Scenarios\n",
"\n",
"### Scenario 1 - Home directory and ODC\n",
"\n",
"Michael has some small data files that he wants to upload and use in conjunction with datacube data. Where is the best place for him to upload this data?\n",
"\n",
"*Answer*: Home directory\n",
"\n",
"The Home Directory is great for smaller data files that you need for your work. It is easy to upload small files from your desktop with drag-and-drop in JupyterLab. There are many python notebook examples (to borrow or adapt from) that query the datacube API (which returns an xarray object) then combine with other data (such as shapefiles, polygons or non-datacube data) with standard python tools.\n",
"\n",
"For larger files and datasets you may consider uploading to User Scratch or a Project Data bucket."
]
},
{
"cell_type": "markdown",
"id": "5628209c-9a27-4eb1-b210-d9512e91664d",
"metadata": {},
"source": [
"### Scenario 2 - User scratch and Project data buckets\n",
"\n",
"Jenny is generating a set of intermediate or final data files from her workflow. The total size is in the Gigabytes. Where should Jenny write the data files to?\n",
"\n",
"*Answer*: User scratch or Project data (if applicable)\n",
"\n",
"*Extended response*:\n",
"\n",
"S3 bucket storage is preferred for data sets larger than a few GBs. Workflows (notebooks and dask workers) can read and write to S3 buckets if permitted. All EASI users have access to the User Scratch bucket. Some users may also have access to a Project data bucket, in which case the data can also be used by project colleagues.\n",
"\n",
"S3 bucket storage is preferred for larger datasets because it is cheaper, efficient and programmable from notebooks and dask workers. Your home directory is more expensive and not accessible from dask workers, so is less suitable for workflows generating or using large datasets."
]
},
{
"cell_type": "markdown",
"id": "42a61595-436a-41e6-91d2-402eca49288d",
"metadata": {},
"source": [
"### Scenario 3: Cloud or external online data\n",
"\n",
"Catherine would like to access a dataset that is available in the cloud or from an external online service. She checks Explorer and notes that the dataset is not available in the datacube database. What choices does Catherine have?\n",
"\n",
"*Answer*: Consider the details for accessing these Cloud data in EASI and seek advice if required.\n",
"\n",
"*Extended response*:\n",
"\n",
"Typical external data access considerations include:\n",
"\n",
"- Whether there is a programmable request interface, e.g. STAC API, an OGC web service, or a custom API. For STAC APIs, you can use the odc-stac tool to query and read the data into an xarray object. Similarly, there are common python tools for reading from ODC web services. A custom API will likely need specific code to search, download and prepare the data for use.\n",
"- Where the data are located. Cloud data services (including EASI’s host) often charge for each data transaction into and out of their data centres, which add up. Moving data across the internet can be slow as well. Ask the EASI team for advice.\n",
"- Is this a candidate dataset for including in the datacube database? If yes, then a great start is to document how this dataset can be used by you and others, the dataset source and metadata, and some example code for accessing and using the dataset."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d3bd1817-31f7-455a-ade6-57822792837c",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.6"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
notebooks/adding-python-libraries.ipynb
+196
View File
@@ -0,0 +1,196 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "fad43de4",
"metadata": {},
"source": [
"# Adding custom python libraries <img align=\"right\" src=\"../resources/csiro_easi_logo.png\">\n",
"It is possible to add additional python libraries to your notebook environment, but you first need to follow the steps below.\n",
" \n",
"This example creates a new environment called **myenv**. You should replace all instances of **myenv** with whatever name you would like to call your new environment."
]
},
{
"cell_type": "markdown",
"id": "8e8fb95b",
"metadata": {},
"source": [
"### 1. Create new virtual environment\n",
"First, **in a terminal** (File>New>Terminal), create a virtual environment\n",
"```bash\n",
"MYENV=myenv\n",
"python3 -m venv ~/venvs/${MYENV}\n",
"```"
]
},
{
"cell_type": "markdown",
"id": "e491b605-ef63-4532-9b23-3a67ea3046e4",
"metadata": {},
"source": [
"<div class=\"alert alert-info\">\n",
" <p><strong>NOTE: </strong>Make sure that you replace <code>myenv</code> every time with whatever name you would like for your new environment.</p>\n",
"</div>"
]
},
{
"cell_type": "markdown",
"id": "8d4a13dc",
"metadata": {},
"source": [
"### 2. Load default libraries\n",
"In order to load all of the default python libraries that are installed in the default environment (e.g. including gdal, dask, boto, plotly, holoviews and many others), run the following command:\n",
"\n",
"```bash\n",
"PYVERS=`python -c \"import sys; print('{0[0]}.{0[1]}'.format(sys.version_info))\"`\n",
"\n",
"realpath /env/lib/python${PYVERS}/site-packages > ~/venvs/${MYENV}/lib/python${PYVERS}/site-packages/base_venv.pth\n",
"```\n",
"\n",
">__NOTE:__ Replace `myenv` with whatever name you are using for your new environment."
]
},
{
"cell_type": "markdown",
"id": "8bd3e8da",
"metadata": {},
"source": [
"### 3. Install new libraries\n",
"Switch into the new venv if you haven't done so already in step 1\n",
"```bash\n",
"source ~/venvs/${MYENV}/bin/activate\n",
"```\n",
">__NOTE:__ Replace `myenv` with whatever name you are using for your new environment."
]
},
{
"cell_type": "markdown",
"id": "025f7e8f",
"metadata": {},
"source": [
"You will now be able to nstall new libraries into your new environment, e.g. Below we install [geopy](https://geopy.readthedocs.io/en/stable/)\n",
"\n",
"```bash\n",
"pip install geopy\n",
"```"
]
},
{
"cell_type": "markdown",
"id": "a1475b4e",
"metadata": {},
"source": [
"### 4. Virtual Env in EASI Hub Jupyter (optional)\n",
"\n",
"You can then add the new environment as a kernel inside Jupyter by running the following line. \n",
"\n",
">__NOTE:__ you ___MUST___ have your new environment activated (as shown above - `source ~/venvs/myenv/bin/activate`) before running the command below.\n",
"> When activated, the environment name should be visible to the left of the command line, like below\n",
"\n",
"```bash\n",
"(myenv) jovyan@jupyter-userid:~$\n",
"```\n",
"\n",
"Then run:\n",
"\n",
"```sh\n",
"python -m ipykernel install --user --name=\"My environment\"\n",
"```\n",
">__NOTE:__ Replace `My environment` with a human-readable name for your new environment."
]
},
{
"cell_type": "markdown",
"id": "2bf7e38b",
"metadata": {},
"source": [
"After running this example above, refresh the browser and click on the words \"Python 3\" at the top right. This allows you to select a different Kernel, including your new **\"My environment\"** environment or whatever name you choose."
]
},
{
"cell_type": "markdown",
"id": "528c49d5-8253-4278-9c0f-167b3dd60597",
"metadata": {},
"source": [
"### 5. Return, review and clean up"
]
},
{
"cell_type": "markdown",
"id": "2e128e23",
"metadata": {},
"source": [
"---\n",
"To disconnect from your new virtual environment and return to the default, use the following command in the terminal:\n",
"```bash\n",
"deactivate\n",
"```"
]
},
{
"cell_type": "markdown",
"id": "fe6c55da-75c8-4b9c-b213-0af01d0e416a",
"metadata": {},
"source": [
"---\n",
"To list all available Jupyter kernels:\n",
"```bash\n",
"jupyter kernelspec list\n",
"```"
]
},
{
"cell_type": "markdown",
"id": "749d9410-3011-4a48-a503-c01485bb202e",
"metadata": {},
"source": [
"---\n",
"To remove a kernel from Jupyter:\n",
"```bash\n",
"jupyter kernelspec uninstall unwanted-kernel\n",
"```"
]
},
{
"cell_type": "markdown",
"id": "d3d66bb5-035a-4ec0-b5e6-96ea131975f8",
"metadata": {},
"source": [
"And then if you want to fully delete your custom python environment, simply delete it from the filesystem:\n",
"```bash\n",
"rm -r ~/venvs/${MYENV}\n",
"```"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "87786ad4-fa3c-4a66-9b62-9a39b7c64506",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.7"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
notebooks/dask/01-Introduction_to_Dask.ipynb
+696
View File
@@ -0,0 +1,696 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "416ee5f1-2a79-4952-a901-0d1f027f468d",
"metadata": {},
"source": [
"# Introduction to Dask and the Open Data Cube <img align=\"right\" src=\"../../resources/csiro_easi_logo.png\">\n",
"\n",
"**Prerequisites**: This material assumes basic knowledge of the Open Data Cube, Xarray and numerical processing using numpy.\n",
"\n",
"- [Introduction](#Introduction-to-Dask-and-the-Open-Data-Cube)\n",
" - [Planning and writing efficient applications](#Planning-and-writing-efficient-applications)\n",
" - [What you will learn](#What-you-will-learn)\n",
"- [Performance in Python](#Performance-in-Python)\n",
" - [Python list and numpy](#Python-list-and-numpy)\n",
" - [Numba - accelerating Python](#Numba---accelerating-Python)\n",
"- [Parallelism with Dask](#Parallelism-with-Dask)\n",
" - [Review of dask](#Review-of-dask)\n",
" - [Dask local cluster](#Dask-local-cluster)\n",
"\n",
"The Open Data Cube library is written in Python and makes extensive use of scientific and geospatial libraries. For the purposes of this tutorial we will primarily consider five libraries:\n",
"\n",
" 1. `datacube` - EO datacube\n",
" 1. `xarray` - labelled arrays\n",
" 1. (optional) `dask` & `distributed` - distributed parallel programming\n",
" 1. `numpy` - numerical array processing with vectorisation\n",
" 1. (optional) `numba` - a library for high performance python\n",
"\n",
"Whilst the interrelations are intimate it is useful to conceptualise them according to their primary role and how these roles build from low level numerical array processing (`numpy`) through to high-level EO datacube semantics (`datacube` and `xarray`). If you prefer, viewed from top to bottom we can say:\n",
" 1. `datacube.load()` does the necessary file IO and data manipulation to construct a...\n",
" 1. `xarray` which will be labelled with the necessary coordinate systems and band names and made up of...\n",
" 1. (optionally) `dask.array`s which contain many `chunks` which are...\n",
" 1. `numpy` arrays containing the actual data values.\n",
"\n",
"Each higher level of abstraction thus builds on the lower level components that perform the actual storage and computation.\n",
"\n",
"Overlaid on this are libraries like `numba`, `dask` and `distributed` that provide computational components that can accelerate and distribute processing across multiple compute cores and computers. The use of `dask`, `distributed` and `numba` are optional - not all applications require the additional complexity of these tools.\n",
"\n",
"### Planning and writing efficient applications\n",
"\n",
"Achieving performance and scale requires an understanding of the performance of each library and how it interacts with the others. Moreover, and often counterintuitively, adding more compute cores to a problem may not make it faster; in fact it may slow down (as well as waste resources). Added to that is the _deceptive simplicity_ in that some of the tools can be simply _turned on_ with only a few code changes and significant performance increases can be achieved.\n",
"\n",
"However, as the application is scaled or an alternative algorithm is used further challenges may arise, in expected I/O or compute efficiency, that require code refactors and changes in algorithmic approach. These challenges can seem to undo some of the earlier work and be frustrating to address. \n",
"\n",
"The good news is whilst there is complexity (six interrelated libraries mentioned so far), there are common concepts and techniques involved in analysing _how to optimise your algorithm_. If you know from the start your application is going to require scale, then it does help to think in advance where you are heading.\n",
"\n",
"### What you will learn\n",
"\n",
"This course will equip readers with concepts and techniques they can utilise in their algorithm and workflow development. The course will be using computer science terms and a variety of libraries but won't be discussing these in detail in order to keep this course concise. The focus will be on demonstration by example and analysis techniques to identify where to focus effort. The reader is encouraged to use their favorite search engine to dig deeper when needed; there are a lot of tutorials online!\n",
"\n",
"One last thing, in order to maintain a healthy state of mind for \"Dask and ODC\", the reader is encouraged to hold both of these truths in mind at the same time:\n",
" 1. The *best* thing about dask is it makes distributed parallel programming in the datacube easy\n",
" 1. The *worst* thing about dask it is makes distributed parallel programming in the datacube easy\n",
"\n",
"Yep, that's contradictory! By the end of this course, and a couple of your own adventures, you will understand why."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f3fb9221-d423-4a3e-baa4-6cdaefd56c1d",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# EASI defaults\n",
"import git\n",
"import sys\n",
"import os\n",
"os.environ['USE_PYGEOS'] = '0'\n",
"repo = git.Repo('.', search_parent_directories=True).working_tree_dir\n",
"if repo not in sys.path: sys.path.append(repo)\n",
"from easi_tools import EasiDefaults, notebook_utils"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "78b8349c-6698-45d2-a159-8013222232d1",
"metadata": {},
"outputs": [],
"source": [
"easi = EasiDefaults()"
]
},
{
"cell_type": "markdown",
"id": "5be4a71f-d1f0-4c2a-a0cb-f53757d78cb1",
"metadata": {},
"source": [
"# Performance in Python\n",
"\n",
"In this section we will explore python performance for array processing. Python itself, as you will soon see, is quite slow. It is, however, highly expressive and can orchestrate more complex and faster libraries of numerical code (e.g., `numpy`). Python is also ammendable to being accelerated (e.g. using `numba`) and made to run on multiple CPU cores (e.g. via `dask`). \n",
"\n",
"### Python `list` and `numpy`\n",
"\n",
"Let's take a look at the simple addition of two arrays. In Python the nearest data type to an array is a `list` of numbers. This will be our starting point.\n",
"\n",
"Our focus is on performance so we'll use the Jupyter `%%time` and `%%timeit` magics to run our cells and time their execution. The latter will run the cell multiple times and provide us with more representative statistics of performance and variability.\n",
"\n",
"First in pure Python using lists :"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f2dfa3fd-0cc7-4804-8790-ed65f05eaa4e",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"size_of_vec = 2000*2000\n",
"X_list = range(size_of_vec)\n",
"Y_list = range(size_of_vec)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d8c79652-2a47-40f4-87a2-8e90ce9cdf9c",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"%%timeit -n 10 -r 1\n",
"Z = [X_list[i] + Y_list[i] for i in range(len(X_list)) ]"
]
},
{
"cell_type": "markdown",
"id": "9aa9586d-b6ab-49d6-8914-ff016023cae8",
"metadata": {},
"source": [
"Now the same processing using `numpy`."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "5158c9d0-d8a9-48e6-ab1f-df4948dde092",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"import numpy\n",
"X = numpy.arange(size_of_vec)\n",
"Y = numpy.arange(size_of_vec)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a4dc338b-d411-40be-8aa0-4b50bce2201a",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"%%timeit -n 10 -r 1\n",
"Z = X + Y"
]
},
{
"cell_type": "markdown",
"id": "c8e8fb26-0d58-4dc1-b8ea-6c5ad044e4ea",
"metadata": {},
"source": [
"Let's check that the two arrays are identical (note that %%timeit does not make the variables above available due to the way that %%timeit works, so we reconstruct the arrays)."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "476b9020-ba06-47d4-84f7-f6ede912ce88",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"print(\"Are the arrays identical?\")\n",
"([X_list[i] + Y_list[i] for i in range(len(X_list)) ] == X + Y).all()"
]
},
{
"cell_type": "markdown",
"id": "7ff1825c-9bf8-4ab8-baa9-13b43f4f4a3f",
"metadata": {},
"source": [
"At least two orders of magnitude in performance improvement!\n",
"\n",
"Why?\n",
"\n",
"`numpy` provides a python interface to an underlying C array library that makes use of CPU `vectorization` - this allows it to process several add operations at the same time.\n",
"\n",
"`numpy` isn't the only library that does this type of wrapping over a fast optimised library. There are, for example,\n",
"- `cuPy` which uses GPUs for array processing\n",
"- `tensorflow` uses both CPU and GPU optimisations for machine learning\n",
"- `datashader` for large dataset visualisation\n",
"\n",
"It's a very long list and thanks to a great deal of work by a great many software engineers most of these libraries will work together efficiently. \n",
"\n",
"> __Tip__: Where possible use high performance libraries that have python wrappers.\n",
"\n",
"The reader will have noticed the change in abstraction. The pure Python version used list comprehension syntax to add the two arrays, while `numpy` was a much shorter direct addition syntax more in keeping with the mathematics involved. This change in abstraction is seen in most libraries, including the ODC library where `datacube.load()` is shorthand for a complex process of data discovery, reprojection, fusing and array construction. High-level abstractions like this are powerful and greatly simplify development (the good). They can also hide performance bottlenecks and challenges (the bad).\n",
"\n",
"> __Tip__: Use high level API abstractions but be mindful of their use."
]
},
{
"cell_type": "markdown",
"id": "2e01d98d-0090-46a7-adee-52534ee49f74",
"metadata": {},
"source": [
"### `Numba` - accelerating Python\n",
"\n",
"So high performance libraries rock, but what if you don't have one for your purpose and you're back in Python?\n",
"`Numba` translates Python functions into optimized machine code at runtime - https://numba.pydata.org.\n",
"\n",
"Let's see how this works. A more complex example this time with a smoothing function applied over our (random) image, perform an FFT, and save the result.\n",
"These examples are (very) slightly modified versions from the [High Performance Python Processing Pipeline video by Matthew Rocklin](https://youtu.be/wANQkgDuTAk). It's such a good introduction its worth repeating.\n",
"\n",
"We'll also use the `tqdm` library to provide a progress bar."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "30019636-0446-4f0a-85ec-d93139065f74",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"import numpy as np\n",
"from tqdm.notebook import tqdm\n",
"\n",
"def load_eo_data():\n",
" return np.random.random((1000, 1000))\n",
"\n",
"def smooth(x):\n",
" out = np.empty_like(x)\n",
" for i in range(1, x.shape[0] - 1):\n",
" for j in range(1, x.shape[1] - 1):\n",
" out[i, j] = (x[i + -1, j + -1] + x[i + -1, j + 0] + x[i + -1, j + 1] +\n",
" x[i + 0, j + -1] + x[i + 0, j + 0] + x[i + 0, j + 1] +\n",
" x[i + 1, j + -1] + x[i + 1, j + 0] + x[i + 1, j + 1]) // 9\n",
" return out\n",
"\n",
"def save(x, filename):\n",
" pass "
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f83fbe3b-5138-472e-a6f2-7e34f71804bb",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"%%time\n",
"for i in tqdm(range(5)):\n",
" img = load_eo_data()\n",
" img = smooth(img)\n",
" img = np.fft.fft2(img)\n",
" save(img, \"file-\" + str(i) + \"-.dat\")"
]
},
{
"cell_type": "markdown",
"id": "6cc2f3e8-c9e5-4cf2-bb66-bb27c3a7211b",
"metadata": {},
"source": [
"The `smooth(x)` function contains two python loops. Now we could (and would) find a similar high performance library with a `smooth(x)` function but for this example let's use `numba`'s `jit` compiler to translate the python function into optimized machine code at runtime."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "18808bb2-ca62-425c-9e08-f876ceeaaff2",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"import numba\n",
"\n",
"fast_smooth = numba.jit(smooth)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "14d8a5e0-30f9-4676-a436-e5643c92dddf",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"%%time\n",
"\n",
"for i in tqdm(range(5)):\n",
" img = load_eo_data()\n",
" img = fast_smooth(img)\n",
" img = np.fft.fft2(img)\n",
" save(img, \"file-\" + str(i) + \"-.dat\")"
]
},
{
"cell_type": "markdown",
"id": "82412220-68b4-4905-b6f0-2b53b71c55f5",
"metadata": {},
"source": [
"Just a bit quicker! Much of the time in the first run was `numba` performing compilation. Run the cell above again and you'll find it runs faster the second time.\n",
"\n",
"The _recommended_ approach to have `numba` compile a python function is to use python decorator syntax (`@numba.jit`). So the original code now looks like this (single line changed) and we can call `smooth(x)` without having to create `fast_smooth`:"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "820dd545-bae8-477b-9e1e-73c806b953bd",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"import numpy as np\n",
"\n",
"def load_eo_data():\n",
" return np.random.random((1000, 1000))\n",
"\n",
"@numba.jit\n",
"def smooth(x):\n",
" out = np.empty_like(x)\n",
" for i in range(1, x.shape[0] - 1):\n",
" for j in range(1, x.shape[1] - 1):\n",
" out[i, j] = (x[i + -1, j + -1] + x[i + -1, j + 0] + x[i + -1, j + 1] +\n",
" x[i + 0, j + -1] + x[i + 0, j + 0] + x[i + 0, j + 1] +\n",
" x[i + 1, j + -1] + x[i + 1, j + 0] + x[i + 1, j + 1]) // 9\n",
" return out\n",
"\n",
"def save(x, filename):\n",
" pass"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "59f9604e-e0d6-4ac0-bd54-b93f6952131e",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"%%time\n",
"for i in tqdm(range(5)):\n",
" img = load_eo_data()\n",
" img = smooth(img)\n",
" img = np.fft.fft2(img)\n",
" save(img, \"file-\" + str(i) + \"-.dat\")"
]
},
{
"cell_type": "markdown",
"id": "4598e9d1-e154-4309-9bc0-0d5252775975",
"metadata": {},
"source": [
"Why not use `numba` all the time everywhere?\n",
"\n",
"Like most high level abstractions `numba` makes assumption about code, only accelerates a subset of python libraries (not all `numpy` functions are available via `numba`), and it is entirely possible it can make performance worse or not work at all!\n",
"\n",
"There's one additional consideration. If you've run all the cells to this point in order, try running the `fast_smooth` cell again, repeated below for convenience:\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "01a91155-67a3-4ab2-a2ff-712d0c43c71e",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"fast_smooth = numba.jit(smooth)"
]
},
{
"cell_type": "markdown",
"id": "9675cb05-6794-4e9f-8daa-099846dfd3d0",
"metadata": {
"tags": []
},
"source": [
"Error!\n",
"\n",
"The `smooth` function was decorated so is already `jit`-compiled. Attempting to do so again causes this error, and exposes some of the low level changes behind the abstraction.\n",
"This can make debugging code difficult if you are not mindful of what is occuring.\n",
"\n",
"TIP: __Use high level API abstractions but be mindful of their use__"
]
},
{
"cell_type": "markdown",
"id": "25419e83-3fce-49be-b1b4-54f6415d7096",
"metadata": {},
"source": [
"# Parallelism with Dask\n",
"\n",
"Our fake EO processing pipeline only has 5 images and takes about 1 sec to run. In practice we'll have 1000s of images to process (if not more).\n",
"\n",
"Let's repeat our example code but now with more iterations. You can understand why we use The `tqdm` library to provide a progress bar for these larger scale examples rather than printing out each iteration number or staring at a blank screen wondering if it works!\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "af003cf6-3993-4718-a0d0-48c34f96c0e4",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"import numpy as np\n",
"import numba\n",
"from tqdm.notebook import tqdm\n",
"\n",
"\n",
"def load_eo_data():\n",
" return np.random.random((1000, 1000))\n",
"\n",
"@numba.jit\n",
"def smooth(x):\n",
" out = np.empty_like(x)\n",
" for i in range(1, x.shape[0] - 1):\n",
" for j in range(1, x.shape[1] - 1):\n",
" out[i, j] = (x[i + -1, j + -1] + x[i + -1, j + 0] + x[i + -1, j + 1] +\n",
" x[i + 0, j + -1] + x[i + 0, j + 0] + x[i + 0, j + 1] +\n",
" x[i + 1, j + -1] + x[i + 1, j + 0] + x[i + 1, j + 1]) // 9\n",
" return out\n",
"\n",
"def save(x, filename):\n",
" pass"
]
},
{
"cell_type": "markdown",
"id": "a44bb7e4-7f70-40a1-84cc-9b2f63007378",
"metadata": {},
"source": [
"Before running the next code, open a terminal window (File>New>Terminal) and run `htop` at the command line to show current CPU usage per core.\n",
"\n",
"> **TIP:** Drag your terminal window so that it sits below this notebook before you run `htop` to see both windows at the same time."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "88d0fc98-eb4a-445e-b305-84aa493d07b6",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"%%time\n",
"for i in tqdm(range(1000)):\n",
" img = load_eo_data()\n",
" img = smooth(img)\n",
" img = np.fft.fft2(img)\n",
" save(img, \"file-\" + str(i) + \"-.dat\")"
]
},
{
"cell_type": "markdown",
"id": "c2ef6e05-6466-4119-b33d-eaa64445ba24",
"metadata": {},
"source": [
"You'll notice that only one core is showing any load. The above code is not using any of the additional cores.\n",
"\n",
"`dask` can be useful in this scenario even on a local machine. "
]
},
{
"cell_type": "markdown",
"id": "d7fcafa1-75cf-4856-987d-bf1154203a01",
"metadata": {},
"source": [
"### Review of dask\n",
"\n",
"Firstly, a few notes on terminology. A Dask Cluster is comprised of a __client__, a __scheduler__, and __workers__. These terms will be used throughout this tutorial. Figure 1 below shows the relationship between each of these components. The __client__ submits tasks to the __scheduler__, which decides how to submit the tasks to individual workers. During this process, the scheduler creates what is called a __Task Graph__. This is essentially a map of the tasks that need to be carried out. Figure 2 shows an example of a simple task graph (see https://docs.dask.org/en/stable/graphs.html for more information). __Workers__ carry out the actual calculations and either store the results or send them back to the client.\n",
"\n",
"<div>\n",
" <span style=\"border:solid 1px #888;float:left;padding:10px;margin-right:25px;width:550px\">\n",
" <img src=\"../../resources/distributed-overview.png\">\n",
" <figcaption><em>Figure 1. Overview of a Dask Cluster.</em></figcaption>\n",
" </span>\n",
" <span style=\"border:solid 1px #888;float:left;padding:10px;margin-left:25px;width:150px\">\n",
" <img style=\"float:left\" src=\"../../resources/dask-simple.png\">\n",
" <figcaption><em>Figure 2. A simple Task Graph.</em></figcaption>\n",
" </span>\n",
"</div>"
]
},
{
"cell_type": "markdown",
"id": "5d4cae0a-7557-440e-b6b0-54ffd968cced",
"metadata": {},
"source": [
"Dask has several core data types, including __Dask DataFrames__ and __Dask Arrays__. Essentially, Dask DataFrames are parallelized Pandas DataFrames (Figure 3) and Dask Arrays are parallelized Numpy arrays (Figure 4).\n",
"\n",
"<div style=\"width:100%\">\n",
" <span style=\"border:solid 1px #888;float:left;padding:10px;margin-right:25px;width:200px\">\n",
" <img src=\"../../resources/dask-dataframe.svg\">\n",
" <figcaption><em>Figure 3. A Dask DataFrame is comprised of many in-memory pandas DataFrames separated along an index.</em></figcaption>\n",
" </span>\n",
" <span style=\"border:solid 1px #888;float:left;padding:10px;margin-left:25px;width:300px\">\n",
" <img style=\"float:left\" src=\"../../resources/dask-array.svg\">\n",
" <figcaption><em>Figure 4. A Dask Array is a subset of the NumPy <code>ndarray</code> interface using blocked algorithms, cutting up the large array into many small arrays.</em></figcaption>\n",
" </span>\n",
"</div>"
]
},
{
"cell_type": "markdown",
"id": "d3f64100-b5cc-48a8-9be9-71f31e50dd35",
"metadata": {},
"source": [
"EASI and the Open Data Cube primarily make use of __Dask DataArrays__. For more information see https://tutorial.dask.org/02_array.html."
]
},
{
"cell_type": "markdown",
"id": "823a9c12-1c03-4ea8-8a9e-ec498bdbc056",
"metadata": {},
"source": [
"### Dask local cluster\n",
"\n",
"Let's start by creating a local dask cluster."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e0d95f91-a9e0-44c1-a577-f8e708ccbce6",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"from dask.distributed import Client, LocalCluster, fire_and_forget\n",
"\n",
"cluster = LocalCluster()\n",
"client = Client(cluster)\n",
"client"
]
},
{
"cell_type": "markdown",
"id": "e96378b8-ec25-4970-9957-8c482112073a",
"metadata": {},
"source": [
"There is also a utility function in `notebook_utils` which you can use to initialize a local dask cluster. This funtion can also be used to initialize a remote cluster using Dask Gateway, but please complete the other Dask tutorials before using a remote cluster.\n",
"\n",
"The following lines will initialize and display a local dask cluster and can replace the code above.\n",
"\n",
"```python\n",
"cluster, client = notebook_utils.initialize_dask(use_gateway=False, wait=True)\n",
"display(cluster if cluster else client)\n",
"```"
]
},
{
"cell_type": "markdown",
"id": "922c7673-88cf-445c-90c7-ea4c6c2124d9",
"metadata": {},
"source": [
"The Dask Dashboard url will show as \"localhost\" or \"127.0.0.1\" since its running locally in the Jupyter kernel. The dashboard can be accessed via the Jupyter server proxy using the following url:"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "31a07d9e-3b83-4081-9db2-cd70ac3853d9",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"notebook_utils.localcluster_dashboard(client=client, server=easi.hub)"
]
},
{
"cell_type": "markdown",
"id": "bd3c526a-c04f-40ac-b2dc-46701f9ec739",
"metadata": {},
"source": [
"You will want to have this open when running the next cell.\n",
"\n",
"We modify our application to `submit` functions to run on the dask cluster:"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c8b09db6-aea0-4877-a0e8-4cc2497c8936",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"%%time\n",
"\n",
"for i in tqdm(range(1000)):\n",
" img = client.submit(load_eo_data, pure=False)\n",
" img = client.submit(smooth, img)\n",
" img = client.submit(np.fft.fft2, img)\n",
" future = client.submit(save, img, \"file-\" + str(i) + \"-.dat\")\n",
" fire_and_forget(future)"
]
},
{
"cell_type": "markdown",
"id": "032d7d77-3e37-4fb5-bd9d-fa7f1c6db1ff",
"metadata": {},
"source": [
"If you watch `htop` in the terminal you'll see all cores become active. The dask dashboard will also provide a view of the tasks being run in parallel."
]
},
{
"cell_type": "markdown",
"id": "ef6c1656-f68b-4192-a88d-21a8d0b78c7e",
"metadata": {},
"source": [
"You will also see that the cluster remains busy after the cell above finishes. This is because Dask is working in the background processing in parallel the tasks that have been submitted to it."
]
},
{
"cell_type": "markdown",
"id": "bd40a084-1721-4d10-9400-e2bd563f5823",
"metadata": {},
"source": [
"A `dask.distributed.LocalCluster()` will shutdown when this notebook kernel is stopped.\n",
"Still it's a good practice to close the client and the cluster so its all cleaned up. This will be more important when using dask distributed clusters as they are independent of the notebook kernel."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "70edbc54-df60-4f33-8b56-9a10a8bb3fa7",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"client.close()\n",
"\n",
"cluster.close()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8c6e908d-1467-4246-bf12-e1a2cfd07be2",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.6"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
notebooks/dask/02-ODC_with_Dask.ipynb
+1201
View File
File diff suppressed because it is too large Load Diff
notebooks/dask/03-Larger_than_RAM_with_LocalCluster.ipynb
+570
View File
@@ -0,0 +1,570 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "54240a84-8659-4d34-af38-1249129a221a",
"metadata": {},
"source": [
"# Dask Local Cluster - Larger than memory computation <img align=\"right\" src=\"../../resources/csiro_easi_logo.png\">\n",
"\n",
"In the ODC and Dask (LocalCluster) notebook we saw how dask can be used to speed up IO and computation by parallelising operations into _chunks_ and _tasks_, and using _delayed tasks_ and _task graph_ optimization to remove redundant tasks when results are not used.\n",
"\n",
"Using _chunks_ provides one additional capability beyond parallelisation - _the ability to perform computations that are larger than available memory_.\n",
"\n",
"Since dask operations are performed on _chunks_ it is possible for dask to perform operations on smaller pieces that each fit into memory. This is particularly useful if you have a large amount of data that is being reduced, say by performing a seasonal mean.\n",
"\n",
"As with parallelisation, not all algorithms are amenable to being broken into smaller pieces so this won't always be possible. Dask arrays though go a long way to make this easier for a great many operations."
]
},
{
"cell_type": "markdown",
"id": "8d810c6f-46aa-48b0-a7a9-82151b29bb96",
"metadata": {},
"source": [
"Firstly, some initial imports..."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "10bf42b0-b251-4d68-9af0-cf9ab988c886",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"import git\n",
"import sys, os\n",
"from dateutil.parser import parse\n",
"from dateutil.relativedelta import relativedelta\n",
"from dask.distributed import Client, LocalCluster\n",
"import datacube\n",
"from datacube.utils import masking\n",
"from datacube.utils.aws import configure_s3_access\n",
"\n",
"# EASI defaults\n",
"os.environ['USE_PYGEOS'] = '0'\n",
"repo = git.Repo('.', search_parent_directories=True).working_tree_dir\n",
"if repo not in sys.path: sys.path.append(repo)\n",
"from easi_tools import EasiDefaults, notebook_utils\n",
"easi = EasiDefaults()"
]
},
{
"cell_type": "markdown",
"id": "c34ae885-c6bb-4d90-8769-e9a41d8b92cb",
"metadata": {},
"source": [
"We'll continue using the same algorithm as before but this time we're going to modify it's memory usage to exceed the LocalCluster's available memory. This example notebook is setup to run on a compute node with 28 GiB of available memory and 8 cores for the LocalCluster. We'll make that explicit here in case you are blessed with a larger number of resources.\n",
"\n",
"Let's start the cluster..."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2608053d-83db-45fe-9b09-9ea089edaf37",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"cluster = LocalCluster(n_workers=2, threads_per_worker=4)\n",
"cluster.scale(n=2, memory=\"14GiB\")\n",
"client = Client(cluster)\n",
"client"
]
},
{
"cell_type": "markdown",
"id": "a69fa086-3587-4717-9466-8eb3ca6a7fcb",
"metadata": {},
"source": [
"We can monitor memory usage on the workers using the dask dashboard URL below and the Status tab. The workers are local so this will be memory on the same compute node that Jupyter is running in. "
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7303b0e2-abd8-475c-ae4c-14de9e7a3c3d",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"dashboard_address = notebook_utils.localcluster_dashboard(client=client,server=easi.hub)\n",
"print(dashboard_address)"
]
},
{
"cell_type": "markdown",
"id": "25bbfd77-faf4-4834-99ef-7688911e02aa",
"metadata": {},
"source": [
"As we will be using __Requester Pays__ buckets in AWS S3, we need to run the `configure_s3_access()` function below with the `client` option to ensure that Jupyter and the cluster have the correct permissions to be able to access the data."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "bc032bf6-2105-4237-9121-0f557c7f65a6",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"dc = datacube.Datacube()\n",
"configure_s3_access(aws_unsigned=False, requester_pays=True, client=client);"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "026408c9-f44c-4270-b7fa-325646f22d99",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Get the centroid of the coordinates of the default extents\n",
"central_lat = sum(easi.latitude)/2\n",
"central_lon = sum(easi.longitude)/2\n",
"# central_lat = -42.019\n",
"# central_lon = 146.615\n",
"\n",
"# Set the buffer to load around the central coordinates\n",
"# This is a radial distance for the bbox to actual area so bbox 2x buffer in both dimensions\n",
"buffer = 0.05\n",
"\n",
"# Compute the bounding box for the study area\n",
"study_area_lat = (central_lat - buffer, central_lat + buffer)\n",
"study_area_lon = (central_lon - buffer, central_lon + buffer)\n",
"\n",
"# Data product\n",
"products = easi.product('landsat')\n",
"\n",
"# Set the date range to load data over\n",
"set_time = easi.time\n",
"set_time = (set_time[0], parse(set_time[0]) + relativedelta(years=1))\n",
"# set_time = (\"2021-01-01\", \"2021-12-31\")\n",
"\n",
"# Selected measurement names (used in this notebook). None` will load all of them\n",
"alias = easi.aliases('landsat')\n",
"measurements = None\n",
"# measurements = [alias[x] for x in ['qa_band', 'red', 'nir']]\n",
"\n",
"# Set the QA band name and mask values\n",
"qa_band = alias['qa_band']\n",
"qa_mask = easi.qa_mask('landsat')\n",
"\n",
"# Set the resampling method for the bands\n",
"resampling = {qa_band: \"nearest\", \"*\": \"average\"}\n",
"\n",
"# Set the coordinate reference system and output resolution\n",
"set_crs = easi.crs('landsat') # If defined, else None\n",
"set_resolution = easi.resolution('landsat') # If defined, else None\n",
"# set_crs = \"epsg:3577\"\n",
"# set_resolution = (-30, 30)\n",
"\n",
"# Set the scene group_by method\n",
"group_by = \"solar_day\""
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "4f52fb17-cd58-439a-921e-775ce94ebcca",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"dataset = None # clear results from any previous runs\n",
"dataset = dc.load(\n",
" product=products,\n",
" x=study_area_lon,\n",
" y=study_area_lat,\n",
" time=set_time,\n",
" measurements=measurements,\n",
" resampling=resampling,\n",
" output_crs=set_crs,\n",
" resolution=set_resolution,\n",
" dask_chunks = {\"time\":1},\n",
" group_by=group_by,\n",
" )\n",
"dataset"
]
},
{
"cell_type": "markdown",
"id": "41cd8e21-cdae-4ee2-a19e-5803617e85bf",
"metadata": {},
"source": [
"We can check the total size of the dataset using `nbytes`. We'll divide by 2**30 to have the result display in [gibibytes](https://simple.wikipedia.org/wiki/Gibibyte)."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e00f1009-9734-4bb0-b49a-4918af09af8e",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"print(f\"dataset size (GiB) {dataset.nbytes / 2**30:.2f}\")"
]
},
{
"cell_type": "markdown",
"id": "39a30454-290b-4f1f-a8ca-32c697b68b89",
"metadata": {},
"source": [
"As you can see this Region of Interest (ROI) and spatial range (1 year) is tiny, let's scale up by increasing our ROI by increasing the buffer\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "9d53a93b-a050-4d7c-bd50-fc83d35c1126",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"buffer = 1\n",
"\n",
"# Compute the bounding box for the study area\n",
"study_area_lat = (central_lat - buffer, central_lat + buffer)\n",
"study_area_lon = (central_lon - buffer, central_lon + buffer)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "16b66c21-f5e8-48de-bb6e-1ce34f885bd8",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"dataset = None # clear results from any previous runs\n",
"dataset = dc.load(\n",
" product=products,\n",
" x=study_area_lon,\n",
" y=study_area_lat,\n",
" time=set_time,\n",
" measurements=measurements,\n",
" resampling=resampling,\n",
" output_crs=set_crs,\n",
" resolution=set_resolution,\n",
" dask_chunks = {\"time\":1},\n",
" group_by=group_by,\n",
" )\n",
"print(f\"dataset size (GiB) {dataset.nbytes / 2**30:.2f}\")"
]
},
{
"cell_type": "markdown",
"id": "08eb169a-7fee-4703-8203-cc871e3b2bcc",
"metadata": {},
"source": [
"Okay, this should now be larger than the available memory that our Jupyter node has available (which you should be able to see at the bottom of your window - probably 24-30 GB). This creates issues for calculation. We need to have a solution that lets us calculate the information that we want without the machine running out of memory. \n",
"\n",
"Dask can compute many tasks and handle large amounts of data over the course of a series of calculations. Collectively, these calculations might work on more data in total than can fit in RAM, but it is a problem if the final product is too big to fit in RAM. Below we will change the dataset so that the final result can fit in RAM and then use the `.compute()` function to run all the calculations.\n",
"\n",
"Let's take a look at the memory usage for one of the bands, we'll use `red`."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "891babf5-678a-4a40-9502-2418b4820e5e",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"dataset[alias['red']]"
]
},
{
"cell_type": "markdown",
"id": "5ddd4c1c-647a-4a98-8af0-f10ef5f029d0",
"metadata": {},
"source": [
"You can see the year now has more time observations than in the first dataset because we've expanded the area of interest and picked up multiple satellite passes. The spatial dimensions are also much larger.\n",
"\n",
"Take a note of the _Chunk Bytes_ - probably around 80 MiB. This is the smallest unit of this dataset that dask will do work on. To do an NDVI calculation, dask will need two bands, the mask, the result and a few other temporary variables in memory at once. This means whilst this value is an indicator of memory required on a worker to perform an operation it is not the total, which will depend on the operation.\n",
"\n",
"We can adjust the amount of memory per chunk further by _chunking_ the spatial dimension. Let's split it into 2048x2048 size pieces."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "497dfc58-4c65-46e9-81b5-80aba1db87b1",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"dataset = None # clear results from any previous runs\n",
"dataset = dc.load(\n",
" product=products,\n",
" x=study_area_lon,\n",
" y=study_area_lat,\n",
" time=set_time,\n",
" measurements=measurements,\n",
" resampling=resampling,\n",
" output_crs=set_crs,\n",
" resolution=set_resolution,\n",
" dask_chunks = {\"time\":1, \"x\":2048, \"y\":2048}, ## Adjust the chunking spatially as well\n",
" group_by=group_by,\n",
" )\n",
"print(f\"dataset size (GiB) {dataset.nbytes / 2**30:.2f}\")"
]
},
{
"cell_type": "markdown",
"id": "2da60763-1813-43fa-857c-99153c99a6e5",
"metadata": {},
"source": [
"As you can see the total dataset size stays the same. \n",
"\n",
"Look at the `red` data variable below. You can see the chunk size has reduced to 8 MiB, and there are now more chunks (around 700-800) - compared with around 60 previously. This will result in a higher number of Tasks for Dask to work on. This makes sense: smaller chunks, more tasks.\n",
"\n",
"> __TIP__: The _relationship between tasks and chunks_ is a critical tuning parameter.\n",
"\n",
"Workers have limits in memory and compute capacity. The Dask Scheduler has limits in how many tasks it can manage efficiently (and remember it is tracking all of the data variables, not just this one). The trick with Dask is to give it a good number of chunks of data that aren not too big and don't result in too many tasks. There is always a trade-off and each calculation will be different. Ideally, you want chunks to be aligned with how the data is stored, or how the data is going to be used. If those two things are different, then rechunking can result in large amounts of data needing to be held in the cluster memory, which could result in failures. In the same way, if chunks are too large, they might end up taking up too much memory, causing a crash. This is sometimes down to trial and error.\n",
"\n",
"Later, when we move to a fully remote and distributed cluster, _chunks_ also become an important element in communicating between workers over networks.\n",
"\n",
"If you look carefully at the cube-like diagram in the summary below you will see that some internal lines showing the chunk boundaries for the spatial dimensions. 2048 wasn't an even multiplier so dask has made some chunks on the edges smaller. The specification of `chunks` is a guide: the actual data, numpy arrays in this case, are made into `chunk` sized shapes or smaller. These are called `blocks` in dask and represent the actual shape of the numpy array that will be processed.\n",
"\n",
"Somewhat confusingly the terms `blocks` and `chunks` are also used in dask literature and you'll need to check the context to see if it is referring to the _specification_ or the _actual block of data_. For the moment this differentiation doesn't matter but when performing low level custom operations knowing that your `blocks` might be a different shape does matter."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "32b96e79-6c54-4f03-91a8-6b4a840d3225",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"dataset[alias['red']]"
]
},
{
"cell_type": "markdown",
"id": "b7fa1249-f0af-4e83-a356-31cfc34e1c1c",
"metadata": {},
"source": [
"We won't worry to much about tuning these parameters right now and instead will focus on processing this larger dataset. As before we can exploit dask's ability to use _delayed_ tasks and apply our masking and NDVI directly to the full dataset. We'll also add an unweighted seasonal mean calculation using `groupby(\"time.season\").mean(\"time\")`. Dask will seek to complete the reductions (by chunk) first as they reduce memory usage.\n",
"\n",
"It's probably worth monitoring the dask cluster memory usage via the dashboard _Workers Memory_ to see just how little ram is actually used during this calculation despite it being performed on a large dataset."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6258f835-1f34-4dc1-8d99-e89d6233528f",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"print(dashboard_address)"
]
},
{
"cell_type": "markdown",
"id": "eb3b1aba-85d4-415d-84e3-745fb38138a2",
"metadata": {},
"source": [
"We will now calculate NDVI and group the results by season:"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e27e9923-6884-40d1-a53a-2babfae82a66",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Identify pixels that don't have cloud, cloud shadow or water\n",
"cloud_free_mask = masking.make_mask(dataset[qa_band], **qa_mask)\n",
"\n",
"# Apply the mask\n",
"cloud_free = dataset.where(cloud_free_mask)\n",
"\n",
"# Calculate the components that make up the NDVI calculation\n",
"band_diff = cloud_free[alias['nir']] - cloud_free[alias['red']]\n",
"band_sum = cloud_free[alias['nir']] + cloud_free[alias['red']]\n",
"# Calculate NDVI and store it as a measurement in the original dataset ta da\n",
"ndvi = None\n",
"ndvi = band_diff / band_sum\n",
"\n",
"ndvi_unweighted = ndvi.groupby(\"time.season\").mean(\"time\") # Calculate the seasonal mean"
]
},
{
"cell_type": "markdown",
"id": "874bfe1d-799c-4449-9b80-19bcc7f99061",
"metadata": {},
"source": [
"Let's check the shape of our result - it should have 4 seasons now instead of the individual dates."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "009724fc-3c43-4225-9404-85b01f94d1fa",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"ndvi_unweighted"
]
},
{
"cell_type": "markdown",
"id": "db88af8e-beac-4779-b07b-5b001c5505bb",
"metadata": {},
"source": [
"Before we do the `compute()` to get our result we should make sure the final result will fit in memory for the Jupyter kernel"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ced41b22-710e-4603-8f04-49c7f8849bca",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"print(f\"dataset size (GiB) {ndvi_unweighted.nbytes / 2**30:.2f}\")"
]
},
{
"cell_type": "markdown",
"id": "7dcb1346-242d-412e-b824-1ae6a6b1b2e9",
"metadata": {},
"source": [
"This shows that the resulting data should be around 1 GiB of data, which will fit in local memory.\n",
"\n",
"If you are monitoring the cluster when you run the cell below, you might notice a delay between running the next cell and actual computation occuring. Dask performs a _task graph optimisation_ step on the _client_ not the cluster. How long this takes depends on the number of tasks and complexity of the graph. The speed of this step has improved recently due to recent Dask updates. We'll talk more about this later.\n",
"\n",
"In the meantime, run the next cell and watch dask compute the result without running out of memory. You might notice that your cluster spills some data to disk (the grey part of the bars in the _Bytes stored per worker_ graph). This is not normally desirable and slows down the calculation (because reading and writing to/from the disk is slower than to/from RAM), but it is a mechanism used by Dask to help manage large calculations. \n",
"\n",
">__Tip:__ don't forget to look at your Dask Dashboard (URL a few cells above) to watch what is happening in your cluster"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b4d3915b-1ccc-4ece-9559-cd27295a14f5",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"%%time\n",
"actual_result = ndvi_unweighted.compute()"
]
},
{
"cell_type": "markdown",
"id": "181e6946-4a12-4d69-bf0a-3f02d64e2371",
"metadata": {},
"source": [
"To avoid northern/southern hemisphere differences, the `season` values are represented as acronyms of the months that make them up, so:\n",
"- December, January, February = DJF\n",
"- March, April, May = MAM\n",
"- June, July, August = JJA\n",
"- September, October, November = SON\n",
"\n",
"Let's plot the result for `DJF`. This will take a few seconds, the image is several thousand pixels across."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "289cfdb6-c5c7-4468-a379-9094a0003b82",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"actual_result.sel(season='DJF').plot(robust=True, size=6, aspect='equal')"
]
},
{
"cell_type": "markdown",
"id": "e757725f-c923-403a-8943-d79bb42dafc9",
"metadata": {},
"source": [
"Not the most useful visualisation as a small image, and a little slow. Dask can help with this too but that's a topic for another notebook. There are many other ways to work with Dask and optimize performance. This is just the beginning of how to manage large calculations."
]
},
{
"cell_type": "markdown",
"id": "75730dc0-0940-44e2-b656-747f6862915d",
"metadata": {
"tags": []
},
"source": [
"# Be a good dask user - Clean up the cluster resources"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "48eeb0ee-6888-4362-8615-19cf217e80c0",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"client.close()\n",
"\n",
"cluster.close()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "29d22aa2-f239-4a0a-8b8a-331ca32b09d8",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.6"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
notebooks/dask/04-Go_Big_or_Go_Home_Part_1.ipynb
+751
View File
@@ -0,0 +1,751 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "e374c84f-75e4-4859-bf4f-3a7847aef454",
"metadata": {},
"source": [
"# Go Big or Go Home Part 1 - Dask fully distributed <img align=\"right\" src=\"../../resources/csiro_easi_logo.png\">\n",
"\n",
"In the previous notebooks we've been using `dask.distributed.LocalCluster` to split up our tasks and run them on the same compute node that is running the notebook. We noted that this could then use all cores to run tasks in parallel, greatly speeding up loading, and thanks to _chunks_ we can also process _some_ algorithms, like our NDVI seasonal mean, on datasets larger than available RAM.\n",
"\n",
"But what happens if your algorithm and dataset are such they cannot fit the compute nodes' RAM, or the result of the calculation is also massive, or its just so big (memory and computation) that it takes hours to compute?\n",
"\n",
"Well, `dask.distributed.LocalCluster` is just one member of the `dask.distributed` cluster family. There are several others but the one we will be using is Kubernetes Cluster (`KubeCluster`). Kubernetes is an excellent technology that takes advantage of modern Cloud Computing architectures to automatically provision (and remove) compute nodes on demand. It does a lot of other stuff well beyond the scope of this dask tutorial of course. The important point is that using `KubeCluster` we can dramatically expand the number of Compute Nodes to schedule our dask Workers to; potentially very dramatically.\n",
"\n",
"In this notebook we'll expand our NDVI seasonal mean calculation to a larger area and take it back through two decades of observations. Along the way we'll explore the dask data structures, how computation proceeds, and what we can do to tune the performance. At this spatial size we'll also look at how to interactively visualise a result that is larger than the Jupyter notebook can handle.\n",
"\n",
"Everything we do next builds on the concepts of _chunks_, _tasks_, _data locality_, and _task graph_ covered previously."
]
},
{
"cell_type": "markdown",
"id": "5a59f381-17c5-49c1-9579-00a1a6f22e4b",
"metadata": {
"tags": []
},
"source": [
"## Dask Gateway and remote dask schedulers\n",
"\n",
"Once we have multiple compute nodes to run workers on we have a lot more moving parts in the system. These parts are also fully distributed and will need to communicate between each other to pass results, perform tasks, confirm completion of tasks and ask for more work, etc. It is important to understand what the components are and how they interact because it can impact both _performance_ and _stablity_ of calculation, particularly at very large scales. In addition, _data locality_ really matters when your dataset is large - you don't want to `compute()` a 1 TB result and have it brought back to the Jupyter kernel on a 32 GiB machine! There are also subtleties to be aware of in how data gets from your Jupyter notebook to the dask distributed nodes - it has to be communicated somehow.\n",
"\n",
"This can all be a bit overwhelming to think about. Thankfully, you don't hit all of these at once as dask does good job of hiding many of the details but then remember our two \"laws\" of dask from the first notebook:\n",
"1. The best thing about dask is it makes distributed parallel programming in the datacube easy\n",
"1. The worst thing about dask is it makes distributed parallel programming in the datacube easy\n",
"\n",
"The transition point from gain to pain, and back to gain, is connected to these details. So let's define out various parts and their roles and start building our knowledge."
]
},
{
"cell_type": "markdown",
"id": "7b75e69d-237d-4411-bd88-59e2e7346826",
"metadata": {},
"source": [
"### Kubernetes\n",
"\n",
"In a Kubernetes environment all programs that execute things run in __Pods__. What a _pod_ is and how it works is a subject for another course and you can use an internet search to find out more. For our purposes it is sufficient to understand that the Jupyter notebook, the dask scheduler, the dask workers, and all the components that make this work are running in _Pods_.\n",
"\n",
"* _Pods_ have resources - memory, cpu, gpu, storage.\n",
"* _Pods_ request resources and have resource limits.\n",
"* _Pods_ communicate to each other over a network.\n",
"* You can think of a _pod_ as being a kind of virtual PC on which you can run your programs.\n",
"\n",
"_Pods_ run on _Compute Nodes_ - physical hardware with an actual CPU, GPU, memory and storage. _Compute Nodes_ can run more than one _Pod_ so long as the sum of all the requests will fit. For example, if your _Compute Node_ has 64 GiB of RAM and your _worker pods_ request 14 GiB each, then 4 _workers Pods_ will run on 1 _Compute Node_.\n",
"\n",
"Thankfully, you don't need to figure out what Pods get placed where as Kubernetes will do that automatically. There is more to be said about this relationship and the impacts on performance and operational cost but for now just note that _Pods_ are where your code and data lives and they have requests and limits which you can control.\n",
"\n",
"This diagram shows a single user (_Joe_) running a Jupyter Notebook and connected to a single dask cluster with 5 Workers (running on 3 Worker Nodes).\n",
"\n",
"![Dask Cluster](../../resources/DaskCluster-ODCandDask.svg)"
]
},
{
"cell_type": "markdown",
"id": "865c9853-fe04-439c-858f-4f2777eb4c48",
"metadata": {},
"source": [
"The __Jupyter notebook__ is where your code is typed in. It has a Python kernel of its own and will be the __dask client__ that talks to the __dask cluster__. It is running in a Pod, as are all the components. So the Jupyter notebook is _separate_ from the other components in the system and communicates over a network.\n",
"\n",
"The __dask cluster__ is the __dask scheduler__ plus the group of __dask workers__ that process the tasks in a distributed manner. The _scheduler_ and the _workers_ are all Pods, which means they are _separate_ from each other and communicate over a network. This is different to `dask.distributed.LocalCluster` in which the Jupyter notebook, dask scheduler and workers all resided on the same machine and all communicated _very_ rapidly on the local machine's communications channel. Now they can be on entirely different _compute nodes_ and are _communicating over a much slower network_. We have the benefit of more compute resources, at the cost of slower communication.\n",
"\n",
"The __dask gateway__ is a new component and is used to manage __dask clusters__ (note that is plural). The __Jupyter notebook__ acts as a client to the _dask gateway_ and makes requests for a __dask cluster__ (both the _scheduler_ and the _workers_) to the _dask gateway_ to create and destroy them. The _dask gateway_ manages the lifecycle of the cluster on the user's behalf.\n",
"\n",
"> __Tip__: This means the dask clusters have an independent life cycle compared with the Jupyter notebook. Quitting your Jupyter notebook will not necessarily quit your dask cluster.\n",
"\n",
"Moreover, you can have more than one Jupyter notebook talking to the _same_ dask cluster simultaneously. There are some good reasons for doing this but we won't be touching on them in this course."
]
},
{
"cell_type": "markdown",
"id": "4770317f-ec1b-4365-9a23-d4ef684a1a11",
"metadata": {},
"source": [
"### Running our NDVI seasonal mean on the remote dask cluster\n",
"\n",
"Let's move our NDVI seasonal mean from the `LocalCluster` to our _dask gateway_ managed cluster, and add some extra compute resources to it.\n",
"\n",
"The biggest change here is simply how we start and shutdown the dask cluster. The rest of the code, to do the actual computation, is _exactly_ the same.\n",
"\n",
"The first thing we need to do is create a client so we can connect to the _dask gateway_.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c1fdc377-b388-44bc-ae3b-234848cd620f",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Initialize the Gateway client\n",
"from dask.distributed import Client\n",
"from dask_gateway import Gateway\n",
"\n",
"gateway = Gateway()"
]
},
{
"cell_type": "markdown",
"id": "f2529f3c-3b04-4bc0-a036-aab65a3602c9",
"metadata": {},
"source": [
"Easy! We now have a `gateway` client variable. Using this we can start clusters, stop clusters, ask for a list of clusters we have running, set options for our scheduler and workers (like cpu and memory requests).\n",
"\n",
"Let's see what the `cluster_options` are. We don't need to guess, we can ask the `gateway`"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c3fec8f2-91bf-495f-9493-21cfea00f3b7",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"gateway.cluster_options()"
]
},
{
"cell_type": "markdown",
"id": "4d0a08ee-c3aa-4b5a-bda8-20f6fc6b47b0",
"metadata": {},
"source": [
"That's a lot I know. The majority of these don't need to change and most users will simply tweak _worker_ parameters: _cores_, _threads_ (probably keeping it the same as _cores_), _memory_ and the _worker group_.\n",
"\n",
"We will be using the defaults for now."
]
},
{
"cell_type": "markdown",
"id": "8da34ddb-f856-4234-9ef9-cf45095a83ba",
"metadata": {},
"source": [
"### Create the Cluster\n",
"\n",
"Create the cluster with default options if it doesn't already exist. If a cluster exists in your namespace, the code below will connect to the first cluster. List the available clusters with `gateway.list_clusters()`.\n",
"\n",
"The cluster creation may take a little while (minutes) if a suitable _node_ isn't available for the _scheduler_. The same thing will occur for _workers_ when they start. If a _node_ does exist then this can happens in seconds.\n",
"\n",
"> __Tip__: Users are often confused by the changing start up time and think something is wrong.\n",
"\n",
"It can take _minutes_ for a brand new _node_ to be provisioned, please be patient. If it takes 10 minutes then yes, something is wrong and you should probably contact an administrator of the system if that problem persists."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7dc86e6f-6afb-4a8b-83fd-48bed81bd142",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"clusters = gateway.list_clusters()\n",
"if not clusters:\n",
" print('Creating new cluster. Please wait for this to finish.')\n",
" cluster = gateway.new_cluster()\n",
"else:\n",
" print(f'An existing cluster was found. Connecting to: {clusters[0].name}')\n",
" cluster=gateway.connect(clusters[0].name)\n",
"cluster"
]
},
{
"cell_type": "markdown",
"id": "eb7174b3-3e78-4496-9e09-7200aef5780b",
"metadata": {},
"source": [
"### Scale the cluster\n",
"\n",
"Use the GatewayCluster widget (above) to adjust the cluster size. Alternatively use the cluster API methods.\n",
"\n",
"For many tasks 1 or 2 workers will be sufficient, although for larger areas or more complex tasks 5 to 10 workers may be used. If you are new to Dask, start with one worker and then scale your cluster if needed.\n",
"\n",
"In this notebook we'll start with 4 workers - that's 4x the resources for workers compared to our previous `LocalCluster`. In addition the _scheduler_ is also on its own node, and so is the Jupyter notebook kernel. Lots more resources for all the components involved.\n",
"\n",
"The next cell will use the cluster API to add 4 workers programmatically."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "52629b33-88e4-4f2a-8388-9213e4808e2f",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"cluster.scale(4)"
]
},
{
"cell_type": "markdown",
"id": "a3edb753-4309-4196-b2f4-1d10dc0ea93b",
"metadata": {},
"source": [
"### Connect to the cluster\n",
"To connect to your cluster and start doing work, use the `get_client()` method. This step will wait until the workers are ready. You don't actually have to wait for the workers. The Jupyter notebook can be doing other things whilst the workers are coming up. We're waiting in this example so you don't end up with an unexpected wait later.\n",
"\n",
"***This may take a few minutes before your workers will be ready to use. Please wait for the cell to finish and show you the Dask Client.***"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b11f3b60-0981-4090-bb29-a810dbdf15ea",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"client = cluster.get_client()\n",
"# client.wait_for_workers(n_workers=4) # Before release 2023.10.0\n",
"client.sync(client._wait_for_workers,n_workers=4) # Since release 2023.10.0\n",
"client"
]
},
{
"cell_type": "markdown",
"id": "fcefb34f-b1fa-4774-92e9-836bbb6f21b6",
"metadata": {},
"source": [
"The client widget provides a clickable __dask dashboard__ link so click that and you'll see your dashboard. It works the same as before despite the fact that everything is now running in a distributed manner. If you click the _Workers_ tab in the _dashboard_ you will see that we now have 32 cores (up from 8) made up of 4x 8-core workers. Lot's of RAM too.\n",
"\n",
"Go back to the _Status_, so you can watch everything run."
]
},
{
"cell_type": "markdown",
"id": "62379a64-d3ae-43b9-ae09-20024b66e0f9",
"metadata": {},
"source": [
"### Perform the computation\n",
"\n",
"This is the same as in [dask tutorial 03](./03_-_Larger_than_RAM_(LocalCluster).ipynb).\n",
"\n",
"We don't need to change any of our code to run this now, so let's repeat the full calculation.\n",
"\n",
"As we will be using __Requester Pays__ buckets in AWS S3, we need to run the `configure_s3_access()` function below with the `client` option to ensure that Jupyter and the cluster have the correct permissions to be able to access the data."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "5896362b-4d9e-4356-af33-2c2e8a0d07e0",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"import git\n",
"import sys, os\n",
"from dateutil.parser import parse\n",
"from dateutil.relativedelta import relativedelta\n",
"from dask.distributed import Client, LocalCluster\n",
"import datacube\n",
"from datacube.utils import masking\n",
"from datacube.utils.aws import configure_s3_access\n",
"\n",
"# EASI defaults\n",
"os.environ['USE_PYGEOS'] = '0'\n",
"repo = git.Repo('.', search_parent_directories=True).working_tree_dir\n",
"if repo not in sys.path: sys.path.append(repo)\n",
"from easi_tools import EasiDefaults, notebook_utils\n",
"easi = EasiDefaults()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "df12b923-dc2e-4ebe-ad54-59ef7e1402e4",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"dc = datacube.Datacube()\n",
"configure_s3_access(aws_unsigned=False, requester_pays=True, client=client)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c7e594cc-cfcb-41f2-b282-cefc14be13d7",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Get the centroid of the coordinates of the default extents\n",
"central_lat = sum(easi.latitude)/2\n",
"central_lon = sum(easi.longitude)/2\n",
"# central_lat = -42.019\n",
"# central_lon = 146.615\n",
"\n",
"# Set the buffer to load around the central coordinates\n",
"# This is a radial distance for the bbox to actual area so bbox 2x buffer in both dimensions\n",
"buffer = 0.8\n",
"\n",
"# Compute the bounding box for the study area\n",
"study_area_lat = (central_lat - buffer, central_lat + buffer)\n",
"study_area_lon = (central_lon - buffer, central_lon + buffer)\n",
"\n",
"# Data product\n",
"products = easi.product('landsat')\n",
"\n",
"# Set the date range to load data over\n",
"set_time = easi.time\n",
"set_time = (set_time[0], parse(set_time[0]) + relativedelta(years=1))\n",
"# set_time = (\"2021-01-01\", \"2021-12-31\")\n",
"\n",
"# Selected measurement names (used in this notebook). None` will load all of them\n",
"alias = easi.aliases('landsat')\n",
"measurements = None\n",
"# measurements = [alias[x] for x in ['qa_band', 'red', 'nir']]\n",
"\n",
"# Set the QA band name and mask values\n",
"qa_band = alias['qa_band']\n",
"qa_mask = easi.qa_mask('landsat')\n",
"\n",
"# Set the resampling method for the bands\n",
"resampling = {qa_band: \"nearest\", \"*\": \"average\"}\n",
"\n",
"# Set the coordinate reference system and output resolution\n",
"set_crs = easi.crs('landsat') # If defined, else None\n",
"set_resolution = easi.resolution('landsat') # If defined, else None\n",
"# set_crs = \"epsg:3577\"\n",
"# set_resolution = (-30, 30)\n",
"\n",
"# Set the scene group_by method\n",
"group_by = \"solar_day\""
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2abeb772-f4c2-4392-968b-5d5a6fd6f0f4",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"dataset = None # clear results from any previous runs\n",
"dataset = dc.load(\n",
" product=products,\n",
" x=study_area_lon,\n",
" y=study_area_lat,\n",
" time=set_time,\n",
" measurements=measurements,\n",
" resampling=resampling,\n",
" output_crs=set_crs,\n",
" resolution=set_resolution,\n",
" dask_chunks = {\"time\":1, \"x\":2048, \"y\":2048}, ## No change here, chunking spatially just like before\n",
" group_by=group_by,\n",
" )\n",
"print(f\"dataset size (GiB) {dataset.nbytes / 2**30:.2f}\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "04b3c928-43f0-4f53-8f10-6a64e0301c9b",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Identify pixels that are either \"valid\", \"water\" or \"snow\"\n",
"cloud_free_mask = masking.make_mask(dataset[qa_band], **qa_mask)\n",
"\n",
"# Apply the mask\n",
"cloud_free = dataset.where(cloud_free_mask)\n",
"\n",
"# Calculate the components that make up the NDVI calculation\n",
"band_diff = cloud_free[alias['nir']] - cloud_free[alias['red']]\n",
"band_sum = cloud_free[alias['nir']] + cloud_free[alias['red']]\n",
"# Calculate NDVI and store it as a measurement in the original dataset ta da\n",
"ndvi = None\n",
"ndvi = band_diff / band_sum\n",
"\n",
"ndvi_unweighted = ndvi.groupby(\"time.season\").mean(\"time\") # Calculate the seasonal mean"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "39611d88-6e76-4ed6-a4da-fad1bc496287",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"%%time\n",
"actual_result = ndvi_unweighted.compute()"
]
},
{
"cell_type": "markdown",
"id": "6a501db5-d805-4b26-b511-ec2f8d169a52",
"metadata": {},
"source": [
"As before there will be a short delay as the Jupyter kernel (_client_) is used to optimize the task graph before sending it to the _scheduler_ which will then execute _tasks_ on the _workers_.\n",
"\n",
"> __Tip__: You can open a terminal and use `htop` to monitor the Jupyter notebook CPU usage. You'll see at least one core using nearly 100% cpu usage during the optimisation phase. It will then drop back to idle as the _task graph_ is sent to the _scheduler_ at which point the dask dashboard will show activity on the cluster."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6f9845f5-c3dd-4371-9a50-1fb4e8da2180",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"actual_result.sel(season='DJF').plot(robust=True, size=6, aspect='equal')"
]
},
{
"cell_type": "markdown",
"id": "3f463907-1920-4b3c-a33b-0d29467eefae",
"metadata": {
"tags": []
},
"source": [
"### Exploiting our new resources - adjusting our chunk size\n",
"\n",
"Before we \"Go Big\" let's take advantage of our new resources.\n",
"\n",
"We've gained memory. The more memory we have the more data we can operate on at once _and the less we need to communicate between nodes_. Communication over a network is slow relative to local communcation in a _pod_. So if we change the _chunking_ we may see an improvement in performance.\n",
"\n",
"_Chunking_ will also impact the number of tasks - the fewer chunks, the fewer tasks. This in turn will impact how much _task graph optimization_ is required, how hard the _scheduler_ has to work, and how much communication of partial results goes between workers (for example, passing the partial means around to get a final mean).\n",
"\n",
"Let's look at our existing chunk size - `(1,2048,2048)`"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f0d65191-67aa-4919-8b32-8b4762d02a41",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"dataset[alias['red']]"
]
},
{
"cell_type": "markdown",
"id": "c50815c1-fbc8-4853-9539-446e39abb3a7",
"metadata": {},
"source": [
"Each `red` chunk is currently 8 MiB in size. Of course this will vary with data type and stage of computation so when tuning dask you may need to check on the _chunk size_ and _tasks_ as your computation transforms the data. We're doing a simple computation here so we can focus on this initial value. With experience it does get easier to figure out when and where this parameter needs further adjustment. We'll look at some of this later.\n",
"\n",
"For now there are some things we can observe:\n",
"1. 8 MiB is pretty small when our 4 workers have 32 Gigs each so we have room to grow even with all the temporaries to allow for.\n",
" * _We should monitor the worker memory usage (shown as `Bytes stored per worker` in the dashboard as we make changes to ensure none are spilling to disk (shown in grey) as that will slow things down and is unnecessary in this case_\n",
"1. Geospatial operations - the data load, the reprojection, even the masking - may benefit from having a larger spatial area.\n",
" * No point going too large though as the satellite paths have a finite width and we'll just have lots of empty space.\n",
"1. The computation involves a seasonal mean, which means some temporal grouping might improve performance.\n",
" * That said, _chunks_ are a unit for communication and it may mean that we're passing more information around than is necessary if we group too much together across the seasonal boundaries.\n",
"\n",
"So we have good reason to increase our chunks both spatially and temporally; just be mindful of the impact on communication of results between nodes. The mean is seasonal and Landsat 8 performs repeat passes nominally every 16 days, so let's do a small grouping in time. We'll also increase the spatial chunking slightly."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "acbc9f88-7b72-48e0-a1e3-700273a85cfe",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"dataset = None # clear results from any previous runs\n",
"dataset = dc.load(\n",
" product=products,\n",
" x=study_area_lon,\n",
" y=study_area_lat,\n",
" time=set_time,\n",
" measurements=measurements,\n",
" resampling=resampling,\n",
" output_crs=set_crs,\n",
" resolution=set_resolution,\n",
" dask_chunks = {\"time\":2, \"x\":3072, \"y\":3072}, # This line has changed\n",
" group_by=group_by,\n",
" )"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "872da0b0-9a9f-454f-81e7-eac5cd2e3306",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"dataset[alias['red']]"
]
},
{
"cell_type": "markdown",
"id": "67a323f5-fc2a-4167-925e-ecd607812bd7",
"metadata": {},
"source": [
"As you can see the number of tasks has dropped and our chunks are larger at 36 MiB.\n",
"\n",
"There is a small slither along the bottom because the chunk size isn't a good fit for the actual array size. Let's expand our chunk size in that direction slightly to give us a better fit."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "42b77c6d-4ea3-4198-bc91-ed00aa20f4b3",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"from math import ceil\n",
"\n",
"y_chunk = ceil(dataset.dims['y']/2)\n",
"print(f'Y-dim chunk: {y_chunk}')"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f9d92fab",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"dataset = None # clear results from any previous runs\n",
"dataset = dc.load(\n",
" product=products,\n",
" x=study_area_lon,\n",
" y=study_area_lat,\n",
" time=set_time,\n",
" measurements=measurements,\n",
" resampling=resampling,\n",
" output_crs=set_crs,\n",
" resolution=set_resolution,\n",
" dask_chunks = {\"time\":2, \"x\":3072, \"y\":y_chunk},\n",
" group_by=group_by,\n",
" )\n",
"dataset[alias['red']]"
]
},
{
"cell_type": "markdown",
"id": "809561da-7fe9-4a0e-8ec8-0f7ee28564a0",
"metadata": {},
"source": [
"Notice how that small change to remove the sliver only marginally increased our chunk memory usage but dramatically reduced the number of chunks.\n",
"\n",
"Let's see what this does to our performance. We need to re-run the code to update all the intermediate variables in our calculation and call `compute()`"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a2ec7285-fa1d-479c-8bf8-b58b2785195a",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Identify pixels that are either \"valid\", \"water\" or \"snow\"\n",
"cloud_free_mask = masking.make_mask(dataset[qa_band], **qa_mask)\n",
"\n",
"# Apply the mask\n",
"cloud_free = dataset.where(cloud_free_mask)\n",
"\n",
"# Calculate the components that make up the NDVI calculation\n",
"band_diff = cloud_free[alias['nir']] - cloud_free[alias['red']]\n",
"band_sum = cloud_free[alias['nir']] + cloud_free[alias['red']]\n",
"# Calculate NDVI and store it as a measurement in the original dataset ta da\n",
"ndvi = None\n",
"ndvi = band_diff / band_sum\n",
"\n",
"ndvi_unweighted = ndvi.groupby(\"time.season\").mean(\"time\") # Calculate the seasonal mean"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1818d321-3019-4ef2-a965-cd99deed996e",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"%%time\n",
"actual_result = ndvi_unweighted.compute()"
]
},
{
"cell_type": "markdown",
"id": "b08376e4-2473-4f63-a80e-da65347a1d8e",
"metadata": {},
"source": [
"You will notice the time for _task graph optimization_ - the delay between executing the cell above and seeing processing in the cluster dashboard - is down significantly. Fewer tasks means less time in optimization. We've decreased the computation time as well.\n",
"\n",
"There is one more thing we can do before we \"Go Big\". We've done this before and its simple enough. Save dask the challenge of figuring out which measurements we aren't using by telling it only to load the ones we do use. Let's add our measurements list in."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "37b423b9-a28e-48e2-81c5-b2524802fb05",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"measurements = [alias[x] for x in ['qa_band', 'red', 'nir']]\n",
"\n",
"dataset = None # clear results from any previous runs\n",
"dataset = dc.load(\n",
" product=products,\n",
" x=study_area_lon,\n",
" y=study_area_lat,\n",
" time=set_time,\n",
" measurements=measurements,\n",
" resampling=resampling,\n",
" output_crs=set_crs,\n",
" resolution=set_resolution,\n",
" dask_chunks = {\"time\":2, \"x\":3072, \"y\":y_chunk},\n",
" group_by=group_by,\n",
" )\n",
"\n",
"# Identify pixels that are either \"valid\", \"water\" or \"snow\"\n",
"cloud_free_mask = masking.make_mask(dataset[qa_band], **qa_mask)\n",
"\n",
"# Apply the mask\n",
"cloud_free = dataset.where(cloud_free_mask)\n",
"\n",
"# Calculate the components that make up the NDVI calculation\n",
"band_diff = cloud_free[alias['nir']] - cloud_free[alias['red']]\n",
"band_sum = cloud_free[alias['nir']] + cloud_free[alias['red']]\n",
"# Calculate NDVI and store it as a measurement in the original dataset ta da\n",
"ndvi = None\n",
"ndvi = band_diff / band_sum\n",
"\n",
"ndvi_unweighted = ndvi.groupby(\"time.season\").mean(\"time\") # Calculate the seasonal mean"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "07d8c1f7-99dd-4907-8cda-b268bb371623",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"%%time\n",
"actual_result = ndvi_unweighted.compute()"
]
},
{
"cell_type": "markdown",
"id": "e13f71e8-63bd-4fc6-b6a4-cb95109377aa",
"metadata": {},
"source": [
"This didn't make to much difference to computational time but it has shortened the _task graph optimisation_ phase a little more. That time isn't a problem in this example but as we \"Go Big\" it will be.\n",
"\n",
"Now you can continue on to [Part 2](./05_-_Go_Big_or_Go_Home_Part_2.ipynb) of this part to test out a much bigger area."
]
},
{
"cell_type": "markdown",
"id": "0cfcad62-a6e4-4504-9e41-d33f872ade7a",
"metadata": {},
"source": [
"# Be a good dask user - Clean up the cluster resources"
]
},
{
"cell_type": "markdown",
"id": "66acca97-686c-4d09-ba5b-a9aeede4bfba",
"metadata": {},
"source": [
"Disconnecting your client is good practice, but the cluster will still be up so we need to shut it down as well"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0578682e-917e-49b7-9f3e-88584a572936",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"client.close()\n",
"\n",
"cluster.shutdown()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "4dfc916f-ad27-44cb-9381-6c7179d6ac34",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.12"
},
"vscode": {
"interpreter": {
"hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1"
}
}
},
"nbformat": 4,
"nbformat_minor": 5
}
notebooks/dask/05-Go_Big_or_Go_Home_Part_2.ipynb
+727
View File
@@ -0,0 +1,727 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "e374c84f-75e4-4859-bf4f-3a7847aef454",
"metadata": {
"tags": []
},
"source": [
"# Go Big or Go Home Part 2 - Working and Visualising on cluster <img align=\"right\" src=\"../../resources/csiro_easi_logo.png\">\n",
"\n",
"In this notebook we finally do our larger area. We're going to need some better visualisation tools and it would be great not to bring the results back to the Jupyter notebook but to leverage the dask clusters resources during visualisation. We'll be using some dask-aware visualiation libraries (holoviews and datashader) to do the heavy lifting.\n",
"\n",
"Let's begin by starting up our cluster and sizing it appropriately to our computational task."
]
},
{
"cell_type": "markdown",
"id": "4770317f-ec1b-4365-9a23-d4ef684a1a11",
"metadata": {},
"source": [
"## Time to go big!\n",
"\n",
"All the code here is the same as the conclusion from the previous notebook, except we'll make the cluster bigger with 10 workers instead of 4. We'll also make the masking and NDVI calculation into a python function since we won't be making any changes to that now.\n",
"\n",
"We'll use the same ROI and time period for this run and we're using all the techniques so far to reduce the computation time:\n",
"1. Dask chunk size selection\n",
"2. Only loading the measurements we intend on using in this calculation to save on the task graph optimisation time"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1619a66a-c80f-4345-8022-a81018dc294f",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Initialize the Gateway client\n",
"from dask.distributed import Client\n",
"from dask_gateway import Gateway\n",
"\n",
"number_of_workers = 10 \n",
"\n",
"gateway = Gateway()\n",
"\n",
"clusters = gateway.list_clusters()\n",
"if not clusters:\n",
" print('Creating new cluster. Please wait for this to finish.')\n",
" cluster = gateway.new_cluster()\n",
"else:\n",
" print(f'An existing cluster was found. Connecting to: {clusters[0].name}')\n",
" cluster=gateway.connect(clusters[0].name)\n",
"\n",
"cluster.scale(number_of_workers)\n",
"\n",
"client = cluster.get_client()\n",
"client"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "01c6ebaa-8f17-4767-b120-8917bd75a466",
"metadata": {},
"outputs": [],
"source": [
"import pyproj\n",
"pyproj.set_use_global_context(True)\n",
"\n",
"import git\n",
"import sys, os\n",
"from dateutil.parser import parse\n",
"from dateutil.relativedelta import relativedelta\n",
"from dask.distributed import Client, LocalCluster\n",
"import datacube\n",
"from datacube.utils import masking\n",
"from datacube.utils.aws import configure_s3_access\n",
"\n",
"# EASI defaults\n",
"os.environ['USE_PYGEOS'] = '0'\n",
"repo = git.Repo('.', search_parent_directories=True).working_tree_dir\n",
"if repo not in sys.path: sys.path.append(repo)\n",
"from easi_tools import EasiDefaults, notebook_utils\n",
"easi = EasiDefaults()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "73eaf35c-f851-4121-b3dc-fe34ee025e0a",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"dc = datacube.Datacube()\n",
"configure_s3_access(aws_unsigned=False, requester_pays=True, client=client)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "fca9b521-6b25-4738-be07-b1d575a7f8ea",
"metadata": {},
"outputs": [],
"source": [
"# Get the centroid of the coordinates of the default extents\n",
"central_lat = sum(easi.latitude)/2\n",
"central_lon = sum(easi.longitude)/2\n",
"# central_lat = -42.019\n",
"# central_lon = 146.615\n",
"\n",
"# Set the buffer to load around the central coordinates\n",
"# This is a radial distance for the bbox to actual area so bbox 2x buffer in both dimensions\n",
"buffer = 0.8\n",
"\n",
"# Compute the bounding box for the study area\n",
"study_area_lat = (central_lat - buffer, central_lat + buffer)\n",
"study_area_lon = (central_lon - buffer, central_lon + buffer)\n",
"\n",
"# Data product\n",
"products = easi.product('landsat')\n",
"\n",
"# Set the date range to load data over\n",
"set_time = easi.time\n",
"set_time = (set_time[0], parse(set_time[0]) + relativedelta(years=1))\n",
"# set_time = (\"2021-01-01\", \"2021-12-31\")\n",
"\n",
"# Selected measurement names (used in this notebook)\n",
"alias = easi.aliases('landsat')\n",
"measurements = [alias[x] for x in ['qa_band', 'red', 'nir']]\n",
"\n",
"# Set the QA band name and mask values\n",
"qa_band = alias['qa_band']\n",
"qa_mask = easi.qa_mask('landsat')\n",
"\n",
"# Set the resampling method for the bands\n",
"resampling = {qa_band: \"nearest\", \"*\": \"average\"}\n",
"\n",
"# Set the coordinate reference system and output resolution\n",
"set_crs = easi.crs('landsat') # If defined, else None\n",
"set_resolution = easi.resolution('landsat') # If defined, else None\n",
"# set_crs = \"epsg:3577\"\n",
"# set_resolution = (-30, 30)\n",
"\n",
"# Set the scene group_by method\n",
"group_by = \"solar_day\""
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e2febae5-307c-4f16-ada5-1236d4a46ffc",
"metadata": {},
"outputs": [],
"source": [
"def masked_seasonal_ndvi(dataset):\n",
" # Identify pixels that are either \"valid\", \"water\" or \"snow\"\n",
" cloud_free_mask = masking.make_mask(dataset[qa_band], **qa_mask)\n",
" # Apply the mask\n",
" cloud_free = dataset.where(cloud_free_mask)\n",
"\n",
" # Calculate the components that make up the NDVI calculation\n",
" band_diff = cloud_free[alias['nir']] - cloud_free[alias['red']]\n",
" band_sum = cloud_free[alias['nir']] + cloud_free[alias['red']]\n",
" # Calculate NDVI\n",
" ndvi = None\n",
" ndvi = band_diff / band_sum\n",
"\n",
" return ndvi.groupby(\"time.season\").mean(\"time\") # Calculate the seasonal mean\n",
"\n",
"dataset = None # clear results from any previous runs\n",
"dataset = dc.load(\n",
" product=products,\n",
" x=study_area_lon,\n",
" y=study_area_lat,\n",
" time=set_time,\n",
" measurements=measurements,\n",
" resampling=resampling,\n",
" output_crs=set_crs,\n",
" resolution=set_resolution,\n",
" dask_chunks = {\"time\":2, \"x\":3072, \"y\":3072},\n",
" group_by=group_by,\n",
" )\n",
"\n",
"ndvi_unweighted = masked_seasonal_ndvi(dataset)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "80a6df3e-d808-47e9-a9a0-cf7328253c3b",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"print(f\"dataset size (GiB) {dataset.nbytes / 2**30:.2f}\")\n",
"print(f\"ndvi_unweighted size (GiB) {ndvi_unweighted.nbytes / 2**30:.2f}\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e50c099d-de4d-42c8-9eef-f9f01b34bac8",
"metadata": {},
"outputs": [],
"source": [
"# client.wait_for_workers(n_workers=10) # Before release 2023.10.0\n",
"client.sync(client._wait_for_workers,n_workers=10) # Since release 2023.10.0"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "42ee9be6-9c84-4eff-91a0-d876cbfd99e9",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"cluster"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ceea43a7-ec03-4413-89b1-91e5fbeabee9",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"%%time\n",
"actual_result = ndvi_unweighted.compute()"
]
},
{
"cell_type": "markdown",
"id": "6a501db5-d805-4b26-b511-ec2f8d169a52",
"metadata": {
"tags": []
},
"source": [
"You'll notice the computation time is slightly faster with more workers - we're IO bound so more workers means more available IO bandwidth and threads. It's not 2-3 x faster though - we're wasting a lot of resources because we can't actually use all of that extra power.\n",
"\n",
"> __Tip__: More isn't always better. Be mindful of your computational resource usage and cost. This size cluster is a tremendous waste for this size computational job. Size things appropriately."
]
},
{
"cell_type": "markdown",
"id": "eb5ee837-faae-4928-b9a4-dbe6d0d43476",
"metadata": {},
"source": [
"And visualise the result"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "43c304d8-407d-48e3-95cd-41aa51537382",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"actual_result.sel(season='DJF').plot()"
]
},
{
"cell_type": "markdown",
"id": "3495ba8f-301c-4751-af73-a5333348e2bc",
"metadata": {
"tags": []
},
"source": [
"We'll save the coordinates of this section from the array (as slices) so we can use them later for visualising the same ROI from a larger dataset."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "04dc7f0e-b2f0-4093-9d81-7cdc00cd8afa",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"x_slice = slice(ndvi_unweighted.x[0], ndvi_unweighted.x[-1])\n",
"y_slice = slice(ndvi_unweighted.y[0], ndvi_unweighted.y[-1])"
]
},
{
"cell_type": "markdown",
"id": "2dbfb392-9ffd-418e-a610-e29503980458",
"metadata": {},
"source": [
"## Now for a bigger area\n",
"\n",
"Let's change the area extent to about 4 degrees square.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e08de486-3ff3-44c3-8d0e-29dfbd929982",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Compute the bounding box for the study area\n",
"buffer = 2\n",
"# Compute the bounding box for the study area\n",
"study_area_lat = (central_lat - buffer, central_lat + buffer)\n",
"study_area_lon = (central_lon - buffer, central_lon + buffer)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8ddfb47e-53ec-4e77-8f20-bc687c8f35c0",
"metadata": {},
"outputs": [],
"source": [
"# Check the map below to see if you are including the area that you want. For this example, it would be best to not include too much water.\n",
"from dea_tools.plotting import display_map\n",
"display_map(study_area_lon, study_area_lat)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "85a90609-1d18-4ab9-ab8c-ce83659db9e4",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"dataset = None # clear results from any previous runs\n",
"dataset = dc.load(\n",
" product=products,\n",
" x=study_area_lon,\n",
" y=study_area_lat,\n",
" time=set_time,\n",
" measurements=measurements,\n",
" resampling=resampling,\n",
" output_crs=set_crs,\n",
" resolution=set_resolution,\n",
" dask_chunks = {\"time\":2, \"x\":3072, \"y\":3072},\n",
" group_by=group_by,\n",
" )\n",
"\n",
"ndvi_unweighted = masked_seasonal_ndvi(dataset)"
]
},
{
"cell_type": "markdown",
"id": "449e859a-5cbc-4341-ae78-d5735b11ed35",
"metadata": {},
"source": [
"Before we compute anything let's take a look at our result's shape and size"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "5d66426b-ef44-469a-9675-116f5d3fec1c",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"print(f\"dataset size (GiB) {dataset.nbytes / 2**30:.2f}\")\n",
"print(f\"ndvi_unweighted size (GiB) {ndvi_unweighted.nbytes / 2**30:.2f}\")"
]
},
{
"cell_type": "markdown",
"id": "07021f2b-5f82-4a61-9afb-c8e5b304b94e",
"metadata": {},
"source": [
"This is now much bigger!\n",
"\n",
"The result is getting on the large size for the notebook node __so we will need to pay attention to _data locality_ and the size of results being processed__. The cluster has a LOT more memory than the _notebook node_; bring too much back to the notebook and the notebook will crash.\n",
"\n",
"> __Tip__: Be mindful of the size of the results and their _data locality_. \n",
"\n",
"Now let's check the _shape_, _tasks_ and _chunks_"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d751b9c3-75b6-4143-a484-68d9fab6751f",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"dataset"
]
},
{
"cell_type": "markdown",
"id": "cca340a5-1068-4cdc-b4ae-ed7eda75853e",
"metadata": {
"tags": []
},
"source": [
"Looking at the `red` data variable we can see about 50 GiB for the array, 36 MiB per chunk and 5526 tasks. Noting the `nir` and `qa_band` will be similarly shaped and size.\n",
"\n",
"The number of tasks is climbing so we can expect an increase in _task graph optimisation_ time.\n",
"\n",
"Chunk size and tasks seems okay, but we will monitor the _dask dashboard_ in case there are issues with temporaries causing _workers_ to _spill to disk_ if memory is too full.\n",
"\n",
"_The chunking is resulting in some slivers, particularly on the y axis._ Let's modify the y chunk size so these slivers don't exist as its blowing out the tasks and is likely unnecessary. We will calculate the x and y chunk sizes below to get a nice fit. _Make sure to check the chunk size afterwards to make sure it doesn't get too large. If it does we can make the chunks smaller to reduce slivers too._"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b2ff1968-87fb-4b02-8957-b78348a30d52",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"from math import ceil\n",
"\n",
"y_chunks = ceil(dataset.dims['y']/5)\n",
"y_chunks"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a2785d84-c578-40be-898f-84536ccc45a4",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"dataset = None # clear results from any previous runs\n",
"dataset = dc.load(\n",
" product=products,\n",
" x=study_area_lon,\n",
" y=study_area_lat,\n",
" time=set_time,\n",
" measurements=measurements,\n",
" resampling=resampling,\n",
" output_crs=set_crs,\n",
" resolution=set_resolution,\n",
" dask_chunks = {\"time\":2, \"x\":3072, \"y\":y_chunks},\n",
" group_by=group_by,\n",
" )\n",
"\n",
"ndvi_unweighted = masked_seasonal_ndvi(dataset)"
]
},
{
"cell_type": "markdown",
"id": "293f8379-60e4-4fd7-b722-bfeb1b09fc18",
"metadata": {
"tags": []
},
"source": [
"Now recheck our chunk size and tasks for `red`"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "fefd593d-741b-423f-95cd-3c501230a6c0",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"dataset"
]
},
{
"cell_type": "markdown",
"id": "b6ea5908-12e0-4c89-af9e-b09a905e1638",
"metadata": {
"tags": []
},
"source": [
"Very marginal increase in _memory per chunk_ but the _tasks_ have dropped from 5526 to 4661. Note that this occurs for every measurement and operation so the benefit is significant."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "12283353-1f88-4abd-b8de-fc490ea02cb6",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"ndvi_unweighted"
]
},
{
"cell_type": "markdown",
"id": "7a7438cf-1d36-4afc-9230-744765bc11cd",
"metadata": {
"tags": []
},
"source": [
"Total task count is sub 100_000 so should be okay but _task graph optimisation_ will take a while. Resulting array is a bit big for the notebook node as stated previously.\n",
"\n",
"The shape spatially is `y:15686, x:13707`. Standard plots aren't going to work very well for visualising the result in the notebook and the result uses a fair amount of memory so we'll need a different approach.\n",
"\n",
"For now let's visualize the same ROI as the small area before. We stashed that ROI in `x_slice, y_slice`.\n",
"\n",
"__If you haven't already, open the dask dashboard so you can watch the cluster make progress__\n",
"\n",
"The code to do this visualisation is basically the same as before except now we specify a slice"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "21b9258c-010b-4a5c-92a7-1e88c8f09fb6",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"%%time\n",
"ndvi_unweighted.sel(season='DJF', x=x_slice, y=y_slice).compute().plot(robust=True, size=6, aspect='equal')"
]
},
{
"cell_type": "markdown",
"id": "a81e9086-b7b9-490c-88a8-a36e4c272862",
"metadata": {},
"source": [
"The computation time is relatively short since we are only materialising the result for a subset of the overall dataset."
]
},
{
"cell_type": "markdown",
"id": "0ecd4988-8843-4cc2-8d82-7be1b63f6b92",
"metadata": {},
"source": [
"## Visualising all of the data\n",
"\n",
"To visualise all of the data we will make use of the dask cluster and some dask-aware visualisation capabilties from `holoviews` and `datashader` python libraries. These libraries provide an _interactive_ visualisation capability that leaves the large datasets on the cluster and transmits only the final visualisation to the Jupyter notebook. This is done on the fly so the user can zoom and pan about the dataset in all dimensions and the dask cluster will scale data to fit in the viewport automatically. Details of how this is done and advanced features available is beyond the scope of this dask and ODC course but the manuals are extensive and the basic example here both powerful and useful.\n",
"\n",
"> __Tip__: The [datashader pipeline](https://datashader.org/getting_started/Pipeline.html) page provides an excellent summary of what's going on."
]
},
{
"cell_type": "markdown",
"id": "0fd9c328-5e93-4aed-bb4f-0f6aab70c71d",
"metadata": {},
"source": [
"### `compute()` and `persist()`\n",
"\n",
"The first thing we will do is `persist()` the results of our calculation to the cluster. This will materialise the results but will keep the result on the cluster (so all lazy tasks are calculated, just like `compute()` but _data locality_ remains on the cluster). This will ensure the result is readily available for the visualisation. The cluster has plenty of (distributed) memory so there is no reason not to materialise the result on the cluster."
]
},
{
"cell_type": "markdown",
"id": "2ec3ae48-5799-4baf-92b2-e853b8b5bfd8",
"metadata": {},
"source": [
"`persist()` is non-blocking so will return just as soon as _task graph optimisation_ (which is performed in the notebook kernel) is complete. Run the next cell and you will see it takes a few seconds to do _task graph optimisation_, and once that is complete the Jupyter notebook will be available for use again. At the same time the _dask dashboard_ will show tasks running as the result is computed and left on the cluster."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d9f0ed8f-8a01-4329-9998-d817622d6c4f",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"%%time\n",
"on_cluster_result = ndvi_unweighted.persist()\n",
"# wait(on_cluster_result)\n",
"on_cluster_result"
]
},
{
"cell_type": "markdown",
"id": "db6a6d61-4bd8-42b6-8aaa-c9765a8083f6",
"metadata": {},
"source": [
"The `on_cluster_result` will continue to show as a dask array on the cluster - not actual results. Think of it as a handle that links the _Jupyter client_ to the result on the _dask cluster_."
]
},
{
"cell_type": "markdown",
"id": "744fd152-50d4-45f8-ab2e-7311aecb7548",
"metadata": {},
"source": [
"The cluster will start the computation, but we can continue working in the notebook. Let's import a new visualization library: `hvplot`. We'll be using `datashader.rasterize` via `hvplot` to handle the visualisation of the full dataset which has many more pixels than what what is being displayed in the notebook. `hvplot.xarray` makes visualising `xarray` data a very natural experience, so the code is quite simple, a lot is taken care of for you.\n",
"\n",
"Notice also there are no bounds set on the dataset, we are viewing the entire result, including the _season_ dimension. `hvplot` will provide an interface for pan, zoom and season selection and you can use the mouse to move around the data.\n",
"\n",
">__Tip:__ Keep watching your Dask dashboard to see how the calculations are progressing."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1e653ad4-155e-4bbe-86d5-fbc4e90149b7",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# quick calculation so the interactive UI is no more than 700 pixels wide and maintains aspect ratio.\n",
"aspect = on_cluster_result.sizes['y']/on_cluster_result.sizes['x']\n",
"width = 700\n",
"height = int(width*aspect)"
]
},
{
"cell_type": "markdown",
"id": "728a39dd-ea5a-445b-b3c7-33f39d113dc8",
"metadata": {},
"source": [
"The next cell will display the result - when its ready. The `rasterize` function will calculate a representative pixel for display from the full array on the dask cluster. If you monitor the dashboard you will see small bursts of activity across the workers and quite some waiting whilst data transfers occur to bring all the summary information back and transmit it to the Jupyter notebook. It's a large dataset, only the pixels you can see on the screen are sent to your web browser.\n",
"\n",
"You can use the controls on the right to pan and zoom around the full image. If you zoom in, `rasterize` will take a moment to generate a new summary for the current zoom level and show more or less detail. Similarly for panning."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "748c7958-4bce-45dc-895f-255347f91c4f",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"import hvplot.xarray \n",
"import xarray as xr\n",
"on_cluster_result.hvplot.image(groupby='season', rasterize=True).opts(\n",
" title=\"NDVI Seasonal Mean\",\n",
" cmap=\"RdYlGn\", # NDVI more green the larger the value. \n",
" clim=(-0.3, 0.8), # we'll clamp the range for visualisation to enhance the visualisation\n",
" colorbar=True,\n",
" frame_width=width,\n",
" frame_height=height\n",
" )"
]
},
{
"cell_type": "markdown",
"id": "0cfcad62-a6e4-4504-9e41-d33f872ade7a",
"metadata": {
"tags": []
},
"source": [
"# Be a good dask user - Clean up the cluster resources"
]
},
{
"cell_type": "markdown",
"id": "66acca97-686c-4d09-ba5b-a9aeede4bfba",
"metadata": {},
"source": [
"Disconnecting your client is good practice, but the cluster will still be up so we need to shut it down as well"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0578682e-917e-49b7-9f3e-88584a572936",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"client.close()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d144e53a-fb47-493f-9dbd-adef37566e07",
"metadata": {},
"outputs": [],
"source": [
"cluster.shutdown()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "69b02631-c13f-481b-bc1c-f183538e7f63",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.12"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
notebooks/dask/06-On_Chunks-The_Art_of_Dask_Tuning_Part_1.ipynb
+519
View File
@@ -0,0 +1,519 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "0e4faa01-790c-4c1f-ac68-d46730fb570a",
"metadata": {},
"source": [
"# On Chunks - The Art of Dask Part 1 <img align=\"right\" src=\"../../resources/csiro_easi_logo.png\">\n",
"\n",
"In this notebook we'll be exploring the impact of chunking choices for dask arrays. We'll use an ODC example but this isn't specific to ODC, it applies to all usage of dask `Array`s. Chunking choices have a _significant_ impact on performance for three reasons:\n",
"1. Chunks are the unit of work during processing\n",
"2. Chunks are the unit of transport in communicating information between workers\n",
"3. Chunks are directly related to the number of _tasks_ being executed\n",
"\n",
"Performance is thus impacted in multiple ways - this is all about tradeoffs:\n",
"* if chunks are too small, there will be too many _tasks_ and processing may be inefficient _BUT_\n",
"* if chunks are too big, communication may be too long and the combined total of all chunks required for a calculation may exceed worker memory causing spilling to disk or worse, workers are killed\n",
"\n",
"It's not just size that matters either, the relative contiguity of dimensions matters:\n",
"* Temporal processing is enhanced by larger chunks along the time dimension\n",
"* Spatial processing is enhanced by larger chunks along the spatial dimensions, _BUT_\n",
"* Earth Observation data can be sparse spatially, if chunks are too large spatially there will be a lot of empty chunks\n",
"\n",
"Thankfully it is possible to _re-chunk_ data for different stages of computation. Whilst _re-chunking is an *expensive* operation_ the efficiency gains for downstream computation can be very significant and sometimes are simply essential to support the numerical processing required. For example, it is often necessary to have a single chunk on the time dimension for temporal calculations.\n",
"\n",
"To understand the impact of chunking choices on _your code_ (it is very algorithm dependent) it is essential to understand both the:\n",
"* _Static_ impact of chunking (e.g. task count, chunk size in memory), and;\n",
"* _Dynamic_ impact of chunking (e.g. CPU load, thread utilisation, network communication, task count and scheduler load).\n",
"\n",
"`Dask` provides tools for viewing all of these when you print out arrays in the notebook (static) and when viewing the various graphs in the dask dashboard (dynamic)."
]
},
{
"cell_type": "markdown",
"id": "205583d7-7b6f-413a-819c-f3e2b6a1b0c9",
"metadata": {},
"source": [
"## Our example\n",
"\n",
"The code below will be familiar, it's the same example from previous notebooks (seasonal mean NDVI over a large area). A normalised burn ratio (NBR2) calculation has been added as well to provide some additional load to assist in making the performance differences more noticeable in various graphs. The NBR2 uses two additional bands but is effectively the same type of calculation as the NDVI (a normalised difference ratio).\n",
"\n",
"The primary difference for this example is the calculation (both NDVI and NBR) is performed 4 times, each with a different chunking regime. See the `chunk_settings` list.\n",
"\n",
"__When running this notebook, be sure to have the dask dashboard open and preferably visible as calculations proceed.__\n",
"\n",
"There are several sections to pay attention too:\n",
"* Status\n",
" * Short term snapshot of the Memory use (total and per worker - also shows Managed, Unmanaged and Spilled to Disk splits), Processing and CPU usage (change the Tab to switch between them)\n",
" * Progress of the optimized Task graph\n",
" * Near term Task Stream (Red is comms, White space is \"doing nothing\", other colours mostly match the tasks and you can hover over them with the mouse to get more information)\n",
"* Tasks\n",
" * Longer term Task Stream. This is a more comprehensive and accurate view of the Execution over time\n",
"* System\n",
" * Scheduler CPU, Memory and Communications load. You can zoom the graphs out using the control to get a longer term view.\n",
"* Groups\n",
" * High level view of the Task Graph Groups and their execution. The actual task graph is too detailed to display so this provides some insight into how high level aspects of your algorithm are executing.\n",
"\n",
"_All_ of these graphs are dynamic and should be interpreted over time.\n",
"\n",
"The dask _scheduler_ itself is also dynamic and as your code executes it stores information about how the tasks are executing and the communication occuring and adjusts scheduling accordingly. It can take a few minutes for the scheduler to settle into a true pattern. That pattern may also change, particularly in latter parts of a computation when work is completing and there are fewer tasks to execute.\n",
"\n",
"Yes, that is a LOT of information. Thankfully you don't necessarily need to learn it all at once. In time, reading the information available will become easier as will knowing what to do about it.\n",
"\n",
"Now let's run this notebook, remember to watch the execution in the Dask Dashboard.\n",
"\n",
"> __Tip__: It's likely you will want to repeat the calculation in this notebook several times. Because the results are `persisted` to the cluster simply calling it again will result in no execution (none is required because it was `persisted`). Rather than doing `cluster.shutdown()` and creating a new cluster each time you can clear the `persisted` result by performing a `client.restart()`. This will clear out all previous calculations so you can `persist` again. You can do this either by creating a new cell or using a Python Console for this Notebook (right click on the notebook and select _New Console for Notebook_)."
]
},
{
"cell_type": "markdown",
"id": "fc395c36-548d-42bc-bd59-d4f390a6fc0a",
"metadata": {},
"source": [
"### Create a cluster\n",
"A modest cluster will do... _and Open the dashboard_"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1619a66a-c80f-4345-8022-a81018dc294f",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Initialize the Gateway client\n",
"from dask.distributed import Client\n",
"from dask_gateway import Gateway\n",
"\n",
"number_of_workers = 5 \n",
"\n",
"gateway = Gateway()\n",
"\n",
"clusters = gateway.list_clusters()\n",
"if not clusters:\n",
" print('Creating new cluster. Please wait for this to finish.')\n",
" cluster = gateway.new_cluster()\n",
"else:\n",
" print(f'An existing cluster was found. Connecting to: {clusters[0].name}')\n",
" cluster=gateway.connect(clusters[0].name)\n",
"\n",
"cluster.scale(number_of_workers)\n",
"\n",
"client = cluster.get_client()\n",
"client"
]
},
{
"cell_type": "markdown",
"id": "ca446bdc-03b8-49bb-8489-13ad313a9a8f",
"metadata": {},
"source": [
"### Setup all our functions and query parameters\n",
"\n",
"Nothing special here"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "008bc505-e1e7-44bf-9c95-d5bb7d96bda4",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"import pyproj\n",
"pyproj.set_use_global_context(True)\n",
"\n",
"import git\n",
"import sys, os\n",
"from dateutil.parser import parse\n",
"from dateutil.relativedelta import relativedelta\n",
"from dask.distributed import Client, LocalCluster, wait\n",
"import datacube\n",
"from datacube.utils import masking\n",
"from datacube.utils.aws import configure_s3_access\n",
"\n",
"# EASI defaults\n",
"os.environ['USE_PYGEOS'] = '0'\n",
"repo = git.Repo('.', search_parent_directories=True).working_tree_dir\n",
"if repo not in sys.path: sys.path.append(repo)\n",
"from easi_tools import EasiDefaults, notebook_utils\n",
"easi = EasiDefaults()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1d8da7e0-5e0b-42a2-833b-ea60962581f4",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"dc = datacube.Datacube()\n",
"configure_s3_access(aws_unsigned=False, requester_pays=True, client=client)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a518ead6-750d-48ef-88f6-4fd4861eb939",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Get the centroid of the coordinates of the default extents\n",
"central_lat = sum(easi.latitude)/2\n",
"central_lon = sum(easi.longitude)/2\n",
"# central_lat = -42.019\n",
"# central_lon = 146.615\n",
"\n",
"# Set the buffer to load around the central coordinates\n",
"# This is a radial distance for the bbox to actual area so bbox 2x buffer in both dimensions\n",
"buffer = 1\n",
"\n",
"# Compute the bounding box for the study area\n",
"study_area_lat = (central_lat - buffer, central_lat + buffer)\n",
"study_area_lon = (central_lon - buffer, central_lon + buffer)\n",
"\n",
"# Data product\n",
"product = easi.product('landsat')\n",
"\n",
"# Set the date range to load data over\n",
"set_time = easi.time\n",
"set_time = (set_time[0], parse(set_time[0]) + relativedelta(months=6))\n",
"#set_time = (\"2021-07-01\", \"2021-12-31\")\n",
"\n",
"# Selected measurement names (used in this notebook)\n",
"alias = easi.aliases('landsat')\n",
"measurements = [alias[x] for x in ['qa_band', 'red', 'nir', 'swir1', 'swir2']]\n",
"\n",
"# Set the QA band name and mask values\n",
"qa_band = alias['qa_band']\n",
"qa_mask = easi.qa_mask('landsat')\n",
"\n",
"# Set the resampling method for the bands\n",
"resampling = {qa_band: \"nearest\", \"*\": \"average\"}\n",
"\n",
"# Set the coordinate reference system and output resolution\n",
"set_crs = easi.crs('landsat') # If defined, else None\n",
"set_resolution = easi.resolution('landsat') # If defined, else None\n",
"# set_crs = \"epsg:3577\"\n",
"# set_resolution = (-30, 30)\n",
"\n",
"# Set the scene group_by method\n",
"group_by = \"solar_day\""
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "71e6ed74-06b4-4450-a7cd-25e487909878",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"def calc_ndvi(dataset):\n",
" # Calculate the components that make up the NDVI calculation\n",
" band_diff = dataset[alias['nir']] - dataset[alias['red']]\n",
" band_sum = dataset[alias['nir']] + dataset[alias['red']]\n",
" # Calculate NDVI\n",
" ndvi = band_diff / band_sum\n",
" return ndvi\n",
"\n",
"def calc_nbr2(dataset):\n",
" # Calculate the components that make up the NDVI calculation\n",
" band_diff = dataset[alias['swir1']] - dataset[alias['swir2']]\n",
" band_sum = dataset[alias['swir1']] + dataset[alias['swir2']]\n",
" # Calculate NBR2\n",
" nbr2 = band_diff / band_sum\n",
" return nbr2\n",
"\n",
"def mask(dataset, bands):\n",
" # Identify pixels that are either \"valid\", \"water\" or \"snow\"\n",
" cloud_free_mask = masking.make_mask(dataset[qa_band], **qa_mask)\n",
" # Apply the mask\n",
" cloud_free = dataset[bands].astype('float32').where(cloud_free_mask)\n",
" return cloud_free\n",
"\n",
"def seasonal_mean(dataset):\n",
" return dataset.resample(time=\"QS-DEC\").mean('time') # perform the seasonal mean for each quarter"
]
},
{
"cell_type": "markdown",
"id": "e6d30531-5747-4b70-9e47-aabd41f60b2e",
"metadata": {},
"source": [
"We have an array of chunk settings to trial.\n",
"\n",
"* Notice the `chunk_settings` are nominally the same size\n",
"* Notice we're varying the temporal chunking from large to small and adjusting the spatial chunking to keep the overall volume similar (nominally this will be 100 Megs per chunk for the original dataset)\n",
"\n",
"There are two `time:1` chunks because 50 doesn't have a clean sqrt. The first is the nearest square, the second simply changes the chunks to be rectangles (no one said the spatial dimensions needed to be the same).\n",
"\n",
"Given the chunk size in memory is roughly the same, the cluster the same, the calculation the same - any differences in execution are a result of the different chunking shape."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1ade0fc8-49a8-43e6-997a-60bae9d5c9c5",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"chunk_settings = [\n",
" {\"chunks\": {\"time\":100, \"x\":300, \"y\":300}, \"comment\": \"This run has small spatial chunks but each chunk has a lot of time steps. This results in many small file reads, but there are more total tasks for the scheduler to handle.\"},\n",
" {\"chunks\": {\"time\":50, \"x\":1*300, \"y\":2*300}, \"comment\": \"This second run has slightly larger spatial chunks but smaller temporal extents in each chunk. This results in fewer total tasks, but each one takes longer to load.\"},\n",
" {\"chunks\": {\"time\":1, \"x\":10*300, \"y\":10*300}, \"comment\": \"This run has only a single time step in each chunk, but large, square spatial extents. As a result, workers need to store much more data in memory and some data is spilled to disk.\"},\n",
" {\"chunks\": {\"time\":1, \"x\":21*300, \"y\":5*300}, \"comment\": \"Again this run has a single time step per chunk, but the spatial extents are rectangles.\"},\n",
"]"
]
},
{
"cell_type": "markdown",
"id": "65b4ff1b-d512-4b6d-9f32-8fb0eeea9398",
"metadata": {},
"source": [
"Now we can loop over all our `chunk_settings` and create all the required `delayed task graphs`. This will take a moment as the ODC database will be interogated for all the necessary dataset information.\n",
"\n",
"_You will notice the calculation is split up so we can see the interim results_ - well the last one at least given its a loop and we're overwriting them.\n",
"\n",
"Different stages of computation will produce different data types and calculations and thus _chunk_ and _task_ counts. We may find that an interim result has a terrible chunk size (e.g. `int16` data variables become `float64` and thus your chunks are now 4x the size, or a dimension is reduced and chunks are too small). It is thus advisable when tuning to make it possible to view these interim stages to see the _static_ impact.\n",
"\n",
"__Remember__: there is a single task graph executing to provide the final result. There is no need to `persist()` or `compute()` the interim results to see their _static_ attributes. In fact, it may be unwise to `persist()` as this will chew up resources on the cluster if you don't intend on using the results."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "89eb9a27-c0d7-44d2-9823-e7f994ddf135",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"for chunkset in chunk_settings:\n",
" chunks = chunkset[\"chunks\"]\n",
" print(chunks)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e4c2276f-71be-4ac6-b29b-5e9ef96ab69c",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"import numpy as np\n",
"results = []\n",
"for chunkset in chunk_settings:\n",
" chunks = chunkset[\"chunks\"]\n",
" dataset = dc.load(\n",
" product=product,\n",
" x=study_area_lon,\n",
" y=study_area_lat,\n",
" time=set_time,\n",
" measurements=measurements,\n",
" resampling=resampling,\n",
" output_crs=set_crs,\n",
" resolution=set_resolution,\n",
" dask_chunks = chunks,\n",
" group_by=group_by,\n",
" )\n",
" \n",
" num_time = dataset.sizes['time']\n",
" time_ind = np.linspace(1, num_time, 100, dtype='int') - 1\n",
" dataset = dataset.isel(time=time_ind) # load exactly 100 evenly spaced timesteps so that we can work more easily with different chunks\n",
"\n",
" masked_dataset = mask(dataset, [alias[x] for x in ['red', 'nir', 'swir1', 'swir2']])\n",
" ndvi = calc_ndvi(masked_dataset)\n",
" nbr2 = calc_nbr2(masked_dataset)\n",
" seasonal_mean_ndvi = seasonal_mean(ndvi)\n",
" seasonal_mean_nbr2 = seasonal_mean(nbr2)\n",
" seasonal_mean_ndvi.name = 'ndvi'\n",
" seasonal_mean_nbr2.name = 'nbr2'\n",
" results.append([seasonal_mean_ndvi, seasonal_mean_nbr2])"
]
},
{
"cell_type": "markdown",
"id": "9ac8a059-82d9-4188-8878-e7cc1dec7efe",
"metadata": {},
"source": [
"### Inspecting _static_ information\n",
"\n",
"Lets take a look at the vital statistics for the final iteration of the loop. All the calculations are the same, just the `chunk` parameters vary so we can infer easily from these what else is happening for the _static_ parameters.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ffb6bc3b-9872-4b11-8599-ba31f1ade74b",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"print(f\"dataset size (GiB) {dataset.nbytes / 2**30:.2f}\")\n",
"print(f\"seasonal_mean_ndvi size (GiB) {seasonal_mean_ndvi.nbytes / 2**30:.2f}\")\n",
"display(dataset)"
]
},
{
"cell_type": "markdown",
"id": "31183e67-0f5c-40d8-aee1-13a8f73da9a8",
"metadata": {},
"source": [
"So the source `dataset` is 150 GB in size - mostly `int16` data type. _We need to be mindful that our calculation will convert these to `floats`._ The code above does an explicit type conversion to `float32` which can fully represent an `int16`. Without the explicit type conversion, Python would use `float64` resulting in double the memory usage for no good reason (for this algorithm).\n",
"\n",
"Open the _cylinder_ to show the `red` dask array details. The chunk is about 100 MiB in size. Generally this is a healthy size though it can be larger and may need to be smaller depending on the calculation involved and communication between workers.\n",
"\n",
"Now let's look at the results for the NDVI and NBR:"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "10ab0c2f-05f8-4ca7-8dd8-789c63d143df",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"display(results[0][0])\n",
"display(results[0][1])"
]
},
{
"cell_type": "markdown",
"id": "cde0a747-5629-48d3-b0c7-de9035c96f01",
"metadata": {},
"source": [
"Notice the result is _much smaller_ in chunk size - 4 MiB. This is due to the seasonal mean. This may have an impact on downstream usage of the result as the _chunks_ may be too small and result in too many tasks reducing later processing performance.\n",
"\n",
"Notice also the Task count. With both results we're pushing towards 100_000 tasks in the scheduler depending on task graph optimisation. The Scheduler has its own overheads (about 1ms per active task, and memory usage for tracking all tasks, including executed ones as it keeps the history in case it needs to reproduce the results e.g. if a worker is lost). Again, it is possible to have more than 100_000 tasks and be efficient depending on your algorithm but its something to keep an eye on. We will be below it in this case (especially after optimisation)."
]
},
{
"cell_type": "markdown",
"id": "ee97957f-7959-422d-b9ae-a47750aa4b50",
"metadata": {},
"source": [
"### Persist the results\n",
"\n",
"Theoretically we could `persist` all of the `results` at once - though we would be well above the 100_000 task limit if we did.\n",
"More importantly we actually want to see the difference in the _dynamics_ of the execution.\n",
"The loop below will persist each result one at a time and _wait()_ for it to be complete.\n",
"\n",
"__You should monitor execution in the Dask Dashboard__\n",
"\n",
"Look at the various tabs as execution proceeds. you will notice differences in memory per worker, Communication between workers (red bars in the Task Stream), white space (idle time), and CPU utilisation (remember to click on the CPU tab to get to this detail).\n",
"The `Tasks` section of the dashboard is particularly useful at looking at a comparison of all four runs' dynamics as the length of all calculations means this snapshot still show all four blocks of computation at once.\n",
"\n",
"Don't forget, if you want to run the code again use `client.restart()` to clear out the previous results from the cluster.\n",
"\n",
">__Tip:__ If you leave your computer while this step is running, make sure that it doesn't go to sleep by adjusting your power settings."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "43a306aa-86e7-432a-bdfc-b4ecf20e8aa0",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# client.wait_for_workers(n_workers=number_of_workers) # Before release 2023.10.0\n",
"client.sync(client._wait_for_workers,n_workers=number_of_workers) # Since release 2023.10.0\n",
"\n",
"for i, result in enumerate(results):\n",
" print(f'Run number {i+1}:')\n",
" print(f'Chunks: {chunk_settings[i][\"chunks\"]}')\n",
" print(chunk_settings[i][\"comment\"])\n",
" client.restart()\n",
" f = client.persist(result)\n",
" %time wait(f)\n",
" client.restart() # clearing the cluster out so each run it cleanly separated\n",
" print()"
]
},
{
"cell_type": "markdown",
"id": "322e2de7-2e11-4edf-bf14-6ca1d0b81a6a",
"metadata": {},
"source": [
"## Understanding the dynamics\n",
"\n"
]
},
{
"cell_type": "markdown",
"id": "0cfcad62-a6e4-4504-9e41-d33f872ade7a",
"metadata": {},
"source": [
"# Be a good dask user - Clean up the cluster resources"
]
},
{
"cell_type": "markdown",
"id": "66acca97-686c-4d09-ba5b-a9aeede4bfba",
"metadata": {},
"source": [
"Disconnecting your client is good practice, but the cluster will still be up so we need to shut it down as well"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0578682e-917e-49b7-9f3e-88584a572936",
"metadata": {},
"outputs": [],
"source": [
"client.close()\n",
"\n",
"cluster.shutdown()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "3de2e63e-138d-401e-944d-27520d3ad8b9",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.12"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
notebooks/data_products/novasar-1-gamma0.ipynb
+952
View File
@@ -0,0 +1,952 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "dae926b2-fa06-4f00-b18d-7f74e92ce676",
"metadata": {
"jp-MarkdownHeadingCollapsed": true
},
"source": [
"## NovaSAR-1 data products <img align=\"right\" src=\"../../resources/csiro_easi_logo.png\">\n",
"\n",
"#### Index\n",
"- [Overview](#Overview)\n",
" - [Tips and tricks](#Tips-and-tricks)\n",
" - [Background on SAR data](#Background-on-SAR-data)\n",
"- [Setup (imports, defaults, dask, odc)](#Setup)\n",
"- [Example query](#Example-query)\n",
" - [Review scene metadata for the query](#Review-scene-metadata-for-the-query)\n",
" - [Filter scenes based on metadata only](#Filter-scenes-based-on-metadata-only)\n",
"- [Load data into a virtual dask array](#Load-data-into-a-virtual-dask-array)\n",
"- [Conversion and helper functions](#Conversion-and-helper-functions)\n",
" - [Filter scenes based on valid data](#Filter-scenes-based-on-valid-data)\n",
" - [Add dB and amplitude values to the dataset](#Add-dB-and-amplitude-values-to-the-dataset)\n",
"- [Persist the data into the dask workers](#Persist-the-data-into-the-dask-workers)\n",
" - [Plot the data](#Plot-the-data)\n",
" - [Plot histograms of the dB data](#Plot-histograms-of-the-dB-data)\n",
" - [Make an RGB image](#Make-an-RGB-image)\n",
"- [Export to Geotiffs](#Export-to-Geotiffs)"
]
},
{
"cell_type": "markdown",
"id": "23474415-05a4-4065-b28a-3acbe7129f7f",
"metadata": {},
"source": [
"## Overview\n",
"\n",
"This notebook demonstrates how to load and use NovaSAR-1 ARD _gamma-0 backscatter_ data products from the [CSIRO NovaSAR Facility](https://research.csiro.au/cceo/novasar/).\n",
"\n",
"The [NovaSAR-1 data products](https://research.csiro.au/cceo/novasar/about/novasar-1-user-guide/#products) include:\n",
"- Level-1 product types Multi-Look Detected (GRD, SCD, SRD) and Single Look Complex (SLC)\n",
"- Level-2 analysis ready data (ARD) gamma-0 radiometric terrain corrected backscatter\n",
"\n",
"The **Level-1** data products are stored in swath (line, row) raster grids, with a \"fake\" WGS-84 bounding box grid for ODC indexing. This means they are not suited to `datacube.load()` directly. Instructions TBC.\n",
"\n",
"The **Level-2 ARD** data products are remapped to WGS-84 grids (various resolutions) as part of the gamma-0 backscatter processing. These data can be opened directly with `datacube.load()`."
]
},
{
"attachments": {},
"cell_type": "markdown",
"id": "35b91cdb-2c5a-48c0-a0bf-7de2c2637bc5",
"metadata": {},
"source": [
"## Tips and tricks\n",
"\n",
"#### 1. NovaSAR-1 product names\n",
"\n",
"The ODC currently requires a “dataset” to contain all bands that are defined for its parent product. In practice, each of the NovaSAR-1 product types (GRD, SCD, SRD, SLC, ARD) can have different combinations of available polarization bands across all scene acquisitions. This means we need multiple ODC products; one for each combination of polarizations.\n",
"\n",
"> WIP: relax this constraint in ODC so that we can have \"optional\" bands defined, which will allow us to aggregate polarization combinations for a single product type.\n",
"\n",
"View the coverage of indexed NovaSAR-1 products and datasets in the [CSIRO EASI Explorer \"SAR products group\"](https://explorer.csiro.easi-eo.solutions/products#synthetic-aperture-radar-sar-group).\n",
"\n",
"#### 2. Load to target grid or crs/resolution\n",
"\n",
"Different scenes (ODC datasets) in a product may have different spatial resolutions depending on the acquisition mode, so its usually appropriate to provide a target grid. For example, use either\n",
"\n",
"- `datacube.load(..., like=target_geobox)`. See [odc-geo](https://github.com/opendatacube/odc-geo) to help create a target \"geobox\", or use an existing (ODC-compatible) xarray object\n",
"- `datacube.load(..., output_crs=target_crs, resolution=target_res, align=target_res/2)`\n",
"\n",
"For convenience and general applicability, default load parameters are defined for all NovaSAR-1 L2 ARD products as:\n",
"\n",
"- `CRS = \"epsg:4326\" (WGS-84)`, `Pixel_size = 0.0002 deg` (20 m) and `Align = 0.0001` (10 m). \n",
"\n",
"This means a `datacube.load()` call without specific output grid parameters (no `like`, `output_crs`, `resolution`) will load data into a _WGS-84, 20 m, aligned to pixel centres_ grid with extents given by the `latitude/longitude/x/y/crs` parameters.\n",
"\n",
"> Native NovaSAR-1 L2 ARD products are aligned to pixel centres (AREA_OR_POINT=Point). You should ensure this is taken into account if mapping pixels to a target grid that is approximately the same resolution as the native scene resolution (otherwise you may be shifting by half-pixel). If mapping to a coarser resolution if may not matter as much.\n",
"\n",
"#### 3. Select or filter scenes before loading\n",
"\n",
"Scenes can be pre-filtered using the scene metadata and the `datacube.find_datasets()` function before datasets are loaded into an xarray object. There are two easy (and similar) ways to achieve this:\n",
"\n",
"1. Use `datacube.find_datasets()` with _product, time, space_ search parameters. This returns a list of dataset items that can be filtered manually, e.g. by scene name or scene metadata. Pass your filtered list to `datacube.load(datasets=my_datasets_list, like=target_geobox)`.\n",
" - This is demonstrated below by parsing likely useful metadata for a list of datasets items into a pandas table, for viewing.\n",
"1. Create a metadata predicate function that returns `True` if a dataset's metadata satisifies the conditions specified in your function. This predicate function can be passed to `datacube.load(..., dataset_predicate=my_filter_function`), which is then applied internally in a call to `find_datasets`. See [datacube.load() help](https://opendatacube.readthedocs.io/en/latest/api/indexed-data/generate/datacube.Datacube.load.html) for an example.\n",
"\n",
"#### 4. Units and conversions\n",
"The `novasar_l2ard_*` data are given in _Digital numbers_ (DN). DNs can be converted to _decibel (dB)_ or _linear amplitude_, and vice-versa, with the following equations. Practical _Xarray_ examples are given below.\n",
"\n",
"Amplitude to/from dB:\n",
"```\n",
"dB = 20 * log10(DN) + K\n",
"DN = 10^((dB-K)/20)\n",
"\n",
"where K is a calibration factor, which for NovaSAR-1 is -83 dB.\n",
"```\n",
"\n",
"Digital numbers to/from Amplitude:\n",
"```\n",
"amplitude = DN / 14125.3754\n",
"DN = amplitude * 14125.3754\n",
"```"
]
},
{
"cell_type": "markdown",
"id": "5e5120c6-e943-46a9-a4d8-c5b759586e62",
"metadata": {},
"source": [
"## Background on SAR data\n",
"\n",
"An excellent introduction and overview to using SAR data is provided in the [CEOS Laymans SAR Interpretation Guide](https://ceos.org/ard/files/Laymans_SAR_Interpretation_Guide_3.0.pdf). This guide has also been converted to a set of set of Jupyter notebooks that you can download from https://github.com/AMA-Labs/cal-notebooks/tree/main/examples/SAR.\n",
"\n",
"The SAR instrument on the NovaSAR-1 satellite operates in the ***S-band at approximately 9.4 cm wavelength***. This means that it can \"see\" objects of about this size and larger, and smaller objects are relatively transparent. Compared to *Sentinel-1 (C-band, 5.6 cm)*, NovaSAR-1 S-band is better able to penetrate through vegetation.\n",
"\n",
"> The SAR signal responds to the orientation and scattering from surface features of comparable size or larger than the wavelength.\n",
"> - A bright backscatter value typically means the surface was orientated perpendicular to the signal incidence angle and most of the signal was reflected back to the satellite (direct backscatter)\n",
"> - A dark backscatter value means most of the signal was reflected away from the satellite (forward scattering) and typically responds to a smooth surface (relative to the wavelength) such as calm water or bare soil\n",
"> - Rough surfaces (relative to the wavelength) result in diffuse scattering where some of the signal is returned to the satellite.\n",
"> - Complex surfaces may result in volume scattering (scattering within a tree canopy) or double-bounce scattering (perpendicular objects such as buildings and structures)\n",
"> - The relative backscatter values of co-polarisation (HH, VV) and cross-polarisation (HV) measurements can provide information on the scattering characteristics of the surface features.\n",
"\n",
"Using NovaSAR-1 backscatter data requires interpretation of the data for different surface features, including as these features change spatially or in time. It may also be necessary to carefully consider the incidence angle of the SAR signal relative to the surface features using the *incidence_angle* band or the satellite direction metadata (descending = north to south; ascending = south to north)."
]
},
{
"cell_type": "markdown",
"id": "dd5c0ff4-e1a2-4adf-98d1-1ca5b9f3126d",
"metadata": {},
"source": [
"## Set up"
]
},
{
"cell_type": "markdown",
"id": "4cb7c012-719e-4f13-a206-36f271c6a91c",
"metadata": {},
"source": [
"#### Imports"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1c848c17-2e7c-4b68-9682-b8994644f521",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Common imports and settings\n",
"import os, sys, re\n",
"from pathlib import Path\n",
"from IPython.display import Markdown\n",
"import pandas as pd\n",
"pd.set_option(\"display.max_rows\", None)\n",
"import xarray as xr\n",
"import numpy as np\n",
"\n",
"# Datacube\n",
"import datacube\n",
"from datacube.utils.aws import configure_s3_access\n",
"import odc.geo.xr # https://github.com/opendatacube/odc-geo\n",
"from datacube.utils import masking # https://github.com/opendatacube/datacube-core/blob/develop/datacube/utils/masking.py\n",
"from dea_tools.plotting import display_map # https://github.com/GeoscienceAustralia/dea-notebooks/tree/develop/Tools\n",
"\n",
"# Dask\n",
"import dask\n",
"from dask.distributed import Client, LocalCluster\n",
"\n",
"# Basic plots\n",
"%matplotlib inline\n",
"# import matplotlib.pyplot as plt\n",
"# plt.rcParams['figure.figsize'] = [12, 8]\n",
"\n",
"# Holoviews\n",
"# https://holoviz.org/tutorial/Composing_Plots.html\n",
"# https://holoviews.org/user_guide/Composing_Elements.html\n",
"import hvplot.xarray\n",
"import holoviews as hv\n",
"import panel as pn\n",
"from datashader import reductions"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "bde8313b-887d-4529-bd57-89f740ff6380",
"metadata": {},
"outputs": [],
"source": [
"# EASI defaults and convenience functions\n",
"# easi-tools is in the repo https://github.com/csiro-easi/easi-notebooks\n",
"# These are convenience functions for using these notebooks in EASI; comment-out if not required\n",
"\n",
"repo = Path.home() / 'easi-notebooks' # Change path as necessary\n",
"if str(repo) not in sys.path:\n",
" sys.path.append(str(repo))\n",
"\n",
"from easi_tools import initialize_dask, xarray_object_size # Useful"
]
},
{
"cell_type": "markdown",
"id": "bb66366a-f2d5-461a-a34f-7eed5a5296fb",
"metadata": {
"tags": []
},
"source": [
"#### Dask cluster\n",
"\n",
"Using a local _Dask_ cluster is a good habit to get into. It can simplify loading and processing of data in many cases, and it provides a dashboard that shows the loading/processing progress.\n",
"\n",
"To learn more about _Dask_ see the set of [dask notebooks](https://github.com/csiro-easi/easi-notebooks/tree/main/html#dask-tutorials)."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "02f9e778-755c-4ff2-a99f-ad5c48438665",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Local cluster - good for small exploratory and testing work\n",
"cluster, client = initialize_dask(workers=8)\n",
"display(client)\n",
"\n",
"# Or use Dask Gateway - this may take a few minutes\n",
"# cluster, client = initialize_dask(use_gateway=True, workers=4)\n",
"# display(client)"
]
},
{
"cell_type": "markdown",
"id": "a746582e-be79-448c-a9c4-7a2ae2d703ce",
"metadata": {},
"source": [
"#### ODC database\n",
"\n",
"Connect to the ODC database. Configure the environment and low-level tools to read from AWS buckets."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0e2232ee-f0a5-43fc-a6c6-d422b437cd69",
"metadata": {},
"outputs": [],
"source": [
"dc = datacube.Datacube()\n",
"\n",
"# Access AWS \"requester-pays\" buckets\n",
"# This is necessary for reading data from most third-party AWS S3 buckets such as for Landsat and Sentinel-2\n",
"configure_s3_access(aws_unsigned=False, requester_pays=True, client=client);"
]
},
{
"cell_type": "markdown",
"id": "1801db2b-e24e-45fa-85d4-e153ebcc8f6f",
"metadata": {},
"source": [
"## Example query\n",
"\n",
"Change any of the parameters in the `query` object below to adjust the location, time, projection, or spatial resolution of the returned datasets.\n",
"\n",
"Use the Explorer interface to check the temporal and spatial coverage for each product."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2a04d875-20d4-4d72-a16d-e76b7f48161c",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Set your own latitude / longitude\n",
"\n",
"# Menindee Lakes, Australia\n",
"latitude_range = (-32.1, -33.6)\n",
"longitude_range = (141.5, 143)\n",
"\n",
"# Great Western Woodlands, Australia\n",
"# latitude_range = (-33, -32.6)\n",
"# longitude_range = (120.5, 121)\n",
"\n",
"time_range = None # All available times\n",
"\n",
"available_ard_products = [\n",
" 'novasar_l2ard_hh', # Moderate coverage, https://explorer.csiro.easi-eo.solutions/products/novasar_l2ard_hh\n",
" 'novasar_l2ard_hh_hv', # High coverage, https://explorer.csiro.easi-eo.solutions/products/novasar_l2ard_hh_hv\n",
" 'novasar_l2ard_hv', # Low coverage, https://explorer.csiro.easi-eo.solutions/products/novasar_l2ard_hv\n",
" 'novasar_l2ard_vv', # Moderate coverage, https://explorer.csiro.easi-eo.solutions/products/novasar_l2ard_vv\n",
" 'novasar_l2ard_vv_hh', # Moderate coverage, https://explorer.csiro.easi-eo.solutions/products/novasar_l2ard_vv_hh\n",
" 'novasar_l2ard_vv_hh_hv', # High coverage, https://explorer.csiro.easi-eo.solutions/products/novasar_l2ard_vv_hh_hv\n",
"]\n",
"\n",
"# Select a product\n",
"product_name = 'novasar_l2ard_hh_hv'\n",
"polarizations = re.findall('_([vh]+)', product_name) # The set of polarization bands for the selected product\n",
"\n",
"query = {\n",
" 'product': product_name, # Product name\n",
" 'measurements': polarizations + ['angle', 'mask', 'scatteringarea', 'gammatosigmaratio'], # All bands for testing\n",
" 'x': longitude_range, # \"x\" axis bounds\n",
" 'y': latitude_range, # \"y\" axis bounds\n",
" 'time': time_range, # Any parsable date strings\n",
"}\n",
"\n",
"# Convenience function to display the selected area of interest\n",
"display_map(longitude_range, latitude_range)"
]
},
{
"cell_type": "markdown",
"id": "b399fe85-ba78-4a8e-8a33-3dcdfa95b7f2",
"metadata": {},
"source": [
"## Review scene metadata for the query\n",
"\n",
"Run `datacube.find_datasets()` for the query and summarise resulting scene information in a pandas table"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "4d84334a-1749-412c-b10c-e7bbc53c43d8",
"metadata": {},
"outputs": [],
"source": [
"datasets = dc.find_datasets(**query)\n",
"\n",
"# Examples\n",
"# datasets[0].metadata_doc --> dict of all metadata\n",
"# datasets[0].measurements --> dict of measurements\n",
"# datasets[0].grids --> dict of grids (shape and transform) defined for measurements\n",
"# datasets[0].crs --> CRS object\n",
"# datasets[0].extent --> geometry object\n",
"# datasets[0].time --> start and end time objects\n",
"# datasets[0].properties --> dict of properties\n",
"\n",
"# Select your own fields\n",
"fields = {\n",
" 'name': None,\n",
" 'time': None,\n",
" 'gsd': None,\n",
" 'polarizations': None,\n",
" 'pass_direction': None,\n",
" 'antenna_pointing': None,\n",
" 'platform_heading': None,\n",
" 'incident_angle_near_range': None,\n",
" 'incident_angle_far_range': None,\n",
" 'source_product_level': None,\n",
" 'operational_mode_name': None,\n",
" 'filter_applied': None,\n",
" 'noise_removal_applied': None,\n",
"}\n",
"\n",
"# Parse into a pandas DataFrame\n",
"df = []\n",
"for ds in datasets:\n",
" ff = fields.copy()\n",
" # Non-properties fields\n",
" ff['name'] = ds.metadata.label\n",
" ff['time'] = ds.time[0] # time->range(start, end) or center_time->(end-start)/2\n",
" # Properties fields\n",
" for xx in fields:\n",
" if xx in ('name', 'time'):\n",
" continue\n",
" pre = 'eo' if xx == 'gsd' else 'novasar'\n",
" ff[xx] = ds.properties[f\"{pre}:{xx}\"]\n",
" df.append(ff)\n",
"df = pd.DataFrame(df)\n",
"display(df.sort_values('time') if not df.empty else Markdown(\"**Dataframe is empty**. Choose different product, space, time parameters\"))"
]
},
{
"cell_type": "markdown",
"id": "1f263ddb-f156-4841-8a01-9ced702cc820",
"metadata": {},
"source": [
"## Filter scenes based on metadata only\n",
"\n",
"This step is optional and configurable. Here we show a selection of simple filters:\n",
"\n",
"- Select descending passes only\n",
"- Select the N most recent scenes"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "93b759fa-21d1-4d4b-929a-38927e6e0448",
"metadata": {},
"outputs": [],
"source": [
"selected_datasets = datasets # Start with the full list\n",
"\n",
"# Select descending passes only\n",
"selected_datasets = [\n",
" xx for xx in selected_datasets if xx.properties[\"novasar:pass_direction\"] == \"DESCENDING\"\n",
"]\n",
"\n",
"# Select the N most recent scenes\n",
"selected_datasets = sorted(selected_datasets, key=lambda ds: ds.time[0])[-10:]\n",
"\n",
"display(Markdown(f\"**Number of selected scenes**: {len(selected_datasets)}\"))\n",
"selected_datasets"
]
},
{
"cell_type": "markdown",
"id": "9fefeef8-0c21-49c2-b86f-c68e5178a7ac",
"metadata": {
"tags": []
},
"source": [
"## Load data into a virtual dask array"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2f4c724f-636a-4833-a165-adb053acaa07",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Target xarray parameters\n",
"# - Target GeoBox or output CRS and resolution\n",
"# - Usually we group input scenes on the same day to a single time layer (groupby)\n",
"# - Select a reasonable Dask chunk size. This should be adjusted depending on the spatial extents and target grid parameters you choose\n",
"\n",
"# Default target grid is WGS-84, 20 m pixels for the given spatial extents\n",
"# - Create or reuse a GeoBox for precise mapping of input pixels to a target grid\n",
"# - Or provide output_crs and resolution, which will create best-fit GeoBox\n",
"\n",
"# Example similar Geobox constructor\n",
"# geobox = odc.geo.geobox.GeoBox.from_bbox(\n",
"# [query['x'][0], query['y'][1], query['x'][1], query['y'][0]],\n",
"# crs='epsg:4326', resolution=0.0002, anchor=0.0001\n",
"# )\n",
"\n",
"load_params = {\n",
" 'datasets': selected_datasets,\n",
" 'group_by': 'solar_day', # Scene grouping\n",
" 'dask_chunks': {'latitude':2048, 'longitude':2048}, # Dask chunks\n",
"}\n",
"\n",
"# Load data\n",
"data = dc.load(**(query | load_params))\n",
"\n",
"display(xarray_object_size(data))\n",
"display(data)"
]
},
{
"cell_type": "markdown",
"id": "e3d86506-39ba-4f02-ac6e-2d1d27119668",
"metadata": {},
"source": [
"## Conversion and helper functions"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "99eb8a2c-fe00-44b7-bd39-250ab3d323cc",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# These functions use numpy, which should be satisfactory for most notebooks.\n",
"# Calculations for larger or more complex arrays may require Xarray's \"ufunc\" capability.\n",
"# https://docs.xarray.dev/en/stable/examples/apply_ufunc_vectorize_1d.html\n",
"#\n",
"# Apply numpy.log10 to the DataArray\n",
"# log10_data = xr.apply_ufunc(np.log10, data)\n",
"\n",
"def dn_to_decibel(da: 'xr.DataArray', K=0):\n",
" \"\"\"Return an array converted to dB values\"\"\"\n",
" xx = da.where(da > 0, np.nan) # Set values <= 0 to NaN\n",
" xx = 20*np.log10(xx) + K\n",
" xx.attrs.update({\"units\": \"dB\"})\n",
" return xx\n",
"\n",
"def decibel_to_dn(da: 'xr.DataArray', K=0):\n",
" \"\"\"Return an array converted to digital number values\"\"\"\n",
" xx = np.power(10, (da-K)/20.0)\n",
" xx.attrs.update({\"units\": \"DN\"})\n",
" return xx\n",
"\n",
"def dn_to_amplitude(da: 'xr.DataArray'):\n",
" \"\"\"Return an array converted to linear amplitude values\"\"\"\n",
" scale_factor = da.attrs['scale_factor'] if 'scale_factor' in da.attrs else 1\n",
" # print(f\"DN to amplitude scale_factor: 1/{1/scale_factor} = {scale_factor}\")\n",
" xx = da.where(da > 0, np.nan) # Set values <= 0 to NaN\n",
" xx = xx * scale_factor\n",
" xx.attrs.update({\"units\": \"amplitude\"})\n",
" return xx\n",
"\n",
"def select_valid_time_layers(ds: 'xarray', percent: float = 5):\n",
" \"\"\"Select time layers that have at least a given percentage of valid data (e.g., >=5%)\n",
"\n",
" Example usage:\n",
" selected = select_valid_time_layers(ds, percent=5)\n",
" filtered == ds.sel(time=selected)\n",
" \"\"\"\n",
" spatial_dims = ds.odc.spatial_dims\n",
" nelements = ds.sizes[spatial_dims[0]] * ds.sizes[spatial_dims[1]]\n",
" if ds.dtype =='bool':\n",
" return ds.sum(dim=spatial_dims).values / nelements >= (percent/100.0)\n",
" return ds.count(dim=spatial_dims).values / nelements >= (percent/100.0)\n",
"\n",
"# Examples to check that the intensity to/from dB functions work as expected\n",
"# xx = data.vv.isel(time=0,latitude=np.arange(0, 5),longitude=np.arange(0, 5))\n",
"# xx[0] = 0 # manually change some values\n",
"# xx[1] = -0.001 # manually change some values\n",
"# display(\"digital numbers:\", xx.values)\n",
"# yy = dn_to_decibel(xx, K=-83)\n",
"# display(\"decibels:\", yy.values)\n",
"# zz = decibel_to_dn(yy, K=-83)\n",
"# display(\"digital numbers:\", zz.values)\n",
"# aa = dn_to_amplitude(xx)\n",
"# display(\"amplitude:\", aa.values)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "57bcf24c-99bb-4986-9afd-3a6e63b9a8cc",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# hvPlot convenience functions\n",
"def make_image(ds: 'xarray', frame_height=300, **kwargs):\n",
" \"\"\"Return a Holoviews DynamicMap (image) object that can be displayed or combined\"\"\"\n",
" spatial_dims = ds.odc.spatial_dims\n",
" defaults = dict(\n",
" cmap=\"Greys_r\",\n",
" y = spatial_dims[0], x = spatial_dims[1],\n",
" groupby = 'time',\n",
" rasterize = True,\n",
" geo = True,\n",
" robust = True,\n",
" frame_height = frame_height,\n",
" clabel = ds.attrs.get('units', None),\n",
" )\n",
" defaults.update(**kwargs)\n",
" return ds.hvplot.image(**defaults)\n",
"\n",
"def rgb_image(ds: 'xarray', frame_height=300, **kwargs):\n",
" \"\"\"Return a Holoviews DynamicMap (RBG image) object that can be displayed or combined\"\"\"\n",
" spatial_dims = ds.odc.spatial_dims\n",
" defaults = dict(\n",
" bands='band',\n",
" y = spatial_dims[0], x = spatial_dims[1],\n",
" groupby = 'time',\n",
" rasterize = True,\n",
" geo = True,\n",
" robust = True,\n",
" frame_height = frame_height,\n",
" )\n",
" defaults.update(**kwargs)\n",
" return ds.hvplot.rgb(**defaults)\n",
"\n",
"def mask_image(ds: 'xarray', frame_height=300, **kwargs):\n",
" \"\"\"Return a Holoviews DynamicMap (mask image) object that can be displayed or combined\"\"\"\n",
" # NovaSAR ARD mask\n",
" color_def = [\n",
" (0, '#ff0004', 'InvalidData'), # red\n",
" (1, '#eeeeee', 'ValidData'), # light grey\n",
" (5, '#ff52ff', 'Layover'), # cyan\n",
" (17, '#774c0b', 'Shadow'), # brown\n",
" (18, 'black', 'upper-limit'),\n",
" ]\n",
" cvals = [x[0] for x in color_def] # values, including upper-limit\n",
" cmap = [x[1] for x in color_def[0:-1]] # colors, excluding upper-limit\n",
" cticks = [(x[0], f\"[{x[0]}] {x[2]}\") for x in color_def[0:-1]] # labels, excluding upper-limit\n",
"\n",
" # Image options\n",
" defaults = {\n",
" 'aggregator': reductions.mode(),\n",
" 'cmap': cmap,\n",
" 'clim': (cvals[0], cvals[-1]),\n",
" 'colorbar': True,\n",
" 'frame_height': frame_height,\n",
" }\n",
" # Colorbar options for categories\n",
" extra_opts = {\n",
" 'color_levels': cvals,\n",
" 'cticks': cticks,\n",
" }\n",
" defaults.update(**kwargs)\n",
" return make_image(ds, **defaults).options(hv.opts.Image(**extra_opts)) #.hist(bins=bin_edges)"
]
},
{
"cell_type": "markdown",
"id": "d0592571-8f7e-471b-acf4-57eb93563b25",
"metadata": {},
"source": [
"## Filter scenes based on valid data\n",
"\n",
"This step is optional and configurable. Here we show another simple filter:\n",
"\n",
"- Exclude time layers with less than XX% valid data"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "91c113df-101a-49d2-911e-5a83e77ad14a",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Exclude time layers with less than XX% valid data\n",
"\n",
"data['mask'] = data.mask.persist()\n",
"valid_data_mask = masking.valid_data_mask(data.mask) # mask != mask.nodata -> bool\n",
"selected = select_valid_time_layers(valid_data_mask, 20) # Exclude time layers with less than 20% valid data\n",
"data = data.sel(time=selected)\n",
"\n",
"display(xarray_object_size(data))\n",
"display(data)"
]
},
{
"cell_type": "markdown",
"id": "8d244be5-8895-4d71-b9af-4ce026e14e01",
"metadata": {},
"source": [
"## Add dB and amplitude values to the dataset"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "961295e1-675b-4dda-92ba-c44b15506de8",
"metadata": {},
"outputs": [],
"source": [
"# Convenience loop for available polarization bands\n",
"\n",
"for pp in polarizations:\n",
" data[f\"{pp}_db\"] = dn_to_decibel(data[pp], K=-83).astype('float32')\n",
" data[f\"{pp}_amp\"] = dn_to_amplitude(data[pp]).astype('float32')\n",
"\n",
"display(xarray_object_size(data))\n",
"display(data)"
]
},
{
"cell_type": "markdown",
"id": "e0f94783-ad8d-4c28-b145-091bd3a5c9ae",
"metadata": {},
"source": [
"## Persist the data into the dask workers\n",
"\n",
"This will begin the data loading and calculations in the background.\n",
"\n",
"View progress in the dask dashboard (link given above where the dask cluster is defined)."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f4715d63-b9ee-410b-814d-f7b94730c516",
"metadata": {},
"outputs": [],
"source": [
"# Refine for selected variables if not plotting/analysing everything\n",
"data = data.persist()"
]
},
{
"cell_type": "markdown",
"id": "5fc56259-2e20-48f8-bab5-13ef56993bf9",
"metadata": {
"tags": []
},
"source": [
"## Plot the data\n",
"\n",
"- Stronger co-polarization (VV) indicates direct backscatter while stronger cross-polarization (VH) may indicate a complex surface or volume scattering.\n",
"- Amplitude data are linear-scaled so can tend to disciminate across a range of backscatter returns.\n",
"- Decibel data are log-scaled so can tend to discriminate high and low backscatter returns.\n",
"\n",
"> Note the different data ranges for plotting (`clim`) between `vv`, `vh`, _amplitude_ and _dB_."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2b306030-ecb7-47b4-a63d-27fdb491c4f9",
"metadata": {},
"outputs": [],
"source": [
"# VV, HH and HV (amplitude and dB) and Angle hvPlots\n",
"\n",
"clim = {\n",
" 'vv_amp': (0, 0.5),\n",
" 'hh_amp': (0, 0.5),\n",
" 'hv_amp': (0, 0.25),\n",
" 'vv_db': (-20, -5),\n",
" 'hh_db': (-20, -5),\n",
" 'hv_db': (-35, -10),\n",
"}\n",
"\n",
"bands_plot_list = []\n",
"for units in ('amplitude', 'dB'):\n",
" for pp in polarizations:\n",
" varname = f\"{pp}_{units[0:3].lower()}\"\n",
" title = f\"{pp.upper()} ({units})\"\n",
" bands_plot_list.append(\n",
" make_image(data[varname], title=title, clim=clim[varname])\n",
" )\n",
"\n",
"# Which is essentially doing this for all polarizations\n",
"# vv_plot = make_image(data.vv_amp, title='VV (amplitude)', clim=(0, 0.5))\n",
"# vv_db_plot = make_image(data.vv_db, title='VV (dB)', clim=(-20, -5))\n",
"\n",
"# # Add plots for the non-polarization bands\n",
"bands_plot_list.append(make_image(data.angle, title='Incidence angle'))\n",
"bands_plot_list.append(make_image(data.scatteringarea, title='Scattering area'))\n",
"bands_plot_list.append(make_image(data.gammatosigmaratio, title='Gamma to sigma ratio'))\n",
"bands_plot_list.append(mask_image(data.mask, title='Mask'))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6531e1f9-d8c0-45bd-aeaa-f7e1c95bbc7d",
"metadata": {},
"outputs": [],
"source": [
"# Arrange plots with linked axes and time slider. Adjust browser window width if required.\n",
"\n",
"num_plot_cols = 2 if len(polarizations) <= 2 else len(polarizations)\n",
"layout = pn.panel(\n",
" hv.Layout(bands_plot_list).cols(num_plot_cols),\n",
" widget_location='top',\n",
")\n",
"\n",
"print(layout) # Helpful to see how the hvplot is constructed\n",
"display(layout)"
]
},
{
"cell_type": "markdown",
"id": "77fbc6c5-4e0e-40c2-8c8f-82757ae0909b",
"metadata": {},
"source": [
"## Plot histograms of the dB data\n",
"\n",
"A histogram can help separate water from land features. Here we show a histogram for the _dB_ channels for all time layers.\n",
"- If the histogram shows two clear peaks then a value between the peaks could be used as a water / land threshold\n",
"- If not then try selected time layers, a different area of interest, or other channels or combinations."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2a122cfe-f2bd-4b63-983c-c625e6849f5e",
"metadata": {},
"outputs": [],
"source": [
"# vals, bins, hist_plot = data.hv_db.plot.hist(bins=np.arange(-30, 0, 1), color='red') # Matplotlib\n",
"\n",
"hist_plot_list = []\n",
"for pp in polarizations:\n",
" varname = f\"{pp}_db\"\n",
" title = f\"{pp.upper()} (dB), combined times\"\n",
" hist_plot_list.append(\n",
" data[varname].hvplot.hist(bins=np.arange(-30, 0, 1), color='red', title=title, height=300)\n",
" )\n",
"\n",
"layout = hv.Layout(hist_plot_list).cols(len(polarizations))\n",
"\n",
"print(layout) # Helpful to see how the hvplot is constructed\n",
"display(layout)"
]
},
{
"cell_type": "markdown",
"id": "1bbb8282-3b77-43ce-a774-88b031fb94a3",
"metadata": {},
"source": [
"## Make an RGB image\n",
"\n",
"A common strategy to create an RGB colour composite image for SAR data from two channels is to use the ratio of the channels to represent the third colour.\n",
"\n",
"Here we choose:\n",
"\n",
"- For a tri-pol (3 polarization bands) product we plot each polarization as red, green and blue.\n",
"- For a dual-pol (2 polarization bands) product we plot each polarization as red and green and the ratio as blue.\n",
"- For a single-pol (1 polarization band) product we can not make a useful RGB image\n",
"\n",
"Recall that:\n",
"- VV or HH ... direct scattering\n",
"- VH or HV ... complex scattering\n",
"- ratio ... relatively more of one than the other"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "082eeed7-dc16-4098-8bad-5dc748b6a79d",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Select bands combinations for RGB\n",
"if len(polarizations) == 3:\n",
" rgb_bands = polarizations\n",
"elif len(polarizations) == 2:\n",
" combo = \"_\".join(polarizations)\n",
" data[f\"{combo}\"] = data[polarizations[0]] / data[polarizations[1]]\n",
" rgb_bands = polarizations + [combo]\n",
"else:\n",
" display(Markdown(\"**Can not make a useful RGB image from one polarization band**. Choose a dual- or tri-pol product\"))\n",
" rgb_bands = []"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "63a142d3-f278-49c8-a635-bb219675f239",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"if rgb_bands:\n",
" # Scale RGB bands by their median so they have a similar range for visualization\n",
" spatial_dims = data.odc.spatial_dims\n",
" for bb in rgb_bands:\n",
" data[f\"{bb}_scaled\"] = (data[bb] / data[bb].median(dim=spatial_dims))\n",
" rgb_bands = [f\"{bb}_scaled\" for bb in rgb_bands]\n",
"\n",
" # odc-geo function\n",
" rgb_data = data.odc.to_rgba(bands=rgb_bands, vmin=0, vmax=2).persist()\n",
"\n",
" # As subplots\n",
" # rgb_plot = rgb_image(\n",
" # rgb_data,\n",
" # ).layout().cols(4)\n",
"\n",
" # As movie. Select \"loop\" and use \"-\" button to adjust the speed to allow for rendering. After a few cycles the images should play reasonably well.\n",
" rgb_plot = rgb_image(\n",
" rgb_data,\n",
" precompute = True,\n",
" widget_type='scrubber', widget_location='bottom',\n",
" frame_height = 500,\n",
" )"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "bd2da45d-a1db-4d75-b725-706d0fa252c9",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"if rgb_bands:\n",
" print(rgb_plot) # Helpful to see how the hvplot is constructed\n",
" display(rgb_plot)"
]
},
{
"cell_type": "markdown",
"id": "fd1df15e-31e8-4ad8-9940-e28bd6822441",
"metadata": {},
"source": [
"## Export to Geotiffs\n",
"\n",
"Recall that to write a dask dataset to a file requires the dataset to be `.compute()`ed. This may result in a large memory increase on your JupyterLab node if the area of interest is large enough, which in turn may kill the kernel. If so then skip this step, choose a smaller area or find a different way to export data."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1347d080-74d5-491a-bb45-45252fc385ea",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Make a directory to save outputs to\n",
"target = Path.home() / 'output'\n",
"if not target.exists(): target.mkdir()\n",
"\n",
"def write_band(ds, varname):\n",
" \"\"\"Write the variable name of the xarray dataset to a Geotiff file for each time layer\"\"\"\n",
" for i in range(len(ds.time)):\n",
" date = ds[varname].isel(time=i).time.dt.strftime('%Y%m%d').data\n",
" fname = f'{target}/example_novasar-1_{varname}_{date}.tif'\n",
" single = ds[varname].isel(time=i).compute()\n",
" single.odc.write_cog(\n",
" fname=fname,\n",
" overwrite=True,\n",
" )\n",
" print(f'Wrote: {fname}')\n",
"\n",
"for pp in polarizations:\n",
" write_band(data, pp)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ef7c0753-3ba2-4736-acf0-3f52ca36bd17",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.11"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
notebooks/data_products/sentinel-1-gamma0.ipynb
+716
View File
@@ -0,0 +1,716 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "dae926b2-fa06-4f00-b18d-7f74e92ce676",
"metadata": {},
"source": [
"## Sentinel-1 RTC Gamma0 data <img align=\"right\" src=\"../../resources/csiro_easi_logo.png\">\n",
"\n",
"#### Index\n",
"- [Overview](#Overview)\n",
"- [Setup (imports, defaults, dask, odc)](#Setup)\n",
"- [Example query](#Example-query)\n",
"- [Product definition](#Product-definition)\n",
"- [Quality layer](#Quality-layer)\n",
"- [Create and apply a good quality pixel mask](#Create-and-apply-a-good-quality-pixel-mask)\n",
"- [Plot and browse the data](#Plot-and-browse-the-data)"
]
},
{
"cell_type": "markdown",
"id": "35b91cdb-2c5a-48c0-a0bf-7de2c2637bc5",
"metadata": {},
"source": [
"## Overview\n",
"\n",
"This notebook demonstrates how to load and use Sentinel-1 Radiometric Terrain Corrected (RTC) Gamma0 data generated in EASI.\n",
"\n",
"These _analysis ready data_ S1 gamma-0 backscatter data are processed from Sentinel-1 GRD scenes using the [SNAP-10 Toolbox](https://step.esa.int/main/download/snap-download/) with Graph Processing Tool (GPT) xml receipes. See the [RTC Gamma0 product variants](#RTC-Gamma0-product-variants) section for further details.\n",
"\n",
"For most uses we recommend the smoothed 20 m product (`sentinel1_grd_gamma0_20m`).\n",
"We can process the 10 m products (`sentinel1_grd_gamma0_10m`, `sentinel1_grd_gamma0_10m_unsmooth`) and other variants on request."
]
},
{
"cell_type": "markdown",
"id": "5e5120c6-e943-46a9-a4d8-c5b759586e62",
"metadata": {},
"source": [
"#### Using Sentinel-1 backscatter data\n",
"\n",
"An excellent introduction and overview to using SAR data is provided in the [CEOS Laymans SAR Interpretation Guide](https://ceos.org/ard/files/Laymans_SAR_Interpretation_Guide_3.0.pdf). This guide has also been converted to a set of set of Jupyter notebooks that you can download from https://github.com/AMA-Labs/cal-notebooks/tree/main/examples/SAR.\n",
"\n",
"Synthetic Aperture Radar operates in the microwave range of the electromagnetic spectrum as an active pulse sent by the satellite and scattered by features on the Earth's surface. The return signal from the surface is measured at the satellite in terms of the signal intensity, phase and polarisation compared to the signal that was sent.\n",
"\n",
"The SAR instrument on the Sentinel-1 satellites operate in the C-band at approximately 5.6 cm wavelength. This means that it can \"see\" objects of about this size and larger, and smaller objects are relatively transparent. This makes Sentinel-1 more sensitive to tree canopies, sparse and low biomass vegetation, and surface water (smooth and wind affected).\n",
"\n",
"> The SAR signal responds to the orientation and scattering from surface features of comparable size or larger than the wavelength.\n",
"> - A bright backscatter value typically means the surface was orientated perpendicular to the signal incidence angle and most of the signal was reflected back to the satellite (direct backscatter)\n",
"> - A dark backscatter value means most of the signal was reflected away from the satellite (forward scattering) and typically responds to a smooth surface (relative to the wavelength) such as calm water or bare soil\n",
"> - Rough surfaces (relative to the wavelength) result in diffuse scattering where some of the signal is returned to the satellite.\n",
"> - Complex surfaces may result in volume scattering (scattering within a tree canopy) or double-bounce scattering (perpendicular objects such as buildings and structures)\n",
"> - The relative backscatter values of co-polarisation (VV) and cross-polarisation (VH) measurements can provide information on the scattering characteristics of the surface features.\n",
"\n",
"Using Sentinel-1 backscatter data requires interpretation of the data for different surface features, including as these features change spatially or in time. It may also be necessary to carefully consider the incidence angle of the SAR signal relative to the surface features using the _incidence_angle_ band or the satellite direction metadata (descending = north to south; ascending = south to north)."
]
},
{
"cell_type": "markdown",
"id": "69f88789-9098-4669-b783-57d8982e9762",
"metadata": {
"jp-MarkdownHeadingCollapsed": true
},
"source": [
"#### Units and conversions\n",
"The `sentinel1_grd_gamma0_*` data are given in _Intensity_ (or backscatter _power_) units. Intensity can be converted to _decibel (dB)_ or _amplitude_, and vice-versa, with the following equations. Practical _Xarray_ examples are given below.\n",
"\n",
"Intensity to/from dB:\n",
"```\n",
" dB = 10 * log10(intensity) + K\n",
"intensity = 10^((dB-K)/10)\n",
"\n",
"where K is a calibration factor, which for Sentinel-1 is 0 dB.\n",
"```\n",
"\n",
"Intensity to/from Amplitude:\n",
"```\n",
"intensity = amplitude * amplitude\n",
"amplitude = sqrt(intensity)\n",
"```\n",
"\n",
"Additional reference: https://forum.step.esa.int/t/what-stage-of-processing-requires-the-linear-to-from-db-command"
]
},
{
"cell_type": "markdown",
"id": "dd5c0ff4-e1a2-4adf-98d1-1ca5b9f3126d",
"metadata": {},
"source": [
"## Set up"
]
},
{
"cell_type": "markdown",
"id": "4cb7c012-719e-4f13-a206-36f271c6a91c",
"metadata": {},
"source": [
"#### Imports"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1c848c17-2e7c-4b68-9682-b8994644f521",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Common imports and settings\n",
"import os, sys, re\n",
"from pathlib import Path\n",
"from IPython.display import Markdown\n",
"import pandas as pd\n",
"pd.set_option(\"display.max_rows\", None)\n",
"import xarray as xr\n",
"import numpy as np\n",
"\n",
"# Datacube\n",
"import datacube\n",
"from datacube.utils.aws import configure_s3_access\n",
"import odc.geo.xr # https://github.com/opendatacube/odc-geo\n",
"from datacube.utils import masking # https://github.com/opendatacube/datacube-core/blob/develop/datacube/utils/masking.py\n",
"from dea_tools.plotting import display_map # https://github.com/GeoscienceAustralia/dea-notebooks/tree/develop/Tools\n",
"\n",
"# Basic plots\n",
"%matplotlib inline\n",
"# import matplotlib.pyplot as plt\n",
"# plt.rcParams['figure.figsize'] = [12, 8]\n",
"\n",
"# Holoviews\n",
"# https://holoviz.org/tutorial/Composing_Plots.html\n",
"# https://holoviews.org/user_guide/Composing_Elements.html\n",
"import hvplot.xarray\n",
"import panel as pn"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "bde8313b-887d-4529-bd57-89f740ff6380",
"metadata": {},
"outputs": [],
"source": [
"# EASI defaults\n",
"# These are convenience functions so that the notebooks in this repository work in all EASI deployments\n",
"\n",
"# The `git.Repo()` part returns the local directory that easi-notebooks has been cloned into\n",
"# If using the `easi-tools` functions from another path, replace `repo` with your local path to `easi-notebooks` directory\n",
"try:\n",
" import git\n",
" repo = git.Repo('.', search_parent_directories=True).working_tree_dir # Path to this cloned local directory\n",
"except (ImportError, git.InvalidGitRepositoryError):\n",
" repo = Path.home() / 'easi-notebooks' # Reasonable default\n",
" if not repo.is_dir():\n",
" raise RuntimeError('To use `easi-tools` please provide the local path to `https://github.com/csiro-easi/easi-notebooks`')\n",
"if repo not in sys.path:\n",
" sys.path.append(str(repo)) # Add the local path to `easi-notebooks` to python\n",
"\n",
"from easi_tools import EasiDefaults\n",
"from easi_tools import initialize_dask, xarray_object_size, heading"
]
},
{
"cell_type": "markdown",
"id": "d2916064-8680-44a2-b537-0ecdd0b3e05b",
"metadata": {
"tags": []
},
"source": [
"#### EASI defaults\n",
"\n",
"These default values are configured for each EASI instance. They help us to use the same training notebooks in each EASI instance. You may find some of the functions convenient for your work or you can easily override the values in your copy of this notebook."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "44d6583e-d2f5-4587-b035-6814d1313740",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"easi = EasiDefaults()\n",
"\n",
"family = 'sentinel-1'\n",
"product = easi.product(family) # 'sentinel1_grd_gamma0_20m'\n",
"display(Markdown(f'Default {family} product for \"{easi.name}\": [{product}]({easi.explorer}/products/{product})'))"
]
},
{
"cell_type": "markdown",
"id": "bb66366a-f2d5-461a-a34f-7eed5a5296fb",
"metadata": {
"tags": []
},
"source": [
"#### Dask cluster\n",
"\n",
"Using a local _Dask_ cluster is a good habit to get into. It can simplify loading and processing of data in many cases, and it provides a dashboard that shows the loading/processing progress.\n",
"\n",
"To learn more about _Dask_ see the set of [dask notebooks](https://github.com/csiro-easi/easi-notebooks/tree/main/html#dask-tutorials)."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "02f9e778-755c-4ff2-a99f-ad5c48438665",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Local cluster\n",
"cluster, client = initialize_dask(workers=4)\n",
"display(client)\n",
"\n",
"# Or use Dask Gateway - this may take a few minutes\n",
"# cluster, client = initialize_dask(use_gateway=True, workers=4)\n",
"# display(client)"
]
},
{
"cell_type": "markdown",
"id": "a746582e-be79-448c-a9c4-7a2ae2d703ce",
"metadata": {},
"source": [
"#### ODC database\n",
"\n",
"Connect to the ODC database. Configure the environment and low-level tools to read from AWS buckets."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0e2232ee-f0a5-43fc-a6c6-d422b437cd69",
"metadata": {},
"outputs": [],
"source": [
"dc = datacube.Datacube()\n",
"\n",
"# Access AWS \"requester-pays\" buckets\n",
"# This is necessary for reading data from most third-party AWS S3 buckets such as for Landsat and Sentinel-2\n",
"configure_s3_access(aws_unsigned=False, requester_pays=True, client=client);"
]
},
{
"cell_type": "markdown",
"id": "1801db2b-e24e-45fa-85d4-e153ebcc8f6f",
"metadata": {},
"source": [
"## Example query\n",
"\n",
"Change any of the parameters in the `query` object below to adjust the location, time, projection, or spatial resolution of the returned datasets.\n",
"\n",
"Use the Explorer interface to check the temporal and spatial coverage for each product."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2a04d875-20d4-4d72-a16d-e76b7f48161c",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Explorer link\n",
"display(Markdown(f'See: {easi.explorer}/products/{product}'))\n",
"\n",
"# EASI defaults\n",
"display(Markdown(f'#### Location: {easi.location}'))\n",
"latitude_range = easi.latitude\n",
"longitude_range = easi.longitude\n",
"time_range = easi.time\n",
"\n",
"# Or set your own latitude / longitude\n",
"# Australia GWW\n",
"# latitude_range = (-33, -32.6)\n",
"# longitude_range = (120.5, 121)\n",
"# time_range = ('2020-01-01', '2020-01-31')\n",
"\n",
"query = {\n",
" 'product': product, # Product name\n",
" 'x': longitude_range, # \"x\" axis bounds\n",
" 'y': latitude_range, # \"y\" axis bounds\n",
" 'time': time_range, # Any parsable date strings\n",
"}\n",
"\n",
"# Convenience function to display the selected area of interest\n",
"display_map(longitude_range, latitude_range)"
]
},
{
"cell_type": "markdown",
"id": "9fefeef8-0c21-49c2-b86f-c68e5178a7ac",
"metadata": {
"tags": []
},
"source": [
"## Load data"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2f4c724f-636a-4833-a165-adb053acaa07",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Target xarray parameters\n",
"# - Select a set of measurements to load\n",
"# - output CRS and resolution\n",
"# - Usually we group input scenes on the same day to a single time layer (groupby)\n",
"# - Select a reasonable Dask chunk size (this should be adjusted depending on the\n",
"# spatial and resolution parameters you choose\n",
"load_params = {\n",
" 'group_by': 'solar_day', # Scene grouping\n",
" 'dask_chunks': {'latitude':2048, 'longitude':2048}, # Dask chunks\n",
"}\n",
"\n",
"# Load data\n",
"data = dc.load(**(query | load_params))\n",
"display(xarray_object_size(data))\n",
"display(data)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "be474a1a-b5f1-40e0-b35a-5b9c9f1a15bd",
"metadata": {},
"outputs": [],
"source": [
"# When happy with the shape and size of chunks, persist() the result\n",
"data = data.persist()"
]
},
{
"cell_type": "markdown",
"id": "e3d86506-39ba-4f02-ac6e-2d1d27119668",
"metadata": {},
"source": [
"## Conversion and helper functions"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "99eb8a2c-fe00-44b7-bd39-250ab3d323cc",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# These functions use numpy, which should be satisfactory for most notebooks.\n",
"# Calculations for larger or more complex arrays may require Xarray's \"ufunc\" capability.\n",
"# https://docs.xarray.dev/en/stable/examples/apply_ufunc_vectorize_1d.html\n",
"#\n",
"# Apply numpy.log10 to the DataArray\n",
"# log10_data = xr.apply_ufunc(np.log10, data)\n",
"\n",
"def intensity_to_db(da: 'xr.DataArray', K=0):\n",
" \"\"\"Return an array converted to dB values\"\"\"\n",
" xx = da.where(da > 0, np.nan) # Set values <= 0 to NaN\n",
" xx = 10*np.log10(xx) + K\n",
" xx.attrs.update({\"units\": \"dB\"})\n",
" return xx\n",
"\n",
"def db_to_intensity(da: 'xr.DataArray', K=0):\n",
" \"\"\"Return an array converted to intensity values\"\"\"\n",
" xx = np.power(10, (da-K)/10.0)\n",
" xx.attrs.update({\"units\": \"intensity\"})\n",
" return xx\n",
"\n",
"def select_valid_time_layers(ds: 'xarray', percent: float = 5):\n",
" \"\"\"Select time layers that have at least a given percentage of valid data (e.g., >=5%)\n",
"\n",
" Example usage:\n",
" selected = select_valid_time_layers(ds, percent=5)\n",
" filtered == ds.sel(time=selected)\n",
" \"\"\"\n",
" spatial_dims = ds.odc.spatial_dims\n",
" return ds.count(dim=spatial_dims).values / (ds.sizes[spatial_dims[0]]*ds.sizes[spatial_dims[1]]) >= (percent/100.0)\n",
"\n",
"# Examples to check that the intensity to/from dB functions work as expected\n",
"# xx = data.vv.isel(time=0,latitude=np.arange(0, 5),longitude=np.arange(0, 5))\n",
"# xx[0] = 0\n",
"# xx[1] = -0.001\n",
"# display(xx.values)\n",
"# yy = intensity_to_db(xx)\n",
"# display(yy.values)\n",
"# zz = db_to_intensity(yy)\n",
"# display(zz.values)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "57bcf24c-99bb-4986-9afd-3a6e63b9a8cc",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# hvPlot convenience functions\n",
"def make_image(ds: 'xarray', frame_height=300, **kwargs):\n",
" \"\"\"Return a Holoviews DynamicMap (image) object that can be displayed or combined\"\"\"\n",
" spatial_dims = ds.odc.spatial_dims\n",
" defaults = dict(\n",
" cmap=\"Greys_r\",\n",
" y = spatial_dims[0], x = spatial_dims[1],\n",
" groupby = 'time',\n",
" rasterize = True,\n",
" geo = True,\n",
" robust = True,\n",
" frame_height = frame_height,\n",
" clabel = ds.attrs.get('units', None),\n",
" )\n",
" defaults.update(**kwargs)\n",
" return ds.hvplot.image(**defaults)\n",
"\n",
"def rgb_image(ds: 'xarray', frame_height=300, **kwargs):\n",
" \"\"\"Return a Holoviews DynamicMap (RBG image) object that can be displayed or combined\"\"\"\n",
" spatial_dims = ds.odc.spatial_dims\n",
" defaults = dict(\n",
" bands='band',\n",
" y = spatial_dims[0], x = spatial_dims[1],\n",
" groupby = 'time',\n",
" rasterize = True,\n",
" geo = True,\n",
" robust = True,\n",
" frame_height = frame_height,\n",
" )\n",
" defaults.update(**kwargs)\n",
" return ds.hvplot.rgb(**defaults)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "91c113df-101a-49d2-911e-5a83e77ad14a",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Optional time layer filter\n",
"\n",
"selected = select_valid_time_layers(data.vv, 10) # Exclude time layers with less than 10% valid data\n",
"data = data.sel(time=selected).persist()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "961295e1-675b-4dda-92ba-c44b15506de8",
"metadata": {},
"outputs": [],
"source": [
"# Add db values to the dataset\n",
"\n",
"data['vh_db'] = intensity_to_db(data.vh).persist()\n",
"data['vv_db'] = intensity_to_db(data.vv).persist()"
]
},
{
"cell_type": "markdown",
"id": "5fc56259-2e20-48f8-bab5-13ef56993bf9",
"metadata": {
"tags": []
},
"source": [
"## Plot the data\n",
"\n",
"> Note the different data ranges for plotting (`clim`) between `vv`, `vh`, _intensity_ and _dB_.\n",
"\n",
"- Stronger co-polarisation (VV) indicates direct backscatter while stronger cross-polarisation (VH) may indicate a complex surface or volume scattering.\n",
"- Intensity data are linear-scaled so can tend to disciminate across a range of backscatter returns.\n",
"- Decibel data are log-scaled so can tend to discriminate high and low backscatter returns."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a7630dec-2f78-41b5-b1b4-0eb32206c120",
"metadata": {},
"outputs": [],
"source": [
"# VV and VH (intensity and dB) and Angle hvPlots\n",
"\n",
"vv_plot = make_image(data.vv, clim=(0, 0.5), title='VV (intensity)')\n",
"vh_plot = make_image(data.vh, clim=(0, 0.1), title='VH (intensity)')\n",
"ia_plot = make_image(data.angle, title='Incidence angle')\n",
"\n",
"vv_db_plot = make_image(data.vv_db, clim=(-30, -3), title='VV (dB)')\n",
"vh_db_plot = make_image(data.vh_db, clim=(-30, -1), title='VH (dB)')"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6531e1f9-d8c0-45bd-aeaa-f7e1c95bbc7d",
"metadata": {},
"outputs": [],
"source": [
"# Arrange plots with linked axes and time slider. Adjust browser window width if required.\n",
"\n",
"layout = pn.panel(\n",
" (vv_plot + vh_plot + ia_plot + vv_db_plot + vh_db_plot).cols(3),\n",
" widget_location='top',\n",
")\n",
"print(layout) # Helpful to see how the hvplot is constructed\n",
"layout"
]
},
{
"cell_type": "markdown",
"id": "77fbc6c5-4e0e-40c2-8c8f-82757ae0909b",
"metadata": {},
"source": [
"## Plot a histogram of the dB data\n",
"\n",
"A histogram can help separate water from land features. Here we show a histogram for the _VH (db)_ channel for all time layers.\n",
"- If the histogram shows two clear peaks then a value between the peaks could be used as a water / land threshold\n",
"- If not then try selected time layers, a different area of interest, or other channels or combinations."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2a122cfe-f2bd-4b63-983c-c625e6849f5e",
"metadata": {},
"outputs": [],
"source": [
"# vals, bins, hist_plot = data.vh_db.plot.hist(bins=np.arange(-30, 0, 1), color='red') # Matplotlib\n",
"hist_plot = data.vh_db.hvplot.hist(bins=np.arange(-30, 0, 1), color='red', title='Combined times', height=400) # hvPlot\n",
"\n",
"print(hist_plot) # Helpful to see how the hvplot is constructed\n",
"hist_plot"
]
},
{
"cell_type": "markdown",
"id": "1bbb8282-3b77-43ce-a774-88b031fb94a3",
"metadata": {},
"source": [
"## Make an RGB image\n",
"\n",
"A common strategy to create an RGB colour composite image for SAR data from two channels is to use the ratio of the channels to represent the third colour. Here we choose\n",
"\n",
"To create an RGB colour composite image we can use the ratio of VH and VV to represent a third channel. Here we choose\n",
"- Red = VH ... complex scattering\n",
"- Green = VV ... direct scattering\n",
"- Blue = VH/VV ... relatively more complex than direct"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "bd2da45d-a1db-4d75-b725-706d0fa252c9",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Add the vh/vv band to represent 'blue'\n",
"data['vh_vv'] = data.vh / data.vv\n",
"\n",
"# Scale the measurements by their median so they have a similar range for visualization\n",
"spatial_dims = data.odc.spatial_dims\n",
"data['vh_scaled'] = data.vh / data.vh.median(dim=spatial_dims).persist()\n",
"data['vv_scaled'] = data.vv / data.vv.median(dim=spatial_dims).persist()\n",
"data['vh_vv_scaled'] = data.vh_vv / data.vh_vv.median(dim=spatial_dims).persist()\n",
"\n",
"# odc-geo function\n",
"rgb_data = data.odc.to_rgba(bands=['vh_scaled','vv_scaled','vh_vv_scaled'], vmin=0, vmax=2)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a8834880-fb2d-42ec-bc01-f1b37ac97038",
"metadata": {},
"outputs": [],
"source": [
"# As subplots\n",
"# rgb_plot = rgb_image(\n",
"# rgb_data,\n",
"# ).layout().cols(4)\n",
"\n",
"# As movie. Select \"loop\" and use \"-\" button to adjust the speed to allow for rendering. After a few cycles the images should play reasonably well.\n",
"rgb_plot = rgb_image(\n",
" rgb_data,\n",
" precompute = True,\n",
" widget_type='scrubber', widget_location='bottom',\n",
" frame_height = 500,\n",
")\n",
"\n",
"print(rgb_plot) # Helpful to see how the hvplot is constructed\n",
"rgb_plot"
]
},
{
"cell_type": "markdown",
"id": "fd1df15e-31e8-4ad8-9940-e28bd6822441",
"metadata": {},
"source": [
"## Export to Geotiffs\n",
"\n",
"Recall that to write a dask dataset to a file requires the dataset to be `.compute()`ed. This may result in a large memory increase on your JupyterLab node if the area of interest is large enough, which in turn may kill the kernel. If so then skip this step, choose a smaller area or find a different way to export data."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1347d080-74d5-491a-bb45-45252fc385ea",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Make a directory to save outputs to\n",
"target = Path.home() / 'output'\n",
"if not target.exists(): target.mkdir()\n",
"\n",
"def write_band(ds, varname):\n",
" \"\"\"Write the variable name of the xarray dataset to a Geotiff file for each time layer\"\"\"\n",
" for i in range(len(ds.time)):\n",
" date = ds[varname].isel(time=i).time.dt.strftime('%Y%m%d').data\n",
" fname = f'{target}/example_sentinel-1_{varname}_{date}.tif'\n",
" single = ds[varname].isel(time=i).compute()\n",
" single.odc.write_cog(\n",
" fname=fname,\n",
" overwrite=True,\n",
" )\n",
" print(f'Wrote: {fname}')\n",
" \n",
"write_band(data, 'vv')\n",
"write_band(data, 'vh')"
]
},
{
"cell_type": "markdown",
"id": "7ed2add5-12ad-4327-801b-9e60b3baf031",
"metadata": {},
"source": [
"## Appendix"
]
},
{
"cell_type": "markdown",
"id": "776027ff-c4ef-4d34-962f-b2a33ba09a60",
"metadata": {},
"source": [
"### RTC Gamma0 product variants\n",
"\n",
"The set of products listed here differ by the selection and configuration of processing steps and options. The set of SNAP operators conform with [CEOS Analysis Ready Data](https://ceos.org/ard/) specifications for _normalised radar backscatter_.\n",
"\n",
"S1 gamma-0 backscatter data are processed from Sentinel-1 GRD scenes using the [SNAP-10 Toolbox](https://step.esa.int/main/download/snap-download/) with Graph Processing Tool (GPT) xml receipes (available on request).\n",
"\n",
"| | sentinel1_grd_gamma0_20m | sentinel1_grd_gamma0_10m | sentinel1_grd_gamma0_10m_unsmooth |\n",
"|--|--|--|--|\n",
"| **DEM** | | | |\n",
"| copernicus_dem_30 | Y | Y | Y |\n",
"| Scene to DEM extent multiplier| 3.0 | 3.0 | 3.0 |\n",
"| **SNAP operator** | | | |\n",
"| Apply-Orbit-File | Y | Y | Y |\n",
"| ThermalNoiseRemoval | Y | Y | Y |\n",
"| Remove-GRD-Border-Noise | Y | Y | Y |\n",
"| Calibration | Y | Y | Y |\n",
"| SetNoDataValue | Y | Y | Y |\n",
"| Terrain-Flattening | Y | Y | Y |\n",
"| Speckle-Filter | Y | Y | N |\n",
"| Multilook | Y | Y | N |\n",
"| Terrain-Correction | Y | Y | Y |\n",
"| **Output** | | | |\n",
"| Projection | WGS84, epsg:4326 | WGS84, epsg:4326 | WGS84, epsg:4326 |\n",
"| Pixel resolution | 20 m | 10 m | 10 m |\n",
"| Pixel alignment</br>_PixelIsArea = top-left_ | PixelIsArea | PixelIsArea | PixelIsArea |"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ef7c0753-3ba2-4736-acf0-3f52ca36bd17",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "folium",
"language": "python",
"name": "folium"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.7"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
notebooks/data_products/sentinel-2-c1-l2a.ipynb
+632
View File
@@ -0,0 +1,632 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {
"tags": []
},
"source": [
"# Sentinel-2 Collection 1, L2A <img align=\"right\" src=\"../../resources/csiro_easi_logo.png\">\n",
"\n",
"#### Index\n",
"- [Overview](#Overview)\n",
"- [Setup (imports, defaults, dask, odc)](#Setup)\n",
"- [Example query](#Example-query)\n",
"- [Product definition](#Product-definition)\n",
"- [Quality layer](#Quality-layer)\n",
"- [Create and apply a good quality pixel mask](#Create-and-apply-a-good-quality-pixel-mask)\n",
"- [Plot and browse the data](#Plot-and-browse-the-data)"
]
},
{
"cell_type": "markdown",
"metadata": {
"tags": []
},
"source": [
"## Overview\n",
"\n",
"Sentinel-2 is an Earth observation mission from the EU Copernicus Programme that systematically acquires optical imagery at high spatial resolution (up to 10 m for some bands). The mission is a constellation of two identical satellites in the same orbit, 180° apart for optimal coverage and data delivery. Together, they cover all Earth's land surfaces, large islands, inland and coastal waters every 3-5 days.\n",
"\n",
"Sentinel-2A was launched on 23 June 2015 and Sentinel-2B followed on 7 March 2017.\n",
"Both of the Sentinel-2 satellites carry a wide swath high-resolution multispectral imager with 13 spectral bands.\n",
"For more information see:\n",
"- [ESA Sentinel missions](https://www.esa.int/Applications/Observing_the_Earth/Copernicus/The_Sentinel_missions)\n",
"- [Sentinel-2 technical specifications](https://sentinels.copernicus.eu/web/sentinel/technical-guides/sentinel-2-msi)\n",
"\n",
"_Selected text adapted from https://github.com/GeoscienceAustralia/dea-notebooks/blob/develop/DEA_datasets/Sentinel_2.ipynb_\n",
"\n",
"#### Data source and documentation\n",
"\n",
"ESA produces a surface reflectance \"S2 Collection 1 L2A\" product using their [sen2cor](https://step.esa.int/main/snap-supported-plugins/sen2cor/) software. [Element84](https://github.com/Element84/earth-search) convert these data to Cloud-Optimized Geotiff format and makes them publicly available at their [Earth Search STAC API](https://earth-search.aws.element84.com/v1) endpoint for programmatic access.\n",
"\n",
"EASI uses its STAC indexing tools to index datasets into our ODC databases.\n",
"\n",
"| Name | Product | Source | Information | Index\n",
"|--|--|--|--|--|\n",
"| Sentinel-2 C1 L2A COGs | `sentinel_2_c1_l2a` | [Earth Search STAC](https://earth-search.aws.element84.com/v1/collections/sentinel-2-c1-l2a) | Use for global (land) surface reflectance | Select COGS via STAC and convert to ODC metadata\n",
"\n",
"#### Collection 1 baseline processing\n",
"\n",
"ESA is reprocessing Sentinel-2 to a [\"Collection 1\"](https://sentinels.copernicus.eu/web/sentinel/sentinel-data-access/sentinel-products/sentinel-2-data-products/collection-1-level-2a) product using processing baseline >=5.00.\n",
"\n",
"Element-84 are kindly processing these data to COGs. Their processing documentation and advice is expected to be made available once the full repreocessing is completed by ESA and Element84 (see [issues](https://github.com/Element84/earth-search/issues)).\n",
"\n",
"In the meantime this notebook will show how we think the data should be loaded and used."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Setup"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Imports"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Common imports and settings\n",
"import os, sys\n",
"from pathlib import Path\n",
"# os.environ['USE_PYGEOS'] = '0'\n",
"from IPython.display import Markdown\n",
"import pandas as pd\n",
"pd.set_option(\"display.max_rows\", None)\n",
"import xarray as xr\n",
"\n",
"# Datacube\n",
"import datacube\n",
"from datacube.utils.aws import configure_s3_access\n",
"import odc.geo.xr # https://github.com/opendatacube/odc-geo\n",
"from datacube.utils import masking # https://github.com/opendatacube/datacube-core/blob/develop/datacube/utils/masking.py\n",
"from odc.algo import enum_to_bool # https://github.com/opendatacube/odc-tools/blob/develop/libs/algo/odc/algo/_masking.py\n",
"from dea_tools.plotting import display_map, rgb # https://github.com/GeoscienceAustralia/dea-notebooks/tree/develop/Tools\n",
"\n",
"# Basic plots\n",
"%matplotlib inline\n",
"# import matplotlib.pyplot as plt\n",
"# plt.rcParams['figure.figsize'] = [12, 8]\n",
"\n",
"# Holoviews\n",
"# https://holoviz.org/tutorial/Composing_Plots.html\n",
"# https://holoviews.org/user_guide/Composing_Elements.html\n",
"import hvplot.xarray\n",
"import panel as pn\n",
"import colorcet as cc\n",
"import cartopy.crs as ccrs\n",
"from datashader import reductions\n",
"from holoviews import opts"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# EASI defaults\n",
"# These are convenience functions so that the notebooks in this repository work in all EASI deployments\n",
"\n",
"# The `git.Repo()` part returns the local directory that easi-notebooks has been cloned into\n",
"# If using the `easi-tools` functions from another path, replace `repo` with your local path to `easi-notebooks` directory\n",
"try:\n",
" import git\n",
" repo = git.Repo('.', search_parent_directories=True).working_tree_dir # Path to this cloned local directory\n",
"except (ImportError, git.InvalidGitRepositoryError):\n",
" repo = Path.home() / 'easi-notebooks' # Reasonable default\n",
" if not repo.is_dir():\n",
" raise RuntimeError('To use `easi-tools` please provide the local path to `https://github.com/csiro-easi/easi-notebooks`')\n",
"if repo not in sys.path:\n",
" sys.path.append(str(repo)) # Add the local path to `easi-notebooks` to python\n",
"\n",
"from easi_tools import EasiDefaults\n",
"from easi_tools import initialize_dask, xarray_object_size, mostcommon_crs, heading"
]
},
{
"cell_type": "markdown",
"metadata": {
"tags": []
},
"source": [
"#### EASI defaults\n",
"\n",
"These default values are configured for each EASI instance. They help us to use the same training notebooks in each EASI instance. You may find some of the functions convenient for your work or you can easily override the values in your copy of this notebook."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"easi = EasiDefaults()\n",
"\n",
"family = 'sentinel-2'\n",
"product = 'sentinel_2_c1_l2a' # Sentinel-2 collection 1, L2A\n",
"display(Markdown(f'Default {family} product for \"{easi.name}\": [{product}]({easi.explorer}/products/{product})'))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Dask cluster\n",
"\n",
"Using a local _Dask_ cluster is a good habit to get into. It can simplify loading and processing of data in many cases, and it provides a dashboard that shows the loading/processing progress.\n",
"\n",
"To learn more about _Dask_ see the set of [dask notebooks](https://github.com/csiro-easi/easi-notebooks/tree/main/html#dask-tutorials)."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Local cluster\n",
"cluster, client = initialize_dask(workers=4)\n",
"display(client)\n",
"\n",
"# Or use Dask Gateway - this may take a few minutes\n",
"# cluster, client = initialize_dask(use_gateway=True, workers=4)\n",
"# display(client)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### ODC database\n",
"\n",
"Connect to the ODC database. Configure the environment and low-level tools to read from AWS buckets."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"dc = datacube.Datacube()\n",
"\n",
"# Access AWS \"requester-pays\" buckets\n",
"# This is necessary for reading data from most third-party AWS S3 buckets such as for Landsat and Sentinel-2\n",
"configure_s3_access(aws_unsigned=False, requester_pays=True, client=client);"
]
},
{
"cell_type": "markdown",
"metadata": {
"tags": []
},
"source": [
"## Example query\n",
"\n",
"Change any of the parameters in the `query` object below to adjust the location, time, projection, or spatial resolution of the returned datasets.\n",
"\n",
"Use the Explorer interface to check the temporal and spatial coverage for each product."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Explorer link\n",
"display(Markdown(f'See: {easi.explorer}/products/{product}'))\n",
"\n",
"# EASI defaults\n",
"display(Markdown(f'#### Location: {easi.location}'))\n",
"latitude_range = easi.latitude\n",
"longitude_range = easi.longitude\n",
"time_range = easi.time\n",
"\n",
"# Or set your own latitude / longitude\n",
"# latitude_range = (21.5, 23.5)\n",
"# longitude_range = (88, 90.8)\n",
"# time_range = ('2022-01-01', '2022-03-01')\n",
"\n",
"query = {\n",
" 'product': product, # Product name\n",
" 'x': longitude_range, # \"x\" axis bounds\n",
" 'y': latitude_range, # \"y\" axis bounds\n",
" 'time': time_range, # Any parsable date strings\n",
"}\n",
"\n",
"# Convenience function to display the selected area of interest\n",
"display_map(longitude_range, latitude_range)"
]
},
{
"cell_type": "markdown",
"metadata": {
"tags": []
},
"source": [
"#### Most common CRS\n",
"\n",
"Sentinel-2 datasets are stored with different coordinate reference systems (CRS), corresponding to the multiple UTM zones that are used for S2 L1B tiling. S2 measurement bands also have different resolutions (10 m, 20 m and 60 m). As such S2 queries need to include the following two query parameters:\n",
"\n",
"* `output_crs` - This sets a consistent CRS that all Sentinel-2 data will be reprojected to, irrespective of the UTM zone the individual image is stored in.\n",
"* `resolution` - This sets the resolution that all Sentinel-2 images will be resampled to. \n",
"\n",
"Use `mostcommon_crs()` to select a CRS. Adapted from https://github.com/GeoscienceAustralia/dea-notebooks/blob/develop/Tools/dea_tools/datahandling.py"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Most common CRS\n",
"native_crs = mostcommon_crs(dc, query)\n",
"print(f'Most common native CRS: {native_crs}')\n",
"\n",
"# Target xarray parameters\n",
"# - Select a set of measurements to load\n",
"# - output CRS and resolution\n",
"# - Usually we group input scenes on the same day to a single time layer (groupby)\n",
"# - Select a reasonable Dask chunk size (this should be adjusted depending on the\n",
"# spatial and resolution parameters you choose\n",
"load_params = {\n",
" 'measurements': ['blue', 'red', 'green', 'nir', 'scl'], # Selected measurement or alias names\n",
" 'output_crs': native_crs, # Target EPSG code\n",
" 'resolution': (-20, 20), # Target resolution\n",
" 'group_by': 'solar_day', # Scene grouping\n",
" 'dask_chunks': {'x': 2048, 'y': 2048}, # Dask chunks\n",
"}"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Load data\n",
"\n",
"data = dc.load(**(query | load_params))\n",
"display(xarray_object_size(data))\n",
"display(data)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# When happy with the shape and size of chunks, persist() the result\n",
"data = data.persist()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Optional\n",
"\n",
"# Create a simple plot to verify that the data look reasonable\n",
"# This will load and create images from the data, which may take a few minutes\n",
"# Here we limit this plot to the first few time layers.\n",
"\n",
"data.isel(time=slice(0,4)).red.plot.imshow(col=\"time\")"
]
},
{
"cell_type": "markdown",
"metadata": {
"tags": []
},
"source": [
"## Product definition\n",
"\n",
"The product definition contains details on the measurements and quality layers available in the product. Datacube provides convenience functions that return this information in `pandas DataFrames`.\n",
"\n",
"Use `list_measurements` to show the details for a product, and `masking.describe_variable_flags` to show the flag definitions."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Measurement definitions for the selected product\n",
"measurement_info = dc.list_measurements().loc[query['product']]\n",
"heading(f'Measurement table for product: {query[\"product\"]}')\n",
"display(measurement_info)\n",
"\n",
"# Flag definitions\n",
"flag_name = 'scl'\n",
"heading(f'Flag definition table for flag name: {flag_name}')\n",
"display(masking.describe_variable_flags(data[flag_name]))\n",
"\n",
"flags_def = masking.describe_variable_flags(data[flag_name]).loc['qa']['values']\n",
"display(flags_def)"
]
},
{
"cell_type": "markdown",
"metadata": {
"tags": []
},
"source": [
"#### Apply the correct _offset_ to the source data\n",
"\n",
"ESA introduced a change to their [L1C processing](#Collection-1-baseline-processing) that encodes their L1C and L2A products with _scale_ and _offset_ value such that\n",
"`phyiscal_value = encoded_value * scale_factor + offset`. The scale and offset details per band are available in the product definition."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Apply scale and offset\n",
"\n",
"for vv in data.data_vars:\n",
" scale = measurement_info.loc[vv, 'scale_factor']\n",
" offset = measurement_info.loc[vv, 'add_offset']\n",
" if not pd.isnull(scale) and not pd.isnull(offset):\n",
" data[vv] = data[vv] * scale + offset"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Quality layer\n",
"\n",
"To visualise the **SCL** layer we create a custom color map following the colors used by ESA.\n",
"\n",
"Here we use `hvplot` to create a dynamic (zoom, scroll) image with an attached histogram. This example shows how a custom color map can be used with `hvplot`, as well as the [datashader aggregator](https://datashader.org/getting_started/Pipeline.html) `reductions.mode()`."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Make SCL image\n",
"# https://sentinel.esa.int/web/sentinel/technical-guides/sentinel-2-msi/level-2a/algorithm\n",
"# https://www.sentinel-hub.com/faq/how-get-s2a-scene-classification-sentinel-2/\n",
"\n",
"from bokeh.models.tickers import FixedTicker\n",
"\n",
"color_def = [\n",
" (0, '#000000', 'No data'), # black\n",
" (1, '#ff0004', 'Saturated or defective'), # red\n",
" (2, '#868686', 'Topographic and casted shadow'), # gray\n",
" (3, '#774c0b', 'Cloud shadows'), # brown\n",
" (4, '#10d32d', 'Vegetation'), # green\n",
" (5, '#ffff53', 'Not vegetated'), # yellow\n",
" (6, '#0000ff', 'Water'), # blue\n",
" (7, '#818181', 'Unclassified'), # medium gray\n",
" (8, '#c0c0c0', 'Cloud medium probability'), # light gray\n",
" (9, '#f2f2f2', 'Cloud high probability'), # very light gray\n",
" (10, '#53fff9', 'Thin cirrus'), # light blue/purple\n",
" (11, '#ff52ff', 'Snow or ice'), # cyan\n",
"]\n",
"color_val = [x[0] for x in color_def]\n",
"color_hex = [x[1] for x in color_def]\n",
"color_txt = [f'{x[0]:2d}: {x[2]}' for x in color_def]\n",
"color_lim = (min(color_val), max(color_val) + 1)\n",
"bin_edges = color_val + [max(color_val) + 1]\n",
"bin_range = (color_val[0] + 0.5, color_val[-1] + 0.5) # No idea why (0.5,11.5) works and (0,11) or (0,12) do not\n",
"\n",
"# These options manipulate the color map and colorbar to show the categories for this product\n",
"options = {\n",
" 'title': f'Flag data for: {query[\"product\"]} ({flag_name})',\n",
" 'cmap': color_hex,\n",
" 'clim': color_lim,\n",
" 'color_levels': bin_edges,\n",
" 'colorbar': True,\n",
" 'width': 800,\n",
" 'height': 450,\n",
" 'aspect': 'equal',\n",
" 'tools': ['hover'],\n",
" 'colorbar_opts': {\n",
" 'major_label_overrides': dict(zip(color_val, color_txt)),\n",
" 'major_label_text_align': 'left',\n",
" 'ticker': FixedTicker(ticks=color_val),\n",
" },\n",
"}\n",
"\n",
"# Set the dataset CRS, if using hvplot's projection and coastlines options\n",
"# plot_crs = native_crs\n",
"# if plot_crs == 'epsg:4326':\n",
"# plot_crs = ccrs.PlateCarree()\n",
"\n",
"# Native data and coastline overlay:\n",
"# - Comment `crs`, `projection`, `coastline` to plot in native_crs coords\n",
"# TODO: Update the axis labels to 'longitude', 'latitude' if `coastline` is used\n",
"\n",
"quality_plot = data[flag_name].hvplot.image(\n",
" x = 'x', y = 'y', # Dataset x,y dimension names\n",
" rasterize = True, # Use Datashader\n",
" aggregator = reductions.mode(), # Datashader selects mode value, requires 'hv.Image'\n",
" precompute = True, # Datashader precomputes what it can\n",
" # crs = plot_crs, # Datset crs\n",
" # projection = ccrs.PlateCarree(), # Output projection (ccrs.PlateCarree() when coastline=True)\n",
" # coastline = '10m', # Coastline = '10m'/'50m'/'110m'\n",
").options(opts.Image(**options)).hist(bin_range = bin_range)\n",
"\n",
"# display(quality_plot)\n",
"# Optional: Change the default time slider to a dropdown list, https://stackoverflow.com/a/54912917\n",
"fig = pn.panel(quality_plot, widgets={'time': pn.widgets.Select}) # widget_location='top_left'\n",
"fig"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Create and apply a good quality pixel mask\n",
"\n",
"Select a set of flag values that represent \"good quality\" for your application. Here we select \"vegetation\", \"not vegetated\" and \"water\"; that is we exclude clouds and low-quality features.\n",
"\n",
"The **SCL** layer uses distinct integer values to represent each class. The datacube `enum_to_bool()` function creates a boolean mask layer corresponding to a set of category values (string names).\n",
"\n",
"Recall that *scale* and *offset* (if required) have already been applied by the `load_s2l2a_with_offset()` function."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Create Mask layer\n",
"\n",
"good_pixel_flags = [flags_def[str(i)] for i in [4, 5, 6]]\n",
"\n",
"good_pixel_mask = enum_to_bool(data[flag_name], good_pixel_flags)\n",
"display(good_pixel_mask) # -> DataArray. Type: bool\n",
"\n",
"# Apply good pixel mask (multiple layers)\n",
"good_data = data.where(good_pixel_mask).persist()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Calculate NDVI"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Create an NDVI layer, as useful way to visualise the data for differences in vegetation and land cover\n",
"# ndvi = (nir - red) / (nir + red)\n",
"\n",
"ndvi = (good_data.nir - good_data.red) / (good_data.nir + good_data.red)\n",
"ndvi.persist()"
]
},
{
"cell_type": "markdown",
"metadata": {
"tags": []
},
"source": [
"## Plot and browse the data\n",
"\n",
"There are numerous tools we can use to plot and interact with the data. Here we use `hvplot` again because it works well with dask and allows us to zoom and scroll quite efficiently. `Hvplot` uses [Datashader](https://datashader.org/getting_started/Pipeline.html) to process and render only the pixels that are required for the viewport.\n",
"\n",
"Various options can be changed such as the data layer, colour map and colour range."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Generate a plot\n",
"\n",
"options = {\n",
" 'title': f'{query[\"product\"]}',\n",
" 'width': 800,\n",
" 'height': 450,\n",
" 'aspect': 'equal',\n",
" 'cmap': cc.rainbow,\n",
" 'clim': (0, 1), # Limit the color range depending on the layer_name\n",
" 'colorbar': True,\n",
" 'tools': ['hover'],\n",
"}\n",
"\n",
"# Set the dataset CRS, if using hvplot's projection and coastlines options\n",
"# plot_crs = native_crs\n",
"# if plot_crs == 'epsg:4326':\n",
"# plot_crs = ccrs.PlateCarree()\n",
"\n",
"# Native data and coastline overlay:\n",
"# - Comment `crs`, `projection`, `coastline` to plot in native_crs coords\n",
"# TODO: Update the axis labels to 'longitude', 'latitude' if `coastline` is used\n",
"\n",
"layer_plot = ndvi.hvplot.image(\n",
" x = 'x', y = 'y', # Dataset x,y dimension names\n",
" rasterize = True, # Use Datashader\n",
" aggregator = reductions.mean(), # Datashader selects mean value\n",
" precompute = True, # Datashader precomputes what it can\n",
" # crs = plot_crs, # Dataset crs\n",
" # projection = ccrs.PlateCarree(), # Output projection (use ccrs.PlateCarree() when coastline=True)\n",
" # coastline='10m', # Coastline = '10m'/'50m'/'110m'\n",
").options(opts.Image(**options)).hist(bin_range = options['clim'])\n",
"\n",
"# display(layer_plot)\n",
"# Optional: Change the default time slider to a dropdown list, https://stackoverflow.com/a/54912917\n",
"fig = pn.panel(layer_plot, widgets={'time': pn.widgets.Select}) # widget_location='top_left'\n",
"display(fig)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.3"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
notebooks/data_products/sentinel-2-l2a.ipynb
+622
View File
@@ -0,0 +1,622 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {
"tags": []
},
"source": [
"# Sentinel-2 L2A <img align=\"right\" src=\"../../resources/csiro_easi_logo.png\">\n",
"\n",
"#### Index\n",
"- [Overview](#Overview)\n",
"- [Setup (imports, defaults, dask, odc)](#Setup)\n",
"- [Example query](#Example-query)\n",
"- [Product definition](#Product-definition)\n",
"- [Quality layer](#Quality-layer)\n",
"- [Create and apply a good quality pixel mask](#Create-and-apply-a-good-quality-pixel-mask)\n",
"- [Plot and browse the data](#Plot-and-browse-the-data)"
]
},
{
"cell_type": "markdown",
"metadata": {
"jp-MarkdownHeadingCollapsed": true,
"tags": []
},
"source": [
"## Overview\n",
"\n",
"Sentinel-2 is an Earth observation mission from the EU Copernicus Programme that systematically acquires optical imagery at high spatial resolution (up to 10 m for some bands). The mission is a constellation of two identical satellites in the same orbit, 180° apart for optimal coverage and data delivery. Together, they cover all Earth's land surfaces, large islands, inland and coastal waters every 3-5 days.\n",
"\n",
"Sentinel-2A was launched on 23 June 2015 and Sentinel-2B followed on 7 March 2017.\n",
"Both of the Sentinel-2 satellites carry a wide swath high-resolution multispectral imager with 13 spectral bands.\n",
"For more information see:\n",
"- [ESA Sentinel missions](https://www.esa.int/Applications/Observing_the_Earth/Copernicus/The_Sentinel_missions)\n",
"- [Sentinel-2 technical specifications](https://sentinels.copernicus.eu/web/sentinel/technical-guides/sentinel-2-msi)\n",
"\n",
"_Selected text adapted from https://github.com/GeoscienceAustralia/dea-notebooks/blob/develop/DEA_datasets/Sentinel_2.ipynb_\n",
"\n",
"#### Data source and documentation\n",
"\n",
"ESA produces a surface reflectance \"S2 L2A\" product using their [sen2cor](https://step.esa.int/main/snap-supported-plugins/sen2cor/) software. [Element84](https://github.com/Element84/earth-search) convert these data to Cloud-Optimized Geotiff format and makes them publicly available at their [Earth Search STAC API](https://earth-search.aws.element84.com/v1) endpoint, and [AWS open data](https://registry.opendata.aws/sentinel-2-l2a-cogs/), for programmatic access.\n",
"\n",
"EASI uses its STAC indexing tools to index datasets into our ODC databases.\n",
"\n",
"| Name | Product | Source | Information | Index\n",
"|--|--|--|--|--|\n",
"| Sentinel-2 COGs | `s2_l2a` | [Earth Search STAC](https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a) | Use for global (land) surface reflectance | Select COGS via STAC and convert to ODC metadata\n",
"\n",
"#### Baseline processing and offset consideration\n",
"\n",
"ESA introduced a change to their L1C processing (processing baseline >=04.00, for data since 25 January 2022) that encodes their L1C and L2A products with an _offset_ value such that\n",
"`phyiscal_value = encoded_value * scale_factor + offset`\n",
"\n",
"Element84 additionally [may have pre-applied the offset or not](https://github.com/Element84/earth-search/issues/23#issuecomment-1834674853) to the data that we index and load. This introduces an inconsistency in the S2A series that we need to account for.\n",
"\n",
"In this notebook we [show how to](#Apply-the-correct-offset-to-the-source-data) correctly load `s2_l2a` data with the offset applied to scenes where required. We use a `load_s2l2a_with_offset` function provided in `easi_tools` that identifies which scenes within a query should and should not have the offset applied, loads the data and applies the offset, and combines the non-offsetted with the offsetted scenes into a single xarray Dataset. This method also applies the scale_factor to all data in the query."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Setup"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Imports"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Common imports and settings\n",
"import os, sys, re\n",
"from pathlib import Path\n",
"from IPython.display import Markdown\n",
"import pandas as pd\n",
"pd.set_option(\"display.max_rows\", None)\n",
"import xarray as xr\n",
"\n",
"# Datacube\n",
"import datacube\n",
"from datacube.utils.aws import configure_s3_access\n",
"import odc.geo.xr # https://github.com/opendatacube/odc-geo\n",
"from datacube.utils import masking # https://github.com/opendatacube/datacube-core/blob/develop/datacube/utils/masking.py\n",
"from odc.algo import enum_to_bool # https://github.com/opendatacube/odc-tools/blob/develop/libs/algo/odc/algo/_masking.py\n",
"from dea_tools.plotting import display_map, rgb # https://github.com/GeoscienceAustralia/dea-notebooks/tree/develop/Tools\n",
"\n",
"# Basic plots\n",
"%matplotlib inline\n",
"# import matplotlib.pyplot as plt\n",
"# plt.rcParams['figure.figsize'] = [12, 8]\n",
"\n",
"# Holoviews\n",
"# https://holoviz.org/tutorial/Composing_Plots.html\n",
"# https://holoviews.org/user_guide/Composing_Elements.html\n",
"import hvplot.pandas\n",
"import hvplot.xarray\n",
"import panel as pn\n",
"import colorcet as cc\n",
"import cartopy.crs as ccrs\n",
"from datashader import reductions\n",
"from holoviews import opts\n",
"# hv.extension('bokeh', logo=False)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# EASI defaults\n",
"# These are convenience functions so that the notebooks in this repository work in all EASI deployments\n",
"\n",
"# The `git.Repo()` part returns the local directory that easi-notebooks has been cloned into\n",
"# If using the `easi-tools` functions from another path, replace `repo` with your local path to `easi-notebooks` directory\n",
"try:\n",
" import git\n",
" repo = git.Repo('.', search_parent_directories=True).working_tree_dir # Path to this cloned local directory\n",
"except (ImportError, git.InvalidGitRepositoryError):\n",
" repo = Path.home() / 'easi-notebooks' # Reasonable default\n",
" if not repo.is_dir():\n",
" raise RuntimeError('To use `easi-tools` please provide the local path to `https://github.com/csiro-easi/easi-notebooks`')\n",
"if repo not in sys.path:\n",
" sys.path.append(str(repo)) # Add the local path to `easi-notebooks` to python\n",
"\n",
"from easi_tools import EasiDefaults\n",
"from easi_tools import initialize_dask, xarray_object_size, mostcommon_crs, heading\n",
"from easi_tools.load_s2l2a import load_s2l2a_with_offset"
]
},
{
"cell_type": "markdown",
"metadata": {
"tags": []
},
"source": [
"#### EASI defaults\n",
"\n",
"These default values are configured for each EASI instance. They help us to use the same training notebooks in each EASI instance. You may find some of the functions convenient for your work or you can easily override the values in your copy of this notebook."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"easi = EasiDefaults()\n",
"\n",
"family = 'sentinel-2'\n",
"product = 's2_l2a' # Sentinel-2 collection 0, L2A\n",
"display(Markdown(f'Default {family} product for \"{easi.name}\": [{product}]({easi.explorer}/products/{product})'))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Dask cluster\n",
"\n",
"Using a local _Dask_ cluster is a good habit to get into. It can simplify loading and processing of data in many cases, and it provides a dashboard that shows the loading/processing progress.\n",
"\n",
"To learn more about _Dask_ see the set of [dask notebooks](https://github.com/csiro-easi/easi-notebooks/tree/main/html#dask-tutorials)."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Local cluster\n",
"cluster, client = initialize_dask(workers=4)\n",
"display(client)\n",
"\n",
"# Or use Dask Gateway - this may take a few minutes\n",
"# cluster, client = initialize_dask(use_gateway=True, workers=4)\n",
"# display(client)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### ODC database\n",
"\n",
"Connect to the ODC database. Configure the environment and low-level tools to read from AWS buckets."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"dc = datacube.Datacube()\n",
"\n",
"# Access AWS \"requester-pays\" buckets\n",
"# This is necessary for reading data from most third-party AWS S3 buckets such as for Landsat and Sentinel-2\n",
"configure_s3_access(aws_unsigned=False, requester_pays=True, client=client);"
]
},
{
"cell_type": "markdown",
"metadata": {
"tags": []
},
"source": [
"## Example query\n",
"\n",
"Change any of the parameters in the `query` object below to adjust the location, time, projection, or spatial resolution of the returned datasets.\n",
"\n",
"Use the Explorer interface to check the temporal and spatial coverage for each product."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Explorer link\n",
"display(Markdown(f'See: {easi.explorer}/products/{product}'))\n",
"\n",
"# EASI defaults\n",
"display(Markdown(f'#### Location: {easi.location}'))\n",
"latitude_range = easi.latitude\n",
"longitude_range = easi.longitude\n",
"time_range = easi.time\n",
"\n",
"# Or set your own latitude / longitude\n",
"# latitude_range = (-36.3, -35.8)\n",
"# longitude_range = (146.8, 147.3)\n",
"# time_range = ('2022-01-01', '2022-03-01')\n",
"\n",
"query = {\n",
" 'product': product, # Product name\n",
" 'x': longitude_range, # \"x\" axis bounds\n",
" 'y': latitude_range, # \"y\" axis bounds\n",
" 'time': time_range, # Any parsable date strings\n",
"}\n",
"\n",
"# Convenience function to display the selected area of interest\n",
"display_map(longitude_range, latitude_range)"
]
},
{
"cell_type": "markdown",
"metadata": {
"tags": []
},
"source": [
"#### Most common CRS\n",
"\n",
"Sentinel-2 datasets are stored with different coordinate reference systems (CRS), corresponding to the multiple UTM zones that are used for S2 L1B tiling. S2 measurement bands also have different resolutions (10 m, 20 m and 60 m). As such S2 queries need to include the following two query parameters:\n",
"\n",
"* `output_crs` - This sets a consistent CRS that all Sentinel-2 data will be reprojected to, irrespective of the UTM zone the individual image is stored in.\n",
"* `resolution` - This sets the resolution that all Sentinel-2 images will be resampled to. \n",
"\n",
"Use `mostcommon_crs()` to select a CRS. Adapted from https://github.com/GeoscienceAustralia/dea-notebooks/blob/develop/Tools/dea_tools/datahandling.py"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Most common CRS\n",
"native_crs = mostcommon_crs(dc, query)\n",
"print(f'Most common native CRS: {native_crs}')\n",
"\n",
"# Target xarray parameters\n",
"# - Select a set of measurements to load\n",
"# - output CRS and resolution\n",
"# - Usually we group input scenes on the same day to a single time layer (groupby)\n",
"# - Select a reasonable Dask chunk size (this should be adjusted depending on the\n",
"# spatial and resolution parameters you choose\n",
"load_params = {\n",
" 'measurements': ['blue', 'red', 'green', 'nir', 'scl'], # Selected measurement or alias names\n",
" 'output_crs': native_crs, # Target EPSG code\n",
" 'resolution': (-20, 20), # Target resolution\n",
" 'group_by': 'solar_day', # Scene grouping\n",
" 'dask_chunks': {'x': 2048, 'y': 2048}, # Dask chunks\n",
"}"
]
},
{
"cell_type": "markdown",
"metadata": {
"tags": []
},
"source": [
"#### Apply the correct _offset_ to the source data\n",
"\n",
"We provide a convenience function to load `s2_l2a` data, apply the scale (and offset if required) and return an `xarray.Dataset`. This replaces `datacube.load()` for this product."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# The usual dc.load() may give an incorrect result for this product due to inconsistent upstream handling of any offset value\n",
"# data = dc.load(query | load_params)\n",
"\n",
"# Use the replacement \"dc.load()\" function for this product\n",
"data = load_s2l2a_with_offset(dc, query | load_params) # Combine the two dicts that contain our search and load parameters\n",
"display(xarray_object_size(data))\n",
"display(data)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# When happy with the shape and size of chunks, persist() the result\n",
"data = data.persist()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Optional\n",
"\n",
"# Create a simple plot to verify that the data look reasonable\n",
"# This will load and create images from the data, which may take a few minutes\n",
"# Here we limit this plot to the first few time layers.\n",
"\n",
"data.isel(time=slice(0, 4)).red.plot.imshow(col=\"time\")"
]
},
{
"cell_type": "markdown",
"metadata": {
"tags": []
},
"source": [
"## Product definition\n",
"\n",
"The product definition contains details on the measurements and quality layers available in the product. Datacube provides convenience functions that return this information in `pandas DataFrames`.\n",
"\n",
"Use `list_measurements` to show the details for a product, and `masking.describe_variable_flags` to show the flag definitions."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Measurement definitions for the selected product\n",
"measurement_info = dc.list_measurements().loc[query['product']]\n",
"heading(f'Measurement table for product: {query[\"product\"]}')\n",
"display(measurement_info)\n",
"\n",
"# Flag definitions\n",
"flag_name = 'scl'\n",
"heading(f'Flag definition table for flag name: {flag_name}')\n",
"display(masking.describe_variable_flags(data[flag_name]))\n",
"\n",
"flags_def = masking.describe_variable_flags(data[flag_name]).loc['qa']['values']\n",
"display(flags_def)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Quality layer\n",
"\n",
"To visualise the **SCL** layer we create a custom color map following the colors used by ESA.\n",
"\n",
"Here we use `hvplot` to create a dynamic (zoom, scroll) image with an attached histogram. This example shows how a custom color map can be used with `hvplot`, as well as the [datashader aggregator](https://datashader.org/getting_started/Pipeline.html) `reductions.mode()`."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Make SCL image\n",
"# https://sentiwiki.copernicus.eu/web/s2-processing\n",
"# https://www.sentinel-hub.com/faq/how-get-s2a-scene-classification-sentinel-2/\n",
"\n",
"from bokeh.models.tickers import FixedTicker\n",
"\n",
"color_def = [\n",
" (0, '#000000', 'No data'), # black\n",
" (1, '#ff0004', 'Saturated or defective'), # red\n",
" (2, '#868686', 'Dark features or shadows'), # gray\n",
" (3, '#774c0b', 'Cloud shadows'), # brown\n",
" (4, '#10d32d', 'Vegetation'), # green\n",
" (5, '#ffff53', 'Not vegetated'), # yellow\n",
" (6, '#0000ff', 'Water'), # blue\n",
" (7, '#818181', 'Unclassified'), # medium gray\n",
" (8, '#c0c0c0', 'Cloud medium probability'), # light gray\n",
" (9, '#f2f2f2', 'cloud high probability'), # very light gray\n",
" (10, '#53fff9', 'Thin cirrus'), # cyan\n",
" (11, '#ff52ff', 'Snow or ice'), # pink\n",
"]\n",
"color_val = [x[0] for x in color_def]\n",
"color_hex = [x[1] for x in color_def]\n",
"color_txt = [f'{x[0]:2d}: {x[2]}' for x in color_def]\n",
"color_lim = (min(color_val), max(color_val) + 1)\n",
"bin_edges = color_val + [max(color_val) + 1]\n",
"bin_range = (color_val[0] + 0.5, color_val[-1] + 0.5) # No idea why (0.5,11.5) works and (0,11) or (0,12) do not\n",
"\n",
"# These options manipulate the color map and colorbar to show the categories for this product\n",
"options = {\n",
" 'title': f'Flag data for: {query[\"product\"]} ({flag_name})',\n",
" 'cmap': color_hex,\n",
" 'clim': color_lim,\n",
" 'color_levels': bin_edges,\n",
" 'colorbar': True,\n",
" 'width': 800,\n",
" 'height': 450,\n",
" 'aspect': 'equal',\n",
" 'tools': ['hover'],\n",
" 'colorbar_opts': {\n",
" 'major_label_overrides': dict(zip(color_val, color_txt)),\n",
" 'major_label_text_align': 'left',\n",
" 'ticker': FixedTicker(ticks=color_val),\n",
" },\n",
"}\n",
"\n",
"# Set the dataset CRS, if using hvplot's projection and coastlines options\n",
"# plot_crs = native_crs\n",
"# if plot_crs == 'epsg:4326':\n",
"# plot_crs = ccrs.PlateCarree()\n",
"\n",
"# Native data and coastline overlay:\n",
"# - Comment `crs`, `projection`, `coastline` to plot in native_crs coords\n",
"# TODO: Update the axis labels to 'longitude', 'latitude' if `coastline` is used\n",
"\n",
"quality_plot = data[flag_name].hvplot.image(\n",
" x = 'x', y = 'y', # Dataset x,y dimension names\n",
" rasterize = True, # Use Datashader\n",
" aggregator = reductions.mode(), # Datashader selects mode value, requires 'hv.Image'\n",
" precompute = True, # Datashader precomputes what it can\n",
" # crs = plot_crs, # Datset crs\n",
" # projection = ccrs.PlateCarree(), # Output projection (ccrs.PlateCarree() when coastline=True)\n",
" # coastline = '10m', # Coastline = '10m'/'50m'/'110m'\n",
").options(opts.Image(**options)).hist(bin_range = bin_range)\n",
"\n",
"# display(quality_plot)\n",
"# Optional: Change the default time slider to a dropdown list, https://stackoverflow.com/a/54912917\n",
"fig = pn.panel(quality_plot, widgets={'time': pn.widgets.Select}) # widget_location='top_left'\n",
"fig"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Create and apply a good quality pixel mask\n",
"\n",
"Select a set of flag values that represent \"good quality\" for your application. Here we select \"vegetation\", \"not vegetated\" and \"water\"; that is we exclude clouds and low-quality features.\n",
"\n",
"The **SCL** layer uses distinct integer values to represent each class. The datacube `enum_to_bool()` function creates a boolean mask layer corresponding to a set of category values (string names).\n",
"\n",
"Recall that *scale* and *offset* (if required) have already been applied by the `load_s2l2a_with_offset()` function."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Create Mask layer\n",
"\n",
"good_pixel_flags = [flags_def[str(i)] for i in [4, 5, 6]]\n",
"\n",
"good_pixel_mask = enum_to_bool(data[flag_name], good_pixel_flags)\n",
"display(good_pixel_mask) # -> DataArray. Type: bool\n",
"\n",
"# Apply good pixel mask (multiple layers)\n",
"good_data = data.where(good_pixel_mask).persist()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Calculate NDVI"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Create an NDVI layer, as useful way to visualise the data for differences in vegetation and land cover\n",
"# ndvi = (nir - red) / (nir + red)\n",
"\n",
"ndvi = (good_data.nir - good_data.red) / (good_data.nir + good_data.red)\n",
"ndvi.persist()"
]
},
{
"cell_type": "markdown",
"metadata": {
"tags": []
},
"source": [
"## Plot and browse the data\n",
"\n",
"There are numerous tools we can use to plot and interact with the data. Here we use `hvplot` again because it works well with dask and allows us to zoom and scroll quite efficiently. `Hvplot` uses [Datashader](https://datashader.org/getting_started/Pipeline.html) to process and render only the pixels that are required for the viewport.\n",
"\n",
"Various options can be changed such as the data layer, colour map and colour range."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Generate a plot\n",
"\n",
"options = {\n",
" 'title': f'{query[\"product\"]}',\n",
" 'width': 800,\n",
" 'height': 450,\n",
" 'aspect': 'equal',\n",
" 'cmap': cc.rainbow,\n",
" 'clim': (0, 1), # Limit the color range depending on the layer_name\n",
" 'colorbar': True,\n",
" 'tools': ['hover'],\n",
"}\n",
"\n",
"# Set the dataset CRS, if using hvplot's projection and coastlines options\n",
"# plot_crs = native_crs\n",
"# if plot_crs == 'epsg:4326':\n",
"# plot_crs = ccrs.PlateCarree()\n",
"\n",
"# Native data and coastline overlay:\n",
"# - Comment `crs`, `projection`, `coastline` to plot in native_crs coords\n",
"# TODO: Update the axis labels to 'longitude', 'latitude' if `coastline` is used\n",
"\n",
"layer_plot = ndvi.hvplot.image(\n",
" x = 'x', y = 'y', # Dataset x,y dimension names\n",
" rasterize = True, # Use Datashader\n",
" aggregator = reductions.mean(), # Datashader selects mean value\n",
" precompute = True, # Datashader precomputes what it can\n",
" # crs = plot_crs, # Dataset crs\n",
" # projection = ccrs.PlateCarree(), # Output projection (use ccrs.PlateCarree() when coastline=True)\n",
" # coastline='10m', # Coastline = '10m'/'50m'/'110m'\n",
").options(opts.Image(**options)).hist(bin_range = options['clim'])\n",
"\n",
"# display(layer_plot)\n",
"# Optional: Change the default time slider to a dropdown list, https://stackoverflow.com/a/54912917\n",
"fig = pn.panel(layer_plot, widgets={'time': pn.widgets.Select}) # widget_location='top_left'\n",
"display(fig)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.3"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
notebooks/easi-scratch-bucket.ipynb
+373
View File
@@ -0,0 +1,373 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Using EASI scratch and project buckets <img align=\"right\" src=\"../resources/csiro_easi_logo.png\">\n",
"\n",
"EASI has a **Scratch** bucket available for all users.\n",
"- **Scratch** means temporary: all files will be deleted after 30 days.\n",
"- Use the scratch bucket to save files between processing runs or share files between projects, temporarily.\n",
"\n",
"**Project** buckets are available to selected users as well. A project bucket can exist in another AWS account and be cross-linked to EASI. An EASI admin will assign users to a \"project\", which will enable their access to the bucket. Files in a project bucket are subject to the bucket owner's life cycle rules, administration and costs.\n",
"\n",
"> Cross-account **project** buckets may benefit from additional ACL settings. See [User Guide/08-cross-account-storage-usage](https://docs.csiro.easi-eo.solutions/user-guide/users-guide/08-cross-account-storage-usage/) (in your deployment).\n",
"\n",
"Glossary:\n",
"- S3 storage items are called **objects**. Typically these are files but they could be any blob of data.\n",
"- An **object**'s name is its **key**. The **key** can be [just about any string](https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html). Typically we include a `/` in the key to make it look like a directory path, which we're familiar with from regular file systems.\n",
"\n",
"There are two AWS APIs that can be used to read/write to a **scratch** or **project** bucket. Examples for both are given in this notebook.\n",
"- [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-services-s3-commands.html) - linux program (use in terminal)\n",
"- [boto3](https://boto3.amazonaws.com/v1/documentation/api/latest/index.html) - python library (use in code)\n",
"\n",
"We show *writing* first so that you add a test file for the *reading* section.\n",
"\n",
"- [Writing](#Writing)\n",
" - [User ID](#User-ID)\n",
" - [Select a test file](#Select-a-test-file)\n",
" - [Upload a file](#Upload-a-file)\n",
"- [Reading](#Reading)\n",
" - [List objects](#List-objects)\n",
" - [Read a file directly](#Read-a-file-directly)\n",
" - [Copy a file to local](#Copy-a-file-to-local)\n",
"\n",
"## Imports and setup"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"import sys, os\n",
"import boto3\n",
"from datetime import datetime as dt\n",
"\n",
"# EASI tools\n",
"import git\n",
"repo = git.Repo('.', search_parent_directories=True).working_tree_dir\n",
"if repo not in sys.path: sys.path.append(repo)\n",
"from easi_tools import EasiDefaults"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"client = boto3.client('s3')\n",
"\n",
"easi = EasiDefaults()\n",
"bucket = easi.scratch"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Optional, for parallel uploads and downloads of large files\n",
"# Add a (..., Config=config) parameter to the relevant upload and download functions\n",
"\n",
"# from boto3.s3.transfer import TransferConfig\n",
"# config = TransferConfig(\n",
"# multipart_threshold = 1024 * 25,\n",
"# max_concurrency = 10,\n",
"# multipart_chunksize = 1024 * 25,\n",
"# use_threads = True\n",
"# )"
]
},
{
"cell_type": "markdown",
"metadata": {
"tags": []
},
"source": [
"## Writing\n",
"\n",
"### User ID\n",
"\n",
"To write to the **scratch** bucket the root of the key must be your AWS **User ID**.\n",
"\n",
"For a **project** bucket this restriction probably doesn't apply. Any root key conditions are managed by the bucket owner."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"%%bash\n",
"\n",
"userid=`aws sts get-caller-identity --query 'UserId' | sed 's/[\"]//g'`\n",
"echo $userid"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"userid = boto3.client('sts').get_caller_identity()['UserId']\n",
"print(userid)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Select a test file\n",
"\n",
"For use in this notebook."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"testfile = '/home/jovyan/test-file.txt'"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"%%bash -s \"$testfile\"\n",
" \n",
"testfile=$1\n",
"touch $testfile\n",
"ls -l $testfile"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Upload a file"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"%%bash -s \"$bucket\" \"$userid\" \"$testfile\"\n",
"\n",
"bucket=$1\n",
"userid=$2\n",
"testfile=$3\n",
"\n",
"aws s3 cp ${testfile} s3://${bucket}/${userid}/"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"target = testfile.split('/')[-1]\n",
"try:\n",
" print(f'upload: {testfile} to s3://{bucket}/{userid}/{target}')\n",
" r = client.upload_file(testfile, bucket, f'{userid}/{target}')\n",
" print('Success.')\n",
"except Exception as e:\n",
" print(e)\n",
" print('Failed.')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Reading\n",
"\n",
"### List objects\n",
"\n",
"The `boto3.list_objects_v2` function will return at most 1000 keys. Two options are shown here.\n",
"1. Basic use of `list_objects_v2`\n",
"2. Paginated list objects, for potentially >1000 keys"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"%%bash -s \"$bucket\" \"$userid\"\n",
"\n",
"bucket=$1\n",
"userid=$2\n",
"\n",
"aws s3 ls s3://${bucket}/${userid}/"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Basic use of list_objects_v2\n",
"\n",
"response = client.list_objects_v2(Bucket=bucket, Prefix=f'{userid}/')\n",
"\n",
"# from pprint import pprint\n",
"# pprint(response)\n",
"\n",
"# List each key with its last modified time stamp\n",
"if 'Contents' in response:\n",
" for c in response['Contents']:\n",
" key = c['Key']\n",
" lastmodified = c['LastModified'].strftime('%Y-%d-%m %H:%M:%S')\n",
" size = c['Size']\n",
" print(f'{lastmodified}\\t{size} {key}')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Paginated list objects, for potentially >1000 keys\n",
"\n",
"paginator = client.get_paginator('list_objects_v2')\n",
"page_iterator = paginator.paginate(Bucket=bucket, Prefix=f'{userid}/')\n",
"\n",
"for response in page_iterator:\n",
" if 'Contents' in response:\n",
" for c in response['Contents']:\n",
" key = c['Key']\n",
" lastmodified = c['LastModified'].strftime('%Y-%d-%m %H:%M:%S')\n",
" psize = c['Size']\n",
" print(f'{lastmodified}\\t{size} {key}')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Read a file directly\n",
"\n",
"Many data reading packages can read a file from an *s3://bucket/key* path into memory. Examples include:\n",
"- `rasterio` and `rioxarray`\n",
"- `gdal`\n",
"\n",
"For packages that can not read from an S3 path, first copy the file to your home directory or a temporary directory (e.g., dask workers). Then read the file with a normal file path."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Copy a file to local"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"%%bash -s \"$bucket\" \"$userid\" \"$testfile\"\n",
"\n",
"bucket=$1\n",
"userid=$2\n",
"testfile=$3\n",
"\n",
"source=`basename $testfile`\n",
"aws s3 cp s3://${bucket}/${userid}/${source} ${testfile}\n",
"ls -l $testfile"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"source = testfile.split('/')[-1]\n",
"try:\n",
" print(f'download: s3://{bucket}/{userid}/{source} to {testfile}')\n",
" r = client.download_file(bucket, f'{userid}/{source}', testfile)\n",
" print('Success.')\n",
"except Exception as e:\n",
" print(e)\n",
" print('Failed.')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.6"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
test_cognito_s3.py
+128
View File
@@ -0,0 +1,128 @@
#!/usr/bin/env python3
"""
Test S3 access using Cognito tokens
Test truy cập S3 sử dụng Cognito tokens
"""
import sys
from cognito_auth import CognitoAuthenticator
from datacube.utils.rio import configure_s3_access
def test_cognito_to_s3():
"""
Complete test: Cognito tokens → AWS credentials → S3 access
"""
print("=" * 70)
print("Test S3 Access Using Cognito Tokens")
print("Test Truy Cập S3 Sử Dụng Cognito Tokens")
print("=" * 70)
# Initialize
auth = CognitoAuthenticator(region='ap-southeast-1')
# Load Cognito tokens
print("\n[1/5] Loading Cognito tokens from file...")
if not auth.load_tokens_from_file('train_files/crediential.txt'):
print("✗ Failed to load tokens")
return False
# Show token info
print("\n[2/5] Displaying token information...")
decoded_id, decoded_access = auth.print_token_info()
if not decoded_id:
print("✗ Invalid tokens")
return False
# Get AWS credentials from Cognito
print("\n[3/5] Getting AWS credentials...")
print("ℹ Note: Since we don't have Identity Pool ID, using existing credentials")
# Use existing credentials from file (already exchanged by EASI)
if not auth.get_credentials_from_cognito():
print("✗ Failed to get AWS credentials")
return False
# Set credentials in environment
print("\n[4/5] Configuring environment...")
if not auth.set_environment_credentials():
print("✗ Failed to set environment")
return False
# Configure datacube S3 access
print("\n[5/5] Configuring datacube S3 access...")
try:
configure_s3_access(
aws_unsigned=False,
region_name='us-west-2',
cloud_defaults=True
)
print("✓ Datacube S3 access configured")
except Exception as e:
print(f"⚠ Warning: Could not configure datacube: {e}")
# Test multiple buckets
print("\n" + "=" * 70)
print("Testing S3 Bucket Access")
print("=" * 70)
test_buckets = [
('sentinel-cogs', 'us-west-2', 'sentinel-s2-l2a-cogs/'),
('sentinel-s2-l2a', 'eu-central-1', ''),
]
results = []
for bucket_name, region, prefix in test_buckets:
print(f"\nTesting: {bucket_name} ({region})")
print("-" * 50)
# List up to 50 objects (có thể thay đổi số này)
success = auth.test_s3_access(bucket_name, region, max_keys=50, prefix=prefix)
results.append((bucket_name, success))
# Summary
print("\n" + "=" * 70)
print("Summary / Tổng Kết")
print("=" * 70)
print("\nAuthentication Flow:")
print(" Cognito Tokens → ✓")
print(" AWS Credentials → ✓")
print(" Environment Setup → ✓")
print(f"\nS3 Access Results:")
success_count = sum(1 for _, success in results if success)
for bucket, success in results:
status = "✓" if success else "✗"
print(f" {status} {bucket}")
print(f"\nTotal: {success_count}/{len(results)} buckets accessible")
if success_count > 0:
print("\n✓ SUCCESS: Cognito authentication working!")
print("✓ THÀNH CÔNG: Xác thực Cognito hoạt động!")
else:
print("\n⚠ WARNING: Could not access any S3 buckets")
print("⚠ CẢNH BÁO: Không thể truy cập bucket S3 nào")
print("=" * 70)
return success_count > 0
def main():
try:
success = test_cognito_to_s3()
sys.exit(0 if success else 1)
except KeyboardInterrupt:
print("\n\n✗ Test interrupted")
sys.exit(1)
except Exception as e:
print(f"\n✗ Fatal error: {e}")
import traceback
traceback.print_exc()
sys.exit(1)
if __name__ == "__main__":
main()
test_s3_datacube_access.py
+292
View File
@@ -0,0 +1,292 @@
#!/usr/bin/env python3
"""
Test file for datacube S3 access with limited permissions
File test truy cập S3 qua datacube với quyền hạn chế
"""
import os
import sys
import boto3
from botocore.exceptions import ClientError, NoCredentialsError
def load_credentials(credential_file='train_files/crediential.txt'):
"""Load AWS credentials from file"""
credentials = {}
try:
with open(credential_file, 'r') as f:
for line in f:
line = line.strip()
if line.startswith('export '):
line = line[7:]
if '=' in line:
key, value = line.split('=', 1)
value = value.strip('"')
credentials[key] = value
print("✓ Credentials loaded successfully")
print(f" - AWS_ACCESS_KEY_ID: {credentials.get('AWS_ACCESS_KEY_ID', 'N/A')[:20]}...")
return credentials
except Exception as e:
print(f"✗ Error loading credentials: {e}")
return None
def set_aws_credentials(credentials):
"""Set AWS credentials as environment variables"""
if not credentials:
return False
try:
os.environ['AWS_ACCESS_KEY_ID'] = credentials.get('AWS_ACCESS_KEY_ID', '')
os.environ['AWS_SECRET_ACCESS_KEY'] = credentials.get('AWS_SECRET_ACCESS_KEY', '')
os.environ['AWS_SESSION_TOKEN'] = credentials.get('AWS_SESSION_TOKEN', '')
print("✓ AWS credentials set in environment")
return True
except Exception as e:
print(f"✗ Error setting credentials: {e}")
return False
def test_s3_specific_bucket(bucket_name='deafrica-sentinel-2', region='ap-southeast-1', prefix=''):
"""
Test access to a specific S3 bucket without requiring ListAllMyBuckets permission
Kiểm tra truy cập bucket S3 cụ thể mà không cần quyền ListAllMyBuckets
"""
try:
print(f"\n=== Testing Specific Bucket Access ===")
print(f"Bucket: {bucket_name}")
print(f"Region: {region}")
# Create S3 client
s3_client = boto3.client('s3', region_name=region)
# Try to access bucket with head_bucket (checks if bucket exists and we have access)
try:
s3_client.head_bucket(Bucket=bucket_name)
print(f"✓ Successfully verified access to bucket: {bucket_name}")
except ClientError as e:
error_code = e.response.get('Error', {}).get('Code', 'Unknown')
if error_code == '404':
print(f"✗ Bucket {bucket_name} does not exist or you don't have access")
return None
elif error_code == '403' or error_code == 'AccessDenied':
print(f"✗ Access denied to bucket: {bucket_name}")
return None
else:
print(f"✗ Error: {error_code}")
return None
# Try to list some objects
print(f"\n✓ Attempting to list objects (max 5)...")
response = s3_client.list_objects_v2(
Bucket=bucket_name,
MaxKeys=5,
Prefix=prefix
)
if 'Contents' in response:
print(f"✓ Found {len(response['Contents'])} objects:")
for i, obj in enumerate(response['Contents'], 1):
size_mb = obj['Size'] / (1024 * 1024)
print(f" {i}. {obj['Key']}")
print(f" Size: {size_mb:.2f} MB")
else:
print(f"✓ No objects found with prefix '{prefix}'")
return s3_client
except NoCredentialsError:
print("\n✗ Error: No AWS credentials found")
return None
except ClientError as e:
error_code = e.response.get('Error', {}).get('Code', 'Unknown')
print(f"\n✗ AWS Client Error: {e}")
print(f"✗ Error Code: {error_code}")
if error_code == 'ExpiredToken':
print("✗ AWS session token has expired. Please refresh your credentials.")
elif error_code == 'AccessDenied':
print("✗ Access denied. You may not have permission for this bucket.")
return None
except Exception as e:
print(f"\n✗ Unexpected error: {e}")
return None
def test_datacube_s3_access():
"""
Test S3 access using datacube pattern
Kiểm tra truy cập S3 theo pattern của datacube
"""
try:
print("\n=== Testing Datacube S3 Access Pattern ===")
# Import datacube S3 utilities
try:
from datacube.utils.rio import configure_s3_access
print("✓ datacube library found")
# Configure S3 access for rasterio/datacube
aws_unsigned = False # We have credentials
region_name = 'ap-southeast-1'
print(f"✓ Configuring S3 access for region: {region_name}")
configure_s3_access(
aws_unsigned=aws_unsigned,
region_name=region_name,
cloud_defaults=True
)
print("✓ S3 access configured for datacube/rasterio")
return True
except ImportError:
print("✗ datacube library not found")
print(" You can install it with: pip install datacube")
return False
except Exception as e:
print(f"\n✗ Error configuring datacube S3 access: {e}")
return False
def test_simple_s3_operations(bucket_name, s3_client):
"""
Test basic S3 operations that work with limited permissions
Kiểm tra các thao tác S3 cơ bản với quyền hạn chế
"""
try:
print("\n=== Testing Basic S3 Operations ===")
# Test 1: Get bucket location
try:
response = s3_client.get_bucket_location(Bucket=bucket_name)
location = response.get('LocationConstraint', 'us-east-1')
print(f"✓ Bucket location: {location}")
except ClientError as e:
print(f"✗ Cannot get bucket location: {e.response.get('Error', {}).get('Code', 'Unknown')}")
# Test 2: Check if we can read objects
try:
# Try to list with a common prefix
response = s3_client.list_objects_v2(
Bucket=bucket_name,
MaxKeys=1,
Delimiter='/'
)
if 'CommonPrefixes' in response:
print(f"✓ Found {len(response['CommonPrefixes'])} top-level folders")
for prefix in response['CommonPrefixes'][:3]:
print(f" - {prefix['Prefix']}")
except ClientError as e:
print(f"✗ Cannot list objects: {e.response.get('Error', {}).get('Code', 'Unknown')}")
return True
except Exception as e:
print(f"\n✗ Error in S3 operations: {e}")
return False
def test_multiple_buckets():
"""
Test access to multiple common S3 buckets
Kiểm tra truy cập nhiều bucket S3 phổ biến
"""
# Common buckets used in remote sensing / datacube projects
test_buckets = [
('deafrica-sentinel-2', 'af-south-1', 'sentinel-s2-l2a-cogs'),
('sentinel-cogs', 'us-west-2', 'sentinel-s2-l2a'),
('usgs-landsat', 'us-west-2', 'collection02'),
# EASI bucket might be private
('easi-asia-csiro', 'ap-southeast-1', ''),
]
results = []
for bucket_name, region, prefix in test_buckets:
print(f"\n{'='*60}")
print(f"Testing bucket: {bucket_name}")
print(f"Region: {region}")
s3_client = test_s3_specific_bucket(bucket_name, region, prefix)
if s3_client:
results.append((bucket_name, True))
# If successful, try some operations
test_simple_s3_operations(bucket_name, s3_client)
else:
results.append((bucket_name, False))
return results
def main():
"""Main test function"""
print("=" * 60)
print("AWS S3 Datacube Access Test")
print("Test Truy Cập S3 Datacube (với quyền hạn chế)")
print("=" * 60)
# Step 1: Load credentials
print("\n[Step 1] Loading credentials from file...")
credentials = load_credentials()
if not credentials:
print("\n✗ Test failed: Could not load credentials")
return False
# Step 2: Set credentials in environment
print("\n[Step 2] Setting AWS credentials...")
if not set_aws_credentials(credentials):
print("\n✗ Test failed: Could not set credentials")
return False
# Step 3: Test datacube S3 access configuration
print("\n[Step 3] Testing datacube S3 access configuration...")
test_datacube_s3_access()
# Step 4: Test access to multiple buckets
print("\n[Step 4] Testing access to common S3 buckets...")
results = test_multiple_buckets()
# Summary
print("\n" + "=" * 60)
print("Test Summary / Tổng kết:")
print("=" * 60)
success_count = sum(1 for _, success in results if success)
total_count = len(results)
print(f"\nSuccessful connections: {success_count}/{total_count}")
for bucket_name, success in results:
status = "✓" if success else "✗"
print(f"{status} {bucket_name}")
print("\n" + "=" * 60)
if success_count > 0:
print("✓ Test completed! You have access to some buckets.")
print("✓ Test hoàn thành! Bạn có quyền truy cập một số bucket.")
else:
print("⚠ No buckets accessible with current credentials.")
print("⚠ Không có bucket nào truy cập được với credentials hiện tại.")
print("=" * 60)
return success_count > 0
if __name__ == "__main__":
try:
success = main()
sys.exit(0 if success else 1)
except KeyboardInterrupt:
print("\n\n✗ Test interrupted by user")
sys.exit(1)
except Exception as e:
print(f"\n✗ Fatal error: {e}")
import traceback
traceback.print_exc()
sys.exit(1)
test_s3_direct_access.py
+236
View File
@@ -0,0 +1,236 @@
#!/usr/bin/env python3
"""
Test file for direct AWS S3 access
File test truy cập trực tiếp AWS S3
"""
import os
import sys
import boto3
from botocore.exceptions import ClientError, NoCredentialsError
def load_credentials(credential_file='train_files/crediential.txt'):
"""
Load AWS credentials from file
Đọc thông tin xác thực AWS từ file
"""
credentials = {}
try:
with open(credential_file, 'r') as f:
for line in f:
line = line.strip()
if line.startswith('export '):
# Remove 'export ' prefix
line = line[7:]
if '=' in line:
key, value = line.split('=', 1)
# Remove quotes if present
value = value.strip('"')
credentials[key] = value
print("✓ Credentials loaded successfully")
print(f" - AWS_ACCESS_KEY_ID: {credentials.get('AWS_ACCESS_KEY_ID', 'N/A')[:20]}...")
return credentials
except FileNotFoundError:
print(f"✗ Error: Credential file not found: {credential_file}")
return None
except Exception as e:
print(f"✗ Error loading credentials: {e}")
return None
def set_aws_credentials(credentials):
"""
Set AWS credentials as environment variables
Thiết lập thông tin xác thực AWS vào biến môi trường
"""
if not credentials:
return False
try:
os.environ['AWS_ACCESS_KEY_ID'] = credentials.get('AWS_ACCESS_KEY_ID', '')
os.environ['AWS_SECRET_ACCESS_KEY'] = credentials.get('AWS_SECRET_ACCESS_KEY', '')
os.environ['AWS_SESSION_TOKEN'] = credentials.get('AWS_SESSION_TOKEN', '')
print("✓ AWS credentials set in environment")
return True
except Exception as e:
print(f"✗ Error setting credentials: {e}")
return False
def test_s3_connection(region='ap-southeast-1'):
"""
Test S3 connection and list buckets
Kiểm tra kết nối S3 và liệt kê các bucket
"""
try:
# Create S3 client
s3_client = boto3.client('s3', region_name=region)
print("\n=== Testing S3 Connection ===")
print("Đang kiểm tra kết nối S3...")
# List buckets
response = s3_client.list_buckets()
print(f"\n✓ Successfully connected to S3!")
print(f"✓ Total buckets found: {len(response['Buckets'])}")
print("\nAvailable S3 Buckets:")
print("Các S3 Bucket có sẵn:")
for i, bucket in enumerate(response['Buckets'], 1):
print(f" {i}. {bucket['Name']} (Created: {bucket['CreationDate']})")
return s3_client, response['Buckets']
except NoCredentialsError:
print("\n✗ Error: No AWS credentials found")
print("✗ Lỗi: Không tìm thấy thông tin xác thực AWS")
return None, None
except ClientError as e:
print(f"\n✗ AWS Client Error: {e}")
error_code = e.response.get('Error', {}).get('Code', 'Unknown')
print(f"✗ Error Code: {error_code}")
if error_code == 'ExpiredToken':
print("✗ AWS session token has expired. Please refresh your credentials.")
print("✗ Token AWS đã hết hạn. Vui lòng làm mới thông tin xác thực.")
return None, None
except Exception as e:
print(f"\n✗ Unexpected error: {e}")
return None, None
def test_bucket_access(s3_client, bucket_name, max_objects=10):
"""
Test access to a specific bucket and list objects
Kiểm tra truy cập bucket cụ thể và liệt kê các object
"""
try:
print(f"\n=== Testing Bucket Access: {bucket_name} ===")
print(f"Đang kiểm tra truy cập bucket: {bucket_name}")
# List objects in bucket
response = s3_client.list_objects_v2(
Bucket=bucket_name,
MaxKeys=max_objects
)
if 'Contents' in response:
print(f"\n✓ Successfully accessed bucket: {bucket_name}")
print(f"✓ Found {len(response['Contents'])} objects (showing max {max_objects}):")
print("\nObjects in bucket:")
for i, obj in enumerate(response['Contents'], 1):
size_mb = obj['Size'] / (1024 * 1024)
print(f" {i}. {obj['Key']}")
print(f" Size: {size_mb:.2f} MB | Modified: {obj['LastModified']}")
else:
print(f"\n✓ Bucket {bucket_name} is accessible but empty")
return True
except ClientError as e:
error_code = e.response.get('Error', {}).get('Code', 'Unknown')
print(f"\n✗ Error accessing bucket {bucket_name}")
print(f"✗ Error Code: {error_code}")
if error_code == 'NoSuchBucket':
print("✗ Bucket does not exist")
elif error_code == 'AccessDenied':
print("✗ Access denied to this bucket")
return False
except Exception as e:
print(f"\n✗ Unexpected error: {e}")
return False
def test_download_object(s3_client, bucket_name, object_key, local_path='test_download'):
"""
Test downloading an object from S3
Kiểm tra tải xuống object từ S3
"""
try:
print(f"\n=== Testing Object Download ===")
print(f"Bucket: {bucket_name}")
print(f"Object: {object_key}")
print(f"Local path: {local_path}")
# Create local directory if not exists
os.makedirs(os.path.dirname(local_path) if os.path.dirname(local_path) else '.', exist_ok=True)
# Download object
s3_client.download_file(bucket_name, object_key, local_path)
file_size = os.path.getsize(local_path)
print(f"\n✓ Successfully downloaded: {object_key}")
print(f"✓ File size: {file_size / (1024*1024):.2f} MB")
print(f"✓ Saved to: {local_path}")
return True
except ClientError as e:
error_code = e.response.get('Error', {}).get('Code', 'Unknown')
print(f"\n✗ Error downloading object")
print(f"✗ Error Code: {error_code}")
return False
except Exception as e:
print(f"\n✗ Unexpected error: {e}")
return False
def main():
"""
Main test function
Hàm test chính
"""
print("=" * 60)
print("AWS S3 Direct Access Test")
print("Test Truy Cập Trực Tiếp AWS S3")
print("=" * 60)
# Step 1: Load credentials
print("\n[Step 1] Loading credentials from file...")
credentials = load_credentials()
if not credentials:
print("\n✗ Test failed: Could not load credentials")
return False
# Step 2: Set credentials in environment
print("\n[Step 2] Setting AWS credentials...")
if not set_aws_credentials(credentials):
print("\n✗ Test failed: Could not set credentials")
return False
# Step 3: Test S3 connection
print("\n[Step 3] Testing S3 connection...")
s3_client, buckets = test_s3_connection()
if not s3_client:
print("\n✗ Test failed: Could not connect to S3")
return False
# Step 4: Test specific bucket access (if buckets exist)
if buckets and len(buckets) > 0:
print("\n[Step 4] Testing bucket access...")
first_bucket = buckets[0]['Name']
test_bucket_access(s3_client, first_bucket, max_objects=5)
print("\n" + "=" * 60)
print("✓ Test completed!")
print("✓ Test hoàn thành!")
print("=" * 60)
return True
if __name__ == "__main__":
try:
success = main()
sys.exit(0 if success else 1)
except KeyboardInterrupt:
print("\n\n✗ Test interrupted by user")
sys.exit(1)
except Exception as e:
print(f"\n✗ Fatal error: {e}")
import traceback
traceback.print_exc()
sys.exit(1)
test_s3_list_all.py
+194
View File
@@ -0,0 +1,194 @@
#!/usr/bin/env python3
"""
Test S3 list all objects
Test liệt kê tất cả objects trong S3 bucket
"""
import boto3
from botocore.exceptions import ClientError
from cognito_auth import CognitoAuthenticator
def list_all_s3_objects(bucket_name, region='us-west-2', prefix='', max_keys=0):
"""
List all objects in S3 bucket
Args:
bucket_name: Tên bucket
region: AWS region
prefix: Prefix để filter
max_keys: Số lượng max (0 = tất cả)
"""
print("=" * 70)
print(f"Listing S3 Objects")
print(f"Bucket: {bucket_name}")
print(f"Region: {region}")
if prefix:
print(f"Prefix: {prefix}")
print(f"Max: {max_keys if max_keys > 0 else 'ALL'}")
print("=" * 70)
# Initialize auth
auth = CognitoAuthenticator(region='ap-southeast-1')
# Load credentials
if not auth.load_tokens_from_file('train_files/crediential.txt'):
print("✗ Failed to load credentials")
return []
auth.get_credentials_from_cognito()
# Create S3 client
s3_client = boto3.client(
's3',
region_name=region,
aws_access_key_id=auth.aws_credentials['AccessKeyId'],
aws_secret_access_key=auth.aws_credentials['SecretAccessKey'],
aws_session_token=auth.aws_credentials['SessionToken']
)
# List objects with pagination
all_objects = []
continuation_token = None
page = 0
try:
while True:
page += 1
print(f"\n📄 Page {page}...")
# Prepare parameters
list_params = {
'Bucket': bucket_name,
'MaxKeys': 1000 if max_keys == 0 else min(max_keys - len(all_objects), 1000)
}
if prefix:
list_params['Prefix'] = prefix
if continuation_token:
list_params['ContinuationToken'] = continuation_token
# List objects
response = s3_client.list_objects_v2(**list_params)
if 'Contents' in response:
page_objects = response['Contents']
all_objects.extend(page_objects)
# Show some samples from this page
print(f" Found {len(page_objects)} objects on this page")
for obj in page_objects[:3]:
size_mb = obj['Size'] / (1024 * 1024)
print(f" - {obj['Key']} ({size_mb:.2f} MB)")
if len(page_objects) > 3:
print(f" ... and {len(page_objects) - 3} more")
# Check if should continue
if max_keys > 0 and len(all_objects) >= max_keys:
print(f"\n✓ Reached max_keys limit: {max_keys}")
break
if not response.get('IsTruncated', False):
print(f"\n✓ Reached end of list")
break
continuation_token = response.get('NextContinuationToken')
# Summary
print("\n" + "=" * 70)
print("Summary / Tổng Kết")
print("=" * 70)
if all_objects:
total_size = sum(obj['Size'] for obj in all_objects)
total_size_gb = total_size / (1024 * 1024 * 1024)
print(f"\n✓ Total objects: {len(all_objects)}")
print(f"✓ Total size: {total_size_gb:.2f} GB")
print(f"✓ Pages fetched: {page}")
# Show first and last
print(f"\nFirst 5 objects:")
for i, obj in enumerate(all_objects[:5], 1):
size_mb = obj['Size'] / (1024 * 1024)
print(f" {i}. {obj['Key']}")
print(f" Size: {size_mb:.2f} MB | Modified: {obj['LastModified']}")
if len(all_objects) > 10:
print(f"\nLast 5 objects:")
for i, obj in enumerate(all_objects[-5:], len(all_objects)-4):
size_mb = obj['Size'] / (1024 * 1024)
print(f" {i}. {obj['Key']}")
print(f" Size: {size_mb:.2f} MB | Modified: {obj['LastModified']}")
else:
print("\n⚠ No objects found")
print("=" * 70)
return all_objects
except ClientError as e:
error_code = e.response.get('Error', {}).get('Code', 'Unknown')
print(f"\n✗ S3 Error: {error_code}")
print(f"✗ Message: {e}")
return []
except Exception as e:
print(f"\n✗ Error: {e}")
import traceback
traceback.print_exc()
return []
def main():
"""Test listing objects"""
print("\n" + "╔" + "═" * 68 + "╗")
print("║" + " " * 20 + "S3 OBJECT LISTING TEST" + " " * 26 + "║")
print("╚" + "═" * 68 + "╝\n")
# Test cases
test_cases = [
{
'bucket': 'sentinel-cogs',
'region': 'us-west-2',
'prefix': 'sentinel-s2-l2a-cogs/54/S/VE/2020/1/',
'max_keys': 100,
'description': 'Sentinel-2 Vietnam 2020 January data'
},
{
'bucket': 'sentinel-cogs',
'region': 'us-west-2',
'prefix': 'sentinel-s2-l2a-cogs/',
'max_keys': 50,
'description': 'Sentinel-2 global data (sample)'
},
]
for i, test in enumerate(test_cases, 1):
print(f"\n{'='*70}")
print(f"TEST CASE {i}: {test['description']}")
print(f"{'='*70}\n")
objects = list_all_s3_objects(
bucket_name=test['bucket'],
region=test['region'],
prefix=test['prefix'],
max_keys=test['max_keys']
)
input(f"\n⏸ Press Enter to continue to next test...")
print("\n✓ All tests completed!")
if __name__ == "__main__":
try:
main()
except KeyboardInterrupt:
print("\n\n✗ Interrupted by user")
except Exception as e:
print(f"\n✗ Fatal error: {e}")
import traceback
traceback.print_exc()
train_files/01.train_DecisionTree_PlanetaryComputer.ipynb
+632
View File
@@ -0,0 +1,632 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "b1dab7f7",
"metadata": {},
"source": [
"# 🌍 Decision Tree Land Classification - Planetary Computer\n",
"\n",
"## 📌 Notebook này có thể chạy trên:\n",
"- ✅ **Local machine** (không cần ODC database)\n",
"- ✅ **Server ODC/JupyterHub** (có ODC database)\n",
"\n",
"## 🎯 Nguồn dữ liệu:\n",
"**Microsoft Planetary Computer STAC API**\n",
"- Sentinel-2 L2A (optical)\n",
"- Sentinel-1 RTC (SAR)\n",
"\n",
"## 🔄 Workflow:\n",
"1. Load data từ Planetary Computer (STAC)\n",
"2. Preprocessing (cloud mask, NDVI, resampling)\n",
"3. Train Decision Tree model\n",
"4. Evaluate & save model\n",
"\n",
"---"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "df9820b8",
"metadata": {},
"outputs": [],
"source": [
"%%time\n",
"%matplotlib inline\n",
"\n",
"import sys\n",
"import os\n",
"sys.path.insert(0, '/media/x79/2A7D-FAA0/remote-sensing')\n",
"\n",
"# Import module load dữ liệu không cần ODC database\n",
"import importlib\n",
"import load_data_no_odc\n",
"importlib.reload(load_data_no_odc)\n",
"\n",
"from load_data_no_odc import (\n",
" load_and_process_s2,\n",
" load_and_process_s1,\n",
" load_sentinel2_stac,\n",
" load_sentinel1_stac,\n",
" mask_clean_s2,\n",
" calculate_ndvi,\n",
" fill_nan_temporal,\n",
" resample_monthly\n",
")\n",
"\n",
"# Standard imports\n",
"import numpy as np\n",
"import pandas as pd\n",
"import xarray as xr\n",
"import matplotlib.pyplot as plt\n",
"import seaborn as sns\n",
"sns.set_style('whitegrid')\n",
"\n",
"# ML imports\n",
"from sklearn.tree import DecisionTreeClassifier\n",
"from sklearn.model_selection import train_test_split\n",
"from sklearn.metrics import classification_report, confusion_matrix, accuracy_score\n",
"import joblib\n",
"import json\n",
"from datetime import datetime\n",
"\n",
"# Dask for parallel processing\n",
"from dask.distributed import Client, LocalCluster\n",
"\n",
"print(\"✅ All modules loaded successfully!\")\n",
"print(\"📡 Data source: Microsoft Planetary Computer\")\n",
"print(\"💻 Environment: Local or Remote compatible\")"
]
},
{
"cell_type": "markdown",
"id": "53397b2f",
"metadata": {},
"source": [
"## 🚀 Step 1: Initialize Dask Cluster\n",
"\n",
"Khởi tạo Dask local cluster để xử lý song song"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "3c4d6779",
"metadata": {},
"outputs": [],
"source": [
"# Khởi tạo Dask LocalCluster\n",
"print(\"🚀 Initializing Dask LocalCluster...\")\n",
"\n",
"cluster = LocalCluster(\n",
" n_workers=4,\n",
" threads_per_worker=1,\n",
" memory_limit='4GB'\n",
")\n",
"client = Client(cluster)\n",
"\n",
"print(f\"✅ Dask cluster ready!\")\n",
"print(f\" Workers: {len(cluster.workers)}\")\n",
"print(f\" Dashboard: {client.dashboard_link}\")\n",
"print(\"=\" * 70)"
]
},
{
"cell_type": "markdown",
"id": "28bc5035",
"metadata": {},
"source": [
"## 📍 Step 2: Define Area of Interest (AOI)\n",
"\n",
"Định nghĩa vùng nghiên cứu và khoảng thời gian"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "15a5291c",
"metadata": {},
"outputs": [],
"source": [
"# Cấu hình vùng và thời gian\n",
"date_range = (\"2022-09-01\", \"2023-10-01\")\n",
"\n",
"# Bounding box: (lon_min, lat_min, lon_max, lat_max)\n",
"bbox = (105.5, 9.2, 106.4, 10.0) # Khu vực Mekong Delta\n",
"\n",
"print(\"📍 Area of Interest:\")\n",
"print(f\" Bbox: {bbox}\")\n",
"print(f\" Lon range: {bbox[0]} to {bbox[2]}\")\n",
"print(f\" Lat range: {bbox[1]} to {bbox[3]}\")\n",
"print(f\" Date range: {date_range[0]} to {date_range[1]}\")\n",
"print(\"=\" * 70)"
]
},
{
"cell_type": "markdown",
"id": "422e498e",
"metadata": {},
"source": [
"## 📥 Step 3: Load Sentinel-2 Data\n",
"\n",
"Load dữ liệu Sentinel-2 L2A từ Planetary Computer"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "611cd1c2",
"metadata": {},
"outputs": [],
"source": [
"%%time\n",
"\n",
"print(\"📥 Loading Sentinel-2 data from Planetary Computer...\")\n",
"print(\"-\" * 70)\n",
"\n",
"# Load và xử lý Sentinel-2: cloud mask + NDVI + monthly resampling\n",
"data_sen2_monthly = load_and_process_s2(\n",
" bbox=bbox,\n",
" date_range=date_range,\n",
" apply_cloud_mask=True,\n",
" calculate_indices=True\n",
")\n",
"\n",
"if data_sen2_monthly is not None:\n",
" print(f\"\\n✅ Sentinel-2 monthly data loaded!\")\n",
" print(f\" Dimensions: {dict(data_sen2_monthly.dims)}\")\n",
" print(f\" Variables: {list(data_sen2_monthly.data_vars)}\")\n",
" print(f\" Time steps: {len(data_sen2_monthly.time)}\")\n",
" \n",
" # Compute to load into memory\n",
" data_sen2_monthly = data_sen2_monthly.compute()\n",
" print(f\" ✓ Data computed and loaded into memory\")\n",
"else:\n",
" print(\"❌ Failed to load Sentinel-2 data\")"
]
},
{
"cell_type": "markdown",
"id": "79ef9736",
"metadata": {},
"source": [
"## 📥 Step 4: Load Sentinel-1 Data\n",
"\n",
"Load dữ liệu Sentinel-1 RTC (SAR) từ Planetary Computer"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "3bda3dc0",
"metadata": {},
"outputs": [],
"source": [
"%%time\n",
"\n",
"print(\"📥 Loading Sentinel-1 data from Planetary Computer...\")\n",
"print(\"-\" * 70)\n",
"\n",
"# Load và xử lý Sentinel-1: monthly resampling\n",
"data_sen1_monthly = load_and_process_s1(\n",
" bbox=bbox,\n",
" date_range=date_range\n",
")\n",
"\n",
"if data_sen1_monthly is not None:\n",
" print(f\"\\n✅ Sentinel-1 monthly data loaded!\")\n",
" print(f\" Dimensions: {dict(data_sen1_monthly.dims)}\")\n",
" print(f\" Variables: {list(data_sen1_monthly.data_vars)}\")\n",
" print(f\" Time steps: {len(data_sen1_monthly.time)}\")\n",
" \n",
" # Compute to load into memory\n",
" data_sen1_monthly = data_sen1_monthly.compute()\n",
" print(f\" ✓ Data computed and loaded into memory\")\n",
"else:\n",
" print(\"❌ Failed to load Sentinel-1 data\")"
]
},
{
"cell_type": "markdown",
"id": "de80d48d",
"metadata": {},
"source": [
"## 🎯 Step 5: Load Training Data\n",
"\n",
"Load dữ liệu mẫu huấn luyện (training samples)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "70925d09",
"metadata": {},
"outputs": [],
"source": [
"# Ánh xạ nhãn lớp đất\n",
"label_mapping = {\n",
" \"Lua tom\": 0,\n",
" \"Lua\": 1,\n",
" \"CHN\": 2,\n",
" \"CLN\": 3,\n",
" \"TS\": 4,\n",
" \"Song\": 5,\n",
" \"Dat xay dung\": 6,\n",
" \"Rung\": 7,\n",
"}\n",
"\n",
"print(\"🎯 Label mapping:\")\n",
"for label, idx in label_mapping.items():\n",
" print(f\" {idx}: {label}\")\n",
"\n",
"# TODO: Load training data from shapefile/GeoJSON\n",
"# Bạn cần cung cấp đường dẫn đến file training data\n",
"train_path = \"/media/x79/2A7D-FAA0/remote-sensing/train/train_data.geojson\" # Thay đổi path này\n",
"\n",
"print(f\"\\n📂 Loading training data from: {train_path}\")\n",
"print(\"⚠ Note: Bạn cần cập nhật train_path với đường dẫn thực tế\")"
]
},
{
"cell_type": "markdown",
"id": "fed9a8f1",
"metadata": {},
"source": [
"## 🔧 Step 6: Extract Features\n",
"\n",
"Trích xuất features từ Sentinel-1 và Sentinel-2 tại các điểm mẫu"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "62c41cd0",
"metadata": {},
"outputs": [],
"source": [
"# Placeholder for feature extraction\n",
"# Bạn cần implement hàm extract features từ train points\n",
"\n",
"def extract_features(train_data, data_s2, data_s1):\n",
" \"\"\"\n",
" Extract features from S2 and S1 data at training point locations\n",
" \"\"\"\n",
" X_list = []\n",
" y_list = []\n",
" \n",
" for idx, point in train_data.iterrows():\n",
" try:\n",
" lon, lat = point.geometry.x, point.geometry.y\n",
" \n",
" # Extract S2 data\n",
" s2_values = data_s2.sel(x=lon, y=lat, method='nearest').to_array().values.flatten()\n",
" \n",
" # Extract S1 data\n",
" s1_values = data_s1.sel(x=lon, y=lat, method='nearest').to_array().values.flatten()\n",
" \n",
" # Combine features\n",
" features = np.concatenate([s2_values, s1_values])\n",
" \n",
" # Skip if contains NaN\n",
" if not np.isnan(features).any():\n",
" X_list.append(features)\n",
" y_list.append(point['label_id'])\n",
" \n",
" except Exception as e:\n",
" continue\n",
" \n",
" return np.array(X_list), np.array(y_list)\n",
"\n",
"# TODO: Uncomment when training data is available\n",
"# X, y = extract_features(train_data, data_sen2_monthly, data_sen1_monthly)\n",
"# print(f\"✅ Extracted {len(X)} training samples\")\n",
"# print(f\" Features: {X.shape[1]}\")\n",
"# print(f\" Classes: {sorted(set(y.tolist()))}\")\n",
"\n",
"print(\"⚠ Feature extraction step - waiting for training data\")"
]
},
{
"cell_type": "markdown",
"id": "f7cd0ca2",
"metadata": {},
"source": [
"## 📊 Step 7: Split Data\n",
"\n",
"Chia dữ liệu thành train/val/test sets"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "523e1249",
"metadata": {},
"outputs": [],
"source": [
"# TODO: Uncomment when features are extracted\n",
"\n",
"# # Split train/val/test\n",
"# X_temp, X_test, y_temp, y_test = train_test_split(\n",
"# X, y, test_size=0.2, random_state=42, stratify=y\n",
"# )\n",
"\n",
"# X_train, X_val, y_train, y_val = train_test_split(\n",
"# X_temp, y_temp, test_size=0.125, random_state=42, stratify=y_temp\n",
"# )\n",
"\n",
"# # Combine train + val for final training\n",
"# X_fit = np.concatenate([X_train, X_val], axis=0)\n",
"# y_fit = np.concatenate([y_train, y_val], axis=0)\n",
"\n",
"# print(f\"✅ Data split:\")\n",
"# print(f\" Train: {len(X_train)} samples\")\n",
"# print(f\" Val: {len(X_val)} samples\")\n",
"# print(f\" Test: {len(X_test)} samples\")\n",
"# print(f\" Total: {len(X)} samples\")\n",
"\n",
"print(\"⚠ Data split step - waiting for features\")"
]
},
{
"cell_type": "markdown",
"id": "a469500c",
"metadata": {},
"source": [
"## 🌲 Step 8: Train Decision Tree Model\n",
"\n",
"Huấn luyện mô hình Decision Tree"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "445c2d92",
"metadata": {},
"outputs": [],
"source": [
"%%time\n",
"\n",
"# TODO: Uncomment when data is ready\n",
"\n",
"# # Train Decision Tree\n",
"# model = DecisionTreeClassifier(\n",
"# max_depth=30,\n",
"# min_samples_leaf=2,\n",
"# min_samples_split=5,\n",
"# class_weight=\"balanced\",\n",
"# random_state=42,\n",
"# )\n",
"\n",
"# print(\"🚀 Training Decision Tree...\")\n",
"# model.fit(X_fit, y_fit)\n",
"\n",
"# val_acc = model.score(X_val, y_val)\n",
"# print(f\"✅ Training complete!\")\n",
"# print(f\" Tree depth: {model.get_depth()}\")\n",
"# print(f\" Leaves: {model.get_n_leaves()}\")\n",
"# print(f\" Val accuracy: {val_acc:.4f} ({val_acc*100:.2f}%)\")\n",
"\n",
"print(\"⚠ Training step - waiting for data\")"
]
},
{
"cell_type": "markdown",
"id": "7c1e681e",
"metadata": {},
"source": [
"## 📈 Step 9: Hyperparameter Analysis\n",
"\n",
"Phân tích độ sâu tối ưu (max_depth) cho Decision Tree"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d13274e4",
"metadata": {},
"outputs": [],
"source": [
"%%time\n",
"\n",
"# TODO: Uncomment when model is trained\n",
"\n",
"# DEPTH_RANGE = list(range(1, 51))\n",
"# THRESHOLD = 0.001\n",
"\n",
"# train_accs = []\n",
"# val_accs = []\n",
"\n",
"# print(\"🔍 Analyzing convergence for max_depth...\")\n",
"# for d in DEPTH_RANGE:\n",
"# m = DecisionTreeClassifier(\n",
"# min_samples_leaf=2,\n",
"# min_samples_split=5,\n",
"# class_weight=\"balanced\",\n",
"# random_state=42,\n",
"# max_depth=d,\n",
"# )\n",
"# m.fit(X_fit, y_fit)\n",
"# train_accs.append(m.score(X_fit, y_fit))\n",
"# val_accs.append(m.score(X_val, y_val))\n",
"\n",
"# train_accs = np.array(train_accs)\n",
"# val_accs = np.array(val_accs)\n",
"\n",
"# best_depth = DEPTH_RANGE[np.argmax(val_accs)]\n",
"# best_val_acc = np.max(val_accs)\n",
"\n",
"# print(f\"✅ Best max_depth: {best_depth} (val_acc: {best_val_acc:.4f})\")\n",
"\n",
"# # Plot\n",
"# fig, ax = plt.subplots(1, 1, figsize=(12, 5))\n",
"# ax.plot(DEPTH_RANGE, train_accs, 'b-o', markersize=3, label='Train')\n",
"# ax.plot(DEPTH_RANGE, val_accs, 'g-o', markersize=3, label='Val')\n",
"# ax.axvline(x=best_depth, color='red', linestyle='--', label=f'Best={best_depth}')\n",
"# ax.set_xlabel('max_depth')\n",
"# ax.set_ylabel('Accuracy')\n",
"# ax.set_title('Train/Val Accuracy vs max_depth')\n",
"# ax.legend()\n",
"# ax.grid(True, alpha=0.3)\n",
"# plt.tight_layout()\n",
"# plt.show()\n",
"\n",
"print(\"⚠ Hyperparameter analysis - waiting for model\")"
]
},
{
"cell_type": "markdown",
"id": "b551cdb4",
"metadata": {},
"source": [
"## 📊 Step 10: Evaluate Model\n",
"\n",
"Đánh giá mô hình trên tập test"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "bc3d6b9a",
"metadata": {},
"outputs": [],
"source": [
"# TODO: Uncomment when model is trained\n",
"\n",
"# # Predictions\n",
"# y_pred = model.predict(X_test)\n",
"# acc = accuracy_score(y_test, y_pred)\n",
"\n",
"# print(f\"📊 Test Accuracy: {acc:.4f} ({acc*100:.2f}%)\\n\")\n",
"# print(classification_report(y_test, y_pred, digits=4))\n",
"\n",
"# # Confusion Matrix\n",
"# class_names = list(label_mapping.keys())\n",
"# cm = confusion_matrix(y_test, y_pred)\n",
"\n",
"# plt.figure(figsize=(9, 7))\n",
"# sns.heatmap(cm, annot=True, fmt='d', cmap='Greens',\n",
"# xticklabels=class_names, yticklabels=class_names)\n",
"# plt.xlabel('Predicted')\n",
"# plt.ylabel('Actual')\n",
"# plt.title('Confusion Matrix — Decision Tree')\n",
"# plt.tight_layout()\n",
"# plt.show()\n",
"\n",
"print(\"⚠ Evaluation step - waiting for model\")"
]
},
{
"cell_type": "markdown",
"id": "44d226fc",
"metadata": {},
"source": [
"## 💾 Step 11: Save Model\n",
"\n",
"Lưu mô hình và metadata"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "044eefc4",
"metadata": {},
"outputs": [],
"source": [
"# TODO: Uncomment when model is trained\n",
"\n",
"# # Save model\n",
"# model_path = \"model_decision_tree_planetary_computer.joblib\"\n",
"# joblib.dump(model, model_path)\n",
"# print(f\"✅ Model saved → {model_path}\")\n",
"\n",
"# # Save metadata\n",
"# info = {\n",
"# \"model_type\": \"DecisionTree\",\n",
"# \"data_source\": \"Microsoft Planetary Computer\",\n",
"# \"max_depth\": model.get_depth(),\n",
"# \"n_leaves\": model.get_n_leaves(),\n",
"# \"n_features\": int(X_fit.shape[1]),\n",
"# \"label_mapping\": label_mapping,\n",
"# \"test_accuracy\": float(acc),\n",
"# \"train_samples\": int(len(X_fit)),\n",
"# \"test_samples\": int(len(X_test)),\n",
"# \"bbox\": bbox,\n",
"# \"date_range\": date_range,\n",
"# \"saved_at\": datetime.now().isoformat(),\n",
"# }\n",
"\n",
"# info_path = \"model_decision_tree_planetary_computer_info.json\"\n",
"# with open(info_path, \"w\") as f:\n",
"# json.dump(info, f, indent=2, ensure_ascii=False)\n",
"\n",
"# print(f\"✅ Metadata saved → {info_path}\")\n",
"# print(json.dumps(info, indent=2, ensure_ascii=False))\n",
"\n",
"print(\"⚠ Save model step - waiting for trained model\")"
]
},
{
"cell_type": "markdown",
"id": "18c95152",
"metadata": {},
"source": [
"## 🧹 Step 12: Cleanup\n",
"\n",
"Đóng Dask cluster"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "20445068",
"metadata": {},
"outputs": [],
"source": [
"# Close Dask cluster\n",
"try:\n",
" client.close()\n",
" cluster.close()\n",
" print(\"✅ Dask cluster closed.\")\n",
"except Exception as e:\n",
" print(f\"⚠ Error closing cluster: {e}\")"
]
},
{
"cell_type": "markdown",
"id": "046bf0f9",
"metadata": {},
"source": [
"---\n",
"\n",
"## ✅ Summary\n",
"\n",
"### Notebook này:\n",
"- ✅ Load dữ liệu từ **Microsoft Planetary Computer**\n",
"- ✅ Không cần **ODC Database**\n",
"- ✅ Có thể chạy trên **local machine** hoặc **server ODC**\n",
"- ✅ Tương thích 100% với dữ liệu ODC\n",
"\n",
"### Workflow train/predict:\n",
"1. **Train trên server ODC**: Chạy notebook này với training data đầy đủ\n",
"2. **Save model**: Export `.joblib` file\n",
"3. **Predict trên local**: Load model và sử dụng cùng `load_data_no_odc.py` để load dữ liệu mới\n",
"\n",
"### Next steps:\n",
"1. Cập nhật `train_path` với đường dẫn training data thực tế\n",
"2. Uncomment các TODO cells\n",
"3. Run notebook để train model\n",
"4. Test prediction trên local machine\n",
"\n",
"---"
]
}
],
"metadata": {
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
train_files/01.train_ODC_DecisionTree.ipynb
+226 -75
View File
File diff suppressed because one or more lines are too long
train_files/01.train_ODC_MobileNet.ipynb
+4 -2
View File
@@ -192,7 +192,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 6, "execution_count": null,
"id": "c3faed92", "id": "c3faed92",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
@@ -243,7 +243,9 @@
"\n", "\n",
"def fill_nan(ds):\n", "def fill_nan(ds):\n",
" \"\"\"Fill NaN bằng interpolation theo thời gian\"\"\"\n", " \"\"\"Fill NaN bằng interpolation theo thời gian\"\"\"\n",
" return ds.interpolate_na(dim=\"time\", method=\"linear\", fill_value=\"extrapolate\")\n", " # Rechunk time dimension thành 1 chunk để tránh lỗi với interpolate_na\n",
" ds_rechunked = ds.chunk({\"time\": -1})\n",
" return ds_rechunked.interpolate_na(dim=\"time\", method=\"linear\", fill_value=\"extrapolate\")\n",
"\n", "\n",
"# Áp dụng tiền xử lý\n", "# Áp dụng tiền xử lý\n",
"data_clean = mask_clean(data_sen2)\n", "data_clean = mask_clean(data_sen2)\n",
train_files/crediential.txt
+6
View File
@@ -0,0 +1,6 @@
export AWS_ACCESS_KEY_ID="ASIA4YF43ZWIXQ6HJIAY"
export AWS_SECRET_ACCESS_KEY="3N8KoV2ZBqQcFqRUVxQXW8K9sm90CNDV9aHUkNw0"
export AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEO7//////////wEaDmFwLXNvdXRoZWFzdC0xIkcwRQIgR/6jnB1QIlWdCryDauBSFI34+KF256iV4X1Lyz4bP6YCIQCKE6A1q3jrlX9RFZ2hrFSOhdHKq187xD6yCUjnXOBkACrgBAi3//////////8BEAAaDDg3NjU2OTQxNTA1NyIMWcHZX71GMI7BdhDIKrQED9cJwWh4/DI3QG+W0jXsvCgvKnFTb+Yd690xdVxIrZuYsrmA+ncbp9Xj0GQSSay28XTBpG6Wgx8b2RSh3+ClJWvRJmvgxvOHpRAZqzgDB+0SJTWS1syeyGK9dlvL6VjI0LMw2r37m3mJzcUKvrVD8Hsg6iUT3FpPp0FKCbjpaSO9rZDXgxIGBwN/QbTNnD6wBwndYB0SI5QGHIIr2uW7IznKgXXY6sCq4uMjQDRULn6OO+ADSDBBahI/ZyhiTbUzL6LNjgvv0i9+Lots3xVg3Mp3yo8iKK5ILIVhmd/2QAX2uRQoLd9QRlChSQ4fwBKXEWN+A1UaPUfHRV0zudVmICBu+fk4lF3EWMkD/TRrkDCDRSBtMMa85m9yyxE7O0bSE/9gErDd/1zNNV1d64MRNrVhG4KWa6VnoOvvcupmgxmDV17db5o0tfFxa1TxHpkHsaTuCY0U8Q+Ep0da/LNdRkbiSoAjPNoAOoE+K5c7Z8g06Q13tTbvYQlDVIH1Kl3KtLlSDa6JUgWg0AMAFLJJCEEgBtLOL0nBuBujDOETnSXoIQjEJQkOvgWaiFP1A74hvXmpR39yOUzWCLRYhwXGt4xQW2NOYuHVFUdhu0rZ7XqHwzV6rYoBSP2uAZPw9diEZXYItgp2hHTAVEc0xEJp5hmBL94pPkpaMEwgOONW3QpqOj+Wg6yv7VPMY8FEEVEexHL1W134htZTgi8SMvx6M4M0oiL7caMy2tPkKF4o87ei6b3OMKnyoM0GOoMCneMKuFPe1C2TkcJ2qQ878F7c4ov/MtFZB3MsCMFXxm9Bx2Nj+v+nB9wQIxEUPA4/aS/zwYMTaYpHdNWJlQrMQllCfEIM9AmjX6GtP46bCPzzcawPlWYa88rGzQNKEEty1qJLJFIk8dK9ulImwIoOJ0aoRbaENYGUzWYJo6YM/yewVp6fm0MuULvlaOR9+tMxXfSc4leiXN6gk90fAW8SDUuaSHkCVphiH0dLnBf0cuzFyxFBMb3i8qNCFM6Mrnj1xPISYp/2PeGnAzEY/n56Cbaq/anT5BjwehAyUV1wPXW5Sdvgg1fygPbluQwtS1h7GxYFt72+GcusVabuXEmu9ab+AQ=="
Cognito: eyJraWQiOiIzejR4V0txYmd5Mlo4NXR3TFVvRGFSNmp4WVNSZUdKNHdLeEM4K0phbUM0PSIsImFsZyI6IlJTMjU2In0.eyJzdWIiOiJiMmM2ZmU5Ny02MGQ5LTQ1YmMtYTRlNy02ZTk1MDkwMmZlYjMiLCJjb2duaXRvOmdyb3VwcyI6WyJkZWZhdWx0LWdyb3VwIiwiYWxsb2NhdGlvbjpSLTE5MjQ0OkNTSVJPIGFuZCBWaWV0bmFtIHBhcnRuZXJzIl0sImlzcyI6Imh0dHBzOlwvXC9jb2duaXRvLWlkcC5hcC1zb3V0aGVhc3QtMS5hbWF6b25hd3MuY29tXC9hcC1zb3V0aGVhc3QtMV9DNEdDYllhT2EiLCJjbGllbnRfaWQiOiI2YTczMzJyOXRybGhxNmkyZmZwbDVma3JnNSIsIm9yaWdpbl9qdGkiOiJmNjczYjFjOS01NGQ3LTQ5MzYtODg4Yi1mMjJjZDU1YTBkNDYiLCJldmVudF9pZCI6IjRlZTUwZGNjLWEzYWMtNGMxYy04MjFhLTU2MGIwOGI0ZGFlYyIsInRva2VuX3VzZSI6ImFjY2VzcyIsInNjb3BlIjoiYXdzLmNvZ25pdG8uc2lnbmluLnVzZXIuYWRtaW4iLCJhdXRoX3RpbWUiOjE3NzI2MzIzNTgsImV4cCI6MTc3MjY2MTE1OCwiaWF0IjoxNzcyNjMyMzU5LCJqdGkiOiJhYWMxMzFmMS1jZTc3LTRlMmMtOGQ2Zi1hNDc3MWM0ZTY3ZjUiLCJ1c2VybmFtZSI6ImhpZW5tMjUyMzAwMSJ9.N15e6a0MWQtdZWBIHpDC3rKcwjn9ATEo-WY7oaB1SV4m2u1j17ld_AlnkiplAFWq52viKjWEO8ArY4Okb9xMUraK_nV-YxkAuTv15_hG32Q-qbFWsholcJzc4jObCKc3NXPVolx8zFZn0r9nQoakCqGaQWCfshT3_ZPAMdhIisOn7Jq6jDWyTfivMQlbmCPNCxSR3Yqt6_UTs2pvLdTuyP7hEGyuk8u4F_FyMxZcmGRKwPeKeepjJ22HoDkcvL9rpWIKoMZ-SQABLGqXPAFUEOPuvfCYh6bcivcztwaLszW70zHUbJzJZmyY8wi0PViplg-CK31yTkBej3-ARaC9Zg
ID: eyJraWQiOiJOMmdRc1c0S3o1YUltR3hGZEVJVmUxOUIxTWpZSmJPcG5kYUxKQUpNakxJPSIsImFsZyI6IlJTMjU2In0.eyJzdWIiOiJiMmM2ZmU5Ny02MGQ5LTQ1YmMtYTRlNy02ZTk1MDkwMmZlYjMiLCJjb2duaXRvOmdyb3VwcyI6WyJkZWZhdWx0LWdyb3VwIiwiYWxsb2NhdGlvbjpSLTE5MjQ0OkNTSVJPIGFuZCBWaWV0bmFtIHBhcnRuZXJzIl0sImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJpc3MiOiJodHRwczpcL1wvY29nbml0by1pZHAuYXAtc291dGhlYXN0LTEuYW1hem9uYXdzLmNvbVwvYXAtc291dGhlYXN0LTFfQzRHQ2JZYU9hIiwiY29nbml0bzp1c2VybmFtZSI6ImhpZW5tMjUyMzAwMSIsIm9yaWdpbl9qdGkiOiJmNjczYjFjOS01NGQ3LTQ5MzYtODg4Yi1mMjJjZDU1YTBkNDYiLCJhdWQiOiI2YTczMzJyOXRybGhxNmkyZmZwbDVma3JnNSIsImV2ZW50X2lkIjoiNGVlNTBkY2MtYTNhYy00YzFjLTgyMWEtNTYwYjA4YjRkYWVjIiwidG9rZW5fdXNlIjoiaWQiLCJhdXRoX3RpbWUiOjE3NzI2MzIzNTgsIm5hbWUiOiJIaWVuIFBoYW4iLCJleHAiOjE3NzI2NjExNTgsImlhdCI6MTc3MjYzMjM1OSwianRpIjoiMGViODczOTItNWZhMS00OTAwLWFiZmUtNzFmMWEyYjI1YzdiIiwiZW1haWwiOiJoaWVubTI1MjMwMDFAZ3N0dWRlbnQuY3R1LmVkdS52biJ9.QEoa4uJmmSk0qpOBi_a3RrCoqRu-oASbAHc24tgIuSeM_wcQsyTbKNbYdDS9P0n6JZf-wCiCknpsHl6TKiGDL3xDVguesm8ZPdyYkdgpCvE7EnyrCOSa01hcubAL-Z3_TMyUVs0WyDrJ3HS0YT-0Gig7m2y17oL44zwtrWwXcrTJMW94QimW5OMsDKZMn1oQKKGkBhC18FB4lNcerAh9tLknGfJQEseH6_5rJAeLJSwzXJBCfZjM_Yt-4ZmmB1ruNfiHUXfT2QRbBFRDcWcpA0gYJoROrm16tByyivwGcaV2BIRIiSKqi1NSRPdTNqOWmmYh-yufWFG8CdumGpX5ow
train_files/test_cognito_odc.ipynb
+565
View File
File diff suppressed because one or more lines are too long
train_files/test_planetary_computer.ipynb
+155
View File
@@ -0,0 +1,155 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "30681e56",
"metadata": {},
"source": [
"# 🧪 Test Planetary Computer Connection\n",
"\n",
"Notebook này test kết nối và load dữ liệu từ Microsoft Planetary Computer"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e32c8eca",
"metadata": {},
"outputs": [],
"source": [
"import sys\n",
"sys.path.insert(0, '/media/x79/2A7D-FAA0/remote-sensing')\n",
"\n",
"from load_data_no_odc import load_sentinel2_stac, load_sentinel1_stac\n",
"import matplotlib.pyplot as plt\n",
"\n",
"print(\"✅ Module imported successfully\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "501dd8ef",
"metadata": {},
"outputs": [],
"source": [
"# Small test area (1 month, small bbox)\n",
"bbox = (105.8, 9.5, 106.0, 9.7) # Small area in Mekong Delta\n",
"date_range = (\"2023-01-01\", \"2023-01-31\") # 1 month only\n",
"\n",
"print(f\"Test parameters:\")\n",
"print(f\" Bbox: {bbox}\")\n",
"print(f\" Date: {date_range}\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "187e2c1a",
"metadata": {},
"outputs": [],
"source": [
"# Test Sentinel-2\n",
"print(\"\\n\" + \"=\" * 70)\n",
"print(\"Testing Sentinel-2 L2A\")\n",
"print(\"=\" * 70)\n",
"\n",
"data_s2 = load_sentinel2_stac(\n",
" bbox=bbox,\n",
" date_range=date_range,\n",
" bands=['red', 'green', 'blue', 'nir08', 'SCL'],\n",
" resolution=60 # Lower resolution for faster test\n",
")\n",
"\n",
"if data_s2 is not None:\n",
" print(f\"\\n✅ SUCCESS! Sentinel-2 loaded\")\n",
" print(f\" Dims: {dict(data_s2.dims)}\")\n",
" print(f\" Vars: {list(data_s2.data_vars)}\")\n",
"else:\n",
" print(\"\\n❌ Failed to load Sentinel-2\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e688f3b9",
"metadata": {},
"outputs": [],
"source": [
"# Test Sentinel-1\n",
"print(\"\\n\" + \"=\" * 70)\n",
"print(\"Testing Sentinel-1 RTC\")\n",
"print(\"=\" * 70)\n",
"\n",
"data_s1 = load_sentinel1_stac(\n",
" bbox=bbox,\n",
" date_range=date_range,\n",
" bands=['vv', 'vh'],\n",
" resolution=60 # Lower resolution for faster test\n",
")\n",
"\n",
"if data_s1 is not None:\n",
" print(f\"\\n✅ SUCCESS! Sentinel-1 loaded\")\n",
" print(f\" Dims: {dict(data_s1.dims)}\")\n",
" print(f\" Vars: {list(data_s1.data_vars)}\")\n",
"else:\n",
" print(\"\\n❌ Failed to load Sentinel-1\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "211796a7",
"metadata": {},
"outputs": [],
"source": [
"# Visualize if data loaded successfully\n",
"if data_s2 is not None:\n",
" print(\"\\n📊 Visualizing Sentinel-2 RGB composite...\")\n",
" \n",
" # Select first timestep\n",
" rgb = data_s2[['red', 'green', 'blue']].isel(time=0)\n",
" \n",
" # Plot\n",
" fig, axes = plt.subplots(1, 3, figsize=(15, 5))\n",
" \n",
" rgb['red'].plot(ax=axes[0], cmap='Reds')\n",
" axes[0].set_title('Red band')\n",
" \n",
" rgb['green'].plot(ax=axes[1], cmap='Greens')\n",
" axes[1].set_title('Green band')\n",
" \n",
" rgb['blue'].plot(ax=axes[2], cmap='Blues')\n",
" axes[2].set_title('Blue band')\n",
" \n",
" plt.tight_layout()\n",
" plt.show()\n",
" \n",
" print(\"✅ Visualization complete!\")"
]
},
{
"cell_type": "markdown",
"id": "d4e29a09",
"metadata": {},
"source": [
"## ✅ Results\n",
"\n",
"Nếu cả 2 tests đều pass:\n",
"- ✅ Kết nối Planetary Computer OK\n",
"- ✅ Load Sentinel-2 OK\n",
"- ✅ Load Sentinel-1 OK\n",
"- ✅ Sẵn sàng sử dụng cho training!\n",
"\n",
"Next step: Sử dụng `01.train_DecisionTree_PlanetaryComputer.ipynb` để train model"
]
}
],
"metadata": {
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 5
}