basketballcantho/remote-sensing
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

thêm chức năng train trên odc predict trên planetary

Browse Source
This commit is contained in:
Victor Phan
2026-03-04 23:03:04 +07:00
parent ebb8e6e4b3
commit 8a1e7bb22e
39 changed files with 15297 additions and 711 deletions
Download Patch File Download Diff File Expand all files Collapse all files
easi_tools/deployments.py
+354
View File
@@ -0,0 +1,354 @@
#!python3
import sys
import os
import logging
import collections
# A class that provides notebook variables for each of the EASI deployments
# Map an internal deployment name to deployment variables and search parameters.
# Update to ensure that the product/space/time parameters are available in the respective databases
deployment_map = {
'adias': {
'domain': 'adias.aquawatchaus.space',
'db_database': 'adias_prod_db',
'training_shapefile': '',
'scratch': 'adias-prod-user-scratch',
'productmap': {'landsat': 'landsat8_c2l2_sr', 'sentinel-2': 's2_l2a', 'sar': 'asf_s1_grd_gamma0', 'dem': 'copernicus_dem_30'},
'location': 'Lake Tahoe, California',
'latitude': (39.0, 39.3),
'longitude': (-120.2, -119.9),
'time': ('2022-02-01', '2022-05-01'),
'target': {
'landsat': {'crs': 'epsg:26911', 'resolution': (-30,30)},
'sentinel-2': {'crs': 'epsg:26911', 'resolution': (-10,10)}
},
'aliases': {
'landsat': {'qa_band': 'qa_pixel', 'nir': 'nir08', 'swir1': 'swir16', 'swir2': 'swir22'}
},
'qa_mask': {
'landsat': {'nodata': False, 'water': 'land_or_cloud',
'cloud': 'not_high_confidence', 'cloud_shadow': 'not_high_confidence'}
}
},
'asia': {
'domain': 'asia.easi-eo.solutions',
'db_database': 'easi_asia_db',
'training_shapefile': '',
'scratch': 'easi-asia-user-scratch',
'productmap': {'landsat': 'landsat8_c2l2_sr', 'sentinel-2': 'sentinel_2_c1_l2a', 'sentinel-1': 'sentinel1_grd_gamma0_20m', 'dem': 'copernicus_dem_30'},
'location': 'Lake Tempe, Indonesia',
'latitude': (-4.2, -3.9),
'longitude': (119.8, 120.1),
'time': ('2020-02-01', '2020-04-01'),
'proxy': True,
'target': {
'landsat': {'crs': 'epsg:32650', 'resolution': (-30,30)},
'sentinel-2': {'crs': 'epsg:32650', 'resolution': (-10,10)}
},
'aliases': {
'landsat': {'qa_band': 'qa_pixel', 'nir': 'nir08', 'swir1': 'swir16', 'swir2': 'swir22'}
},
'qa_mask': {
'landsat': {'nodata': False, 'water': 'land_or_cloud',
'cloud': 'not_high_confidence', 'cloud_shadow': 'not_high_confidence'}
}
},
'chile': {
'domain': 'datacubechile.cl',
'db_database': 'easido_prod_db',
'training_shapefile': '',
'scratch': 'easido-prod-user-scratch',
'productmap': {'landsat': 'landsat8_c2l2_sr', 'sentinel-2': 'sentinel_2_c1_l2a', 'sar': 'asf_s1_grd_gamma0', 'dem': 'copernicus_dem_30'},
'location': 'La Serena, Chile',
'latitude': (-29.95, -29.85),
'longitude': (-71.3, -71.2),
'latitude_big': (-29.95, -27.95),
'longitude_big': (-71.3, -69.3),
'time': ('2022-02-01', '2022-05-01'),
'target': {
'landsat': {'crs': 'epsg:32718', 'resolution': (-30,30)},
'sentinel-2': {'crs': 'epsg:32718', 'resolution': (-10,10)}
},
'aliases': {
'landsat': {'qa_band': 'qa_pixel', 'nir': 'nir08', 'swir1': 'swir16', 'swir2': 'swir22'}
},
'qa_mask': {
'landsat': {'nodata': False, 'water': 'land_or_cloud',
'cloud': 'not_high_confidence', 'cloud_shadow': 'not_high_confidence'}
}
},
'cal': {
'domain': 'cal.ceos.org',
'db_database': 'easi_cal_db',
'training_shapefile': './ancillary_data/VA_Counties_Newport_News.shp',
'scratch': 'easi-cal-user-scratch',
'ows': False,
'map': False,
'productmap': {'landsat': 'landsat8_c2l2_sr', 'sentinel-2': 's2_l2a', 'sentinel-1': 's1_rtc', 'dem': 'copernicus_dem_30'},
'location': 'Newport News, Virginia',
'latitude': (37.02, 37.12),
'longitude': (-76.55, -76.45),
'time': ('2022-01-01', '2022-04-01'),
'target': {
'landsat': {'crs': 'epsg:32618', 'resolution': (-30,30)},
'sentinel-2': {'crs': 'epsg:32618', 'resolution': (-10,10)}
},
'aliases': {
'landsat': {'qa_band': 'qa_pixel', 'nir': 'nir08', 'swir1': 'swir16', 'swir2': 'swir22'}
},
'qa_mask': {
'landsat': {'nodata': False, 'water': 'land_or_cloud',
'cloud': 'not_high_confidence', 'cloud_shadow': 'not_high_confidence'}
}
},
'csiro': {
'domain': 'csiro.easi-eo.solutions',
'db_database': 'easihub_csiro_db',
'training_shapefile': '',
'scratch': 'easihub-csiro-user-scratch',
'productmap': {'landsat': 'ga_ls8c_ard_3', 'sentinel-2': 'ga_s2am_ard_3', 'sentinel-1': 'sentinel1_grd_gamma0_20m', 'dem': 'copernicus_dem_30'},
'location': 'Lake Hume, Australia',
'latitude': (-36.3, -35.8),
'longitude': (146.8, 147.3),
'time': ('2020-02-01', '2020-04-01'),
'aliases': {
'landsat': {'red': 'nbart_red', 'green': 'nbart_green', 'blue': 'nbart_blue',
'nir': 'nbart_nir', 'swir1': 'nbart_swir_1', 'swir2': 'nbart_swir_2',
'qa_band': 'oa_fmask'}
},
'qa_mask': {
'landsat': {'fmask':'valid'}
}
},
'dcceew': {
'domain': 'dcceew.easi-eo.solutions',
'db_database': 'dcceew_prod_db',
'training_shapefile': '',
'scratch': 'dcceew-prod-user-scratch',
'productmap': {'landsat': 'ga_ls8c_ard_3', 'sentinel-2': 'ga_s2am_ard_3', 'sentinel-1': 'sentinel1_grd_gamma0_20m', 'dem': 'copernicus_dem_30'},
'location': 'Lake Hume, Australia',
'latitude': (-36.3, -35.8),
'longitude': (146.8, 147.3),
'time': ('2020-02-01', '2020-04-01'),
'aliases': {
'landsat': {'red': 'nbart_red', 'green': 'nbart_green', 'blue': 'nbart_blue',
'nir': 'nbart_nir', 'swir1': 'nbart_swir_1', 'swir2': 'nbart_swir_2',
'qa_band': 'oa_fmask'}
},
'qa_mask': {
'landsat': {'fmask':'valid'}
}
},
'sub-apse2': {
'domain': 'sub-apse2.easi-eo.solutions',
'db_database': '',
'training_shapefile': '',
'scratch': '',
'ows': False,
'map': False,
'productmap': {'landsat': 'ga_ls8c_ard_3', 'sentinel-2': 'ga_s2am_ard_3', 'dem': 'copernicus_dem_30'},
'location': 'Lake Hume, Australia',
'latitude': (-36.3, -35.8),
'longitude': (146.8, 147.3),
'time': ('2020-02-01', '2020-04-01'),
'aliases': {
'landsat': {'red': 'nbart_red', 'green': 'nbart_green', 'blue': 'nbart_blue',
'nir': 'nbart_nir', 'swir1': 'nbart_swir_1', 'swir2': 'nbart_swir_2',
'qa_band': 'oa_fmask'}
},
'qa_mask': {
'landsat': {'fmask':'valid'}
}
},
}
class EasiDefaults():
"""Provide deployment-specific default variables for EASI notebooks"""
def __init__(self, deployment=None):
"""Initialise"""
self._log = _getlogger(self.__class__.__name__)
self.name = deployment if deployment else self._find_deployment()
self.deployment = self._validate(self.name)
self.proxy = None
self._aliases = {}
if self.deployment and self.deployment.get('proxy', None):
self.proxy = EasiCachingProxy()
if self.deployment:
self._log.info(f'Successfully found configuration for deployment "{self.name}"')
def _validate(self, deployment) -> dict:
"""Return the dict associated with the deployment name"""
names = deployment_map.keys()
if deployment is None or deployment not in names:
self._log.error(f'Deployment name not recognised: {deployment}')
self._log.error(f'Select one of: {", ".join(names)}')
return None
return deployment_map[deployment]
def _find_deployment(self) -> str:
"""Use the deployment's database environment variable as a lookup into the deployment_map dict"""
db_database = os.environ['DB_DATABASE']
deployment_name = [item for item in deployment_map if deployment_map[item]["db_database"] == db_database]
msg = 'Try specifying one using EasiDefaults(deployment="deployment_name").'
if len(deployment_name) == 0:
self._log.error(f'Deployment could not be found automatically. {msg}')
return None
elif len(deployment_name) > 1:
self._log.error(f'More than one deployment found. {msg}')
return None
return deployment_name[0]
@property
def domain(self):
"""Deployment domain"""
return self.deployment['domain']
@property
def db_database(self):
"""Database name"""
return self.deployment['db_database']
@property
def training_shapefile(self):
"""A local shapefile"""
return self.deployment['training_shapefile']
@property
def hub(self):
"""JupyterLab URL"""
return f'https://hub.{self.domain}'
@property
def explorer(self):
"""Explorer URL"""
return f'https://explorer.{self.domain}'
@property
def ows(self):
"""OWS URL"""
if not self.deployment.get('ows', True):
self._log.warning(f'Deployment does not have an OWS service: {self.name}')
return None
return f'https://ows.{self.domain}'
@property
def terria(self):
"""Terria Map URL"""
if not self.deployment.get('map', True):
self._log.warning(f'Deployment does not have a Map service: {self.name}')
return None
return f'https://map.{self._domain()}'
@property
def scratch(self):
"""Scratch bucket"""
return self.deployment['scratch']
@property
def location(self):
"""Default location name"""
return self.deployment['location']
@property
def latitude(self):
"""Default latitude range"""
return self.deployment['latitude']
@property
def longitude(self):
"""Default longitude range"""
return self.deployment['longitude']
@property
def latitude_big(self):
"""Default big latitude range"""
if 'latitude_big' in self.deployment:
return self.deployment['latitude_big']
self._log.warning(f'Default big latitude range not defined for "{self.deployment}". Using default latitude range')
return self.latitude
@property
def longitude_big(self):
"""Default big longitude range"""
if 'longitude_big' in self.deployment:
return self.deployment['longitude_big']
self._log.warning(f'Default big longitude range not defined for "{self.deployment}". Using default longitude range')
return self.latitude
@property
def time(self):
"""Default time range"""
return self.deployment['time']
def product(self, family='landsat'):
"""Product name. Family loosely describes products from a satellite series or product type."""
p = self.deployment['productmap'].get(family, None)
if p is None:
self._log.warning(f'Product family not defined for "{self.name}": {family}')
out = ', '.join([f'{k} > {v}' for k,v in self.deployment['productmap'].items()])
self._log.warning(f'{self.name}: {out}')
return None
return p
def crs(self, family='landsat'):
"""Default resolution. Family loosely describes products from a satellite series or product type."""
return self.deployment.get('target', {}).get(family, {}).get('crs', None)
def resolution(self, family='landsat'):
"""Default resolution. Family loosely describes products from a satellite series or product type."""
return self.deployment.get('target', {}).get(family, {}).get('resolution', None)
def aliases(self, family='landsat') -> collections.UserDict:
"""Return a dict-like object that maps a common name to a specific measurement/alias name.
Family loosely describes products from a satellite series or product type.
The common name is returned if there is no specific measurement/alias name defined.
That is, the common name should work as a measurement/alias name for the family in this deployment.
Else, provide a specific measurement/alias name in the defaults above.
"""
if family not in self._aliases:
self._aliases[family] = EasiAlias(self.deployment.get('aliases', {}).get(family, {}))
return self._aliases[family]
def qa_mask(self, family='landsat') -> dict:
"""Default QA mask values. Family loosely describes products from a satellite series or product type."""
return self.deployment.get('qa_mask', {}).get(family, {})
class EasiAlias(collections.UserDict):
"""Custom UserDict that returns a default measurement name for a given key if defined.
Else returns the key as the value. Items can not be set."""
def __init__(self, default:dict = {}):
self.data = default
self._log = _getlogger(self.__class__.__name__)
def __getitem__(self, key):
if key in self.data:
return self.data[key]
return key
def __setitem__(self, key, val):
self._log.error(f'Error <{self.__class__.__name__}>: Can not set items')
class EasiCachingProxy():
"""Set, unset and return information about the user's caching-proxy configuration"""
def __init__(self):
pass
def _getlogger(name):
"""Return a logger. Define here to limit external dependencies"""
# Default logger
# log.hasHandlers() = False
# log.getEffectiveLevel() = 30 = warning
# log.propagate = True
logger = logging.getLogger(name)
logger.setLevel(logging.INFO)
if not len(logger.handlers):
logger.addHandler(logging.StreamHandler(sys.stdout))
logger.propagate = False # Do not propagate up to root logger, which may have other handlers
return logger