mirror of
https://git.victorphan.net/basketballcantho/CSIROBoeingPhase5-Vietnam.git
synced 2026-08-05 05:43:10 +07:00
Victor Phan
312 lines
13 KiB
Bash
Executable File
312 lines
13 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Script to download, extract and arrange SEN12MS-CR-TS and SEN12MS-CR.
|
|
# Make this script executable (by running: chmod +x dl_data.sh),
|
|
# then give it a run (by calling: ./dl_data.sh) and
|
|
# follow the prompts in order to get the desired data.
|
|
|
|
clear
|
|
echo "This script is for downloading the SEN12MS-CR-TS data set for cloud removal in satellite data."
|
|
echo See the associated paper: Ebel et al \(2022\) \'SEN12MS-CR-TS: A Remote Sensing Data Set for Multi-modal Multi-temporal Cloud Removal\'
|
|
echo -e 'Click \e]8;;https://patricktum.github.io/cloud_removal/\ahere\e]8;;\a for more information'
|
|
echo
|
|
echo
|
|
|
|
while true; do
|
|
read -p "Do you wish to download the multitemporal SEN12MS-CR-TS data set? " yn
|
|
case $yn in
|
|
[Yy]* ) SEN12MSCRTS=true; break;;
|
|
[Nn]* ) SEN12MSCRTS=false; break;;
|
|
* ) echo "Please answer yes or no.";;
|
|
esac
|
|
done
|
|
|
|
if [ "$SEN12MSCRTS" = "true" ]; then
|
|
while true; do
|
|
read -p "What regions would you like to download? [all|africa|america|asiaEast|asiaWest|europa] " region
|
|
case $region in
|
|
all|africa|america|asiaEast|asiaWest|europa ) reg=$region; break;;
|
|
* ) echo "Please answer [all|africa|america|asiaEast|asiaWest|europa].";;
|
|
esac
|
|
done
|
|
fi
|
|
|
|
while true; do
|
|
read -p "Do you wish to also download the monotemporal SEN12MS-CR data set (all regions)? " yn
|
|
case $yn in
|
|
[Yy]* ) SEN12MSCR=true; break;;
|
|
[Nn]* ) SEN12MSCR=false; break;;
|
|
* ) echo "Please answer yes or no.";;
|
|
esac
|
|
done
|
|
|
|
while true; do
|
|
read -p "Do you wish to also download the Sentinel-1 radar data associated with your previous choices? " yn
|
|
case $yn in
|
|
[Yy]* ) S1=true; break;;
|
|
[Nn]* ) S1=false; break;;
|
|
* ) echo "Please answer yes or no.";;
|
|
esac
|
|
done
|
|
|
|
declare -A url_dict # holding links to data
|
|
declare -A vol_dict # bookkeeping size of data
|
|
|
|
echo "Please enter the path to download and extract the data to: "
|
|
read dl_extract_to
|
|
|
|
|
|
echo
|
|
echo
|
|
if [ "$SEN12MSCRTS" = "true" ]; then
|
|
|
|
echo "Downloading SEN12MS-CR-TS data set."
|
|
mkdir -p $dl_extract_to'/SEN12MSCRTS'
|
|
|
|
# train split
|
|
case $region in
|
|
'all') url_dict['multi_s2_africa']='https://dataserv.ub.tum.de/s/m1639953/download?path=/&files=s2_africa.tar.gz'
|
|
vol_dict['multi_s2_africa']='98233900'
|
|
|
|
url_dict['multi_s2_america']='https://dataserv.ub.tum.de/s/m1639953/download?path=/&files=s2_america.tar.gz'
|
|
vol_dict['multi_s2_america']='110245004'
|
|
|
|
url_dict['multi_s2_asiaEast']='https://dataserv.ub.tum.de/s/m1639953/download?path=/&files=s2_asiaEast.tar.gz'
|
|
vol_dict['multi_s2_asiaEast']='113948560'
|
|
|
|
url_dict['multi_s2_asiaWest']='https://dataserv.ub.tum.de/s/m1639953/download?path=/&files=s2_asiaWest.tar.gz'
|
|
vol_dict['multi_s2_asiaWest']='96082796'
|
|
|
|
url_dict['multi_s2_europa']='https://dataserv.ub.tum.de/s/m1639953/download?path=/&files=s2_europa.tar.gz'
|
|
vol_dict['multi_s2_europa']='196669740'
|
|
;;
|
|
'africa') url_dict['multi_s2_africa']='https://dataserv.ub.tum.de/s/m1639953/download?path=/&files=s2_africa.tar.gz'
|
|
vol_dict['multi_s2_africa']='98233900'
|
|
;;
|
|
'america') url_dict['multi_s2_america']='https://dataserv.ub.tum.de/s/m1639953/download?path=/&files=s2_america.tar.gz'
|
|
vol_dict['multi_s2_america']='110245004'
|
|
;;
|
|
'asiaEast') url_dict['multi_s2_asiaEast']='https://dataserv.ub.tum.de/s/m1639953/download?path=/&files=s2_asiaEast.tar.gz'
|
|
vol_dict['multi_s2_asiaEast']='113948560'
|
|
;;
|
|
'asiaWest') url_dict['multi_s2_asiaWest']='https://dataserv.ub.tum.de/s/m1639953/download?path=/&files=s2_asiaWest.tar.gz'
|
|
vol_dict['multi_s2_asiaWest']='96082796'
|
|
;;
|
|
'europa') url_dict['multi_s2_europa']='https://dataserv.ub.tum.de/s/m1639953/download?path=/&files=s2_europa.tar.gz'
|
|
vol_dict['multi_s2_europa']='196669740'
|
|
;;
|
|
esac
|
|
|
|
|
|
# test split
|
|
case $region in
|
|
'all') url_dict['multi_s2_africa_test']='https://dataserv.ub.tum.de/s/m1659251/download?path=/&files=s2_africa_test.tar.gz'
|
|
vol_dict['multi_s2_africa_test']='25421744'
|
|
|
|
url_dict['multi_s2_america_test']='https://dataserv.ub.tum.de/s/m1659251/download?path=/&files=s2_america_test.tar.gz'
|
|
vol_dict['multi_s2_america_test']='25421824'
|
|
|
|
url_dict['multi_s2_asiaEast_test']='https://dataserv.ub.tum.de/s/m1659251/download?path=/&files=s2_asiaEast_test.tar.gz'
|
|
vol_dict['multi_s2_asiaEast_test']='40534760'
|
|
|
|
url_dict['multi_s2_asiaWest_test']='https://dataserv.ub.tum.de/s/m1659251/download?path=/&files=s2_asiaWest_test.tar.gz'
|
|
vol_dict['multi_s2_asiaWest_test']='15012924'
|
|
|
|
url_dict['multi_s2_europa_test']='https://dataserv.ub.tum.de/s/m1659251/download?path=/&files=s2_europa_test.tar.gz'
|
|
vol_dict['multi_s2_europa_test']='79568460'
|
|
;;
|
|
'africa') url_dict['multi_s2_africa_test']='https://dataserv.ub.tum.de/s/m1659251/download?path=/&files=s2_africa_test.tar.gz'
|
|
vol_dict['multi_s2_africa_test']='25421744'
|
|
;;
|
|
'america') url_dict['multi_s2_america_test']='https://dataserv.ub.tum.de/s/m1659251/download?path=/&files=s2_america_test.tar.gz'
|
|
vol_dict['multi_s2_america_test']='25421824'
|
|
;;
|
|
'asiaEast') url_dict['multi_s2_asiaEast_test']='https://dataserv.ub.tum.de/s/m1659251/download?path=/&files=s2_asiaEast_test.tar.gz'
|
|
vol_dict['multi_s2_asiaEast_test']='40534760'
|
|
;;
|
|
'asiaWest') url_dict['multi_s2_asiaWest_test']='https://dataserv.ub.tum.de/s/m1659251/download?path=/&files=s2_asiaWest_test.tar.gz'
|
|
vol_dict['multi_s2_asiaWest_test']='15012924'
|
|
;;
|
|
'europa') url_dict['multi_s2_europa_test']='https://dataserv.ub.tum.de/s/m1659251/download?path=/&files=s2_europa_test.tar.gz'
|
|
vol_dict['multi_s2_europa_test']='79568460'
|
|
;;
|
|
esac
|
|
|
|
|
|
if [ "$S1" = "true" ]; then
|
|
# train split
|
|
case $region in
|
|
'all') url_dict['multi_s1_africa']='https://dataserv.ub.tum.de/s/m1639953/download?path=/&files=s1_africa.tar.gz'
|
|
vol_dict['multi_s1_africa']='60544524'
|
|
|
|
url_dict['multi_s1_america']='https://dataserv.ub.tum.de/s/m1639953/download?path=/&files=s1_america.tar.gz'
|
|
vol_dict['multi_s1_america']='67947416'
|
|
|
|
url_dict['multi_s1_asiaEast']='https://dataserv.ub.tum.de/s/m1639953/download?path=/&files=s1_asiaEast.tar.gz'
|
|
vol_dict['multi_s1_asiaEast']='70230104'
|
|
|
|
url_dict['multi_s1_asiaWest']='https://dataserv.ub.tum.de/s/m1639953/download?path=/&files=s1_asiaWest.tar.gz'
|
|
vol_dict['multi_s1_asiaWest']='59218848'
|
|
|
|
url_dict['multi_s1_europa']='https://dataserv.ub.tum.de/s/m1639953/download?path=/&files=s1_europa.tar.gz'
|
|
vol_dict['multi_s1_europa']='121213836'
|
|
;;
|
|
'africa') url_dict['multi_s1_africa']='https://dataserv.ub.tum.de/s/m1639953/download?path=/&files=s1_africa.tar.gz'
|
|
vol_dict['multi_s1_africa']='60544524'
|
|
;;
|
|
'america') url_dict['multi_s1_america']='https://dataserv.ub.tum.de/s/m1639953/download?path=/&files=s1_america.tar.gz'
|
|
vol_dict['multi_s1_america']='67947416'
|
|
;;
|
|
'asiaEast') url_dict['multi_s1_asiaEast']='https://dataserv.ub.tum.de/s/m1639953/download?path=/&files=s1_asiaEast.tar.gz'
|
|
vol_dict['multi_s1_asiaEast']='70230104'
|
|
;;
|
|
'asiaWest') url_dict['multi_s1_asiaWest']='https://dataserv.ub.tum.de/s/m1639953/download?path=/&files=s1_asiaWest.tar.gz'
|
|
vol_dict['multi_s1_asiaWest']='59218848'
|
|
;;
|
|
'europa') url_dict['multi_s1_europa']='https://dataserv.ub.tum.de/s/m1639953/download?path=/&files=s1_europa.tar.gz'
|
|
vol_dict['multi_s1_europa']='121213836'
|
|
;;
|
|
esac
|
|
|
|
|
|
# test split
|
|
case $region in
|
|
'all') url_dict['multi_s1_africa_test']='https://dataserv.ub.tum.de/s/m1659251/download?path=/&files=s1_africa_test.tar.gz'
|
|
vol_dict['multi_s1_africa_test']='15668120'
|
|
|
|
url_dict['multi_s1_america_test']='https://dataserv.ub.tum.de/s/m1659251/download?path=/&files=s1_america_test.tar.gz'
|
|
vol_dict['multi_s1_america_test']='15668160'
|
|
|
|
url_dict['multi_s1_asiaEast_test']='https://dataserv.ub.tum.de/s/m1659251/download?path=/&files=s1_asiaEast_test.tar.gz'
|
|
vol_dict['multi_s1_asiaEast_test']='24982736'
|
|
|
|
url_dict['multi_s1_asiaWest_test']='https://dataserv.ub.tum.de/s/m1659251/download?path=/&files=s1_asiaWest_test.tar.gz'
|
|
vol_dict['multi_s1_asiaWest_test']='9252904'
|
|
|
|
url_dict['multi_s1_europa_test']='https://dataserv.ub.tum.de/s/m1659251/download?path=/&files=s1_europa_test.tar.gz'
|
|
vol_dict['multi_s1_europa_test']='49040432'
|
|
;;
|
|
'africa') url_dict['multi_s1_africa_test']='https://dataserv.ub.tum.de/s/m1659251/download?path=/&files=s1_africa_test.tar.gz'
|
|
vol_dict['multi_s1_africa_test']='15668120'
|
|
;;
|
|
'america') url_dict['multi_s1_america_test']='https://dataserv.ub.tum.de/s/m1659251/download?path=/&files=s1_america_test.tar.gz'
|
|
vol_dict['multi_s1_america_test']='15668160'
|
|
;;
|
|
'asiaEast') url_dict['multi_s1_asiaEast_test']='https://dataserv.ub.tum.de/s/m1659251/download?path=/&files=s1_asiaEast_test.tar.gz'
|
|
vol_dict['multi_s1_asiaEast_test']='24982736'
|
|
;;
|
|
'asiaWest') url_dict['multi_s1_asiaWest_test']='https://dataserv.ub.tum.de/s/m1659251/download?path=/&files=s1_asiaWest_test.tar.gz'
|
|
vol_dict['multi_s1_asiaWest_test']='9252904'
|
|
;;
|
|
'europa') url_dict['multi_s1_europa_test']='https://dataserv.ub.tum.de/s/m1659251/download?path=/&files=s1_europa_test.tar.gz'
|
|
vol_dict['multi_s1_europa_test']='49040432'
|
|
;;
|
|
esac
|
|
fi
|
|
fi
|
|
|
|
|
|
# mono-temporal data (all regions)
|
|
if [ "$SEN12MSCR" = "true" ]; then
|
|
echo "Also downloading SEN12MS-CR data set."
|
|
mkdir -p $dl_extract_to'/SEN12MSCR'
|
|
url_dict['mono_s2_spring']='https://dataserv.ub.tum.de/s/m1554803/download?path=/&files=ROIs1158_spring_s2.tar.gz'
|
|
vol_dict['mono_s2_spring']='48568904'
|
|
|
|
url_dict['mono_s2_summer']='https://dataserv.ub.tum.de/s/m1554803/download?path=/&files=ROIs1868_summer_s2.tar.gz'
|
|
vol_dict['mono_s2_summer']='56425520'
|
|
|
|
url_dict['mono_s2_fall']='https://dataserv.ub.tum.de/s/m1554803/download?path=/&files=ROIs1970_fall_s2.tar.gz'
|
|
vol_dict['mono_s2_fall']='68291864'
|
|
|
|
url_dict['mono_s2_winter']='https://dataserv.ub.tum.de/s/m1554803/download?path=/&files=ROIs2017_winter_s2.tar.gz'
|
|
vol_dict['mono_s2_winter']='30580552'
|
|
|
|
url_dict['mono_s2_cloudy_spring']='https://dataserv.ub.tum.de/s/m1554803/download?path=/&files=ROIs1158_spring_s2_cloudy.tar.gz'
|
|
vol_dict['mono_s2_cloudy_spring']='48569368'
|
|
|
|
url_dict['mono_s2_cloudy_summer']='https://dataserv.ub.tum.de/s/m1554803/download?path=/&files=ROIs1868_summer_s2_cloudy.tar.gz'
|
|
vol_dict['mono_s2_cloudy_summer']='56426004'
|
|
|
|
url_dict['mono_s2_cloudy_fall']='https://dataserv.ub.tum.de/s/m1554803/download?path=/&files=ROIs1970_fall_s2_cloudy.tar.gz'
|
|
vol_dict['mono_s2_cloudy_fall']='68292448'
|
|
|
|
url_dict['mono_s2_cloudy_winter']='https://dataserv.ub.tum.de/s/m1554803/download?path=/&files=ROIs2017_winter_s2_cloudy.tar.gz'
|
|
vol_dict['mono_s2_cloudy_winter']='30580812'
|
|
|
|
# S1 data of SEN12MS-CR
|
|
if [ "$S1" = "true" ]; then
|
|
echo "Also downloading associated S1 data."
|
|
url_dict['mono_s1_spring']='https://dataserv.ub.tum.de/s/m1554803/download?path=/&files=ROIs1158_spring_s1.tar.gz'
|
|
vol_dict['mono_s1_spring']='15026120'
|
|
|
|
url_dict['mono_s1_summer']='https://dataserv.ub.tum.de/s/m1554803/download?path=/&files=ROIs1868_summer_s1.tar.gz'
|
|
vol_dict['mono_s1_summer']='17456784'
|
|
|
|
url_dict['mono_s1_fall']='https://dataserv.ub.tum.de/s/m1554803/download?path=/&files=ROIs1970_fall_s1.tar.gz'
|
|
vol_dict['mono_s1_fall']='21127832'
|
|
|
|
url_dict['mono_s1_winter']='https://dataserv.ub.tum.de/s/m1554803/download?path=/&files=ROIs2017_winter_s1.tar.gz'
|
|
vol_dict['mono_s1_winter']='9460956'
|
|
fi
|
|
fi
|
|
|
|
req=0
|
|
# integrate file size across archives
|
|
for key in "${!vol_dict[@]}"; do
|
|
# for each archive: sum up
|
|
curr=${vol_dict[$key]}
|
|
req=$((req+curr))
|
|
done
|
|
|
|
echo
|
|
echo
|
|
# df -h $dl_extract_to
|
|
avail=$(df $dl_extract_to | awk 'NR==2 { print $4 }')
|
|
if (( avail < req )); then
|
|
echo "Not enough space (512-byte disk sectors) on path "$dl_extract_to". Available "$avail". Required "$req #>&2
|
|
exit 1
|
|
else
|
|
echo "Consuming "$req" of "$avail" (512-byte disk sectors) on path "$dl_extract_to
|
|
fi
|
|
echo
|
|
echo
|
|
|
|
# download each archive individually, then extract individually
|
|
|
|
# fetch the actual data
|
|
for key in "${!url_dict[@]}"; do
|
|
url=${url_dict[$key]}
|
|
filename=$(basename "$url")
|
|
filename=${filename:7}
|
|
# download
|
|
wget --no-check-certificate -c -O $dl_extract_to'/'$filename ${url_dict[$key]}
|
|
# unzip and delete archive
|
|
tar --extract --file $dl_extract_to'/'$filename -C $dl_extract_to
|
|
rm $dl_extract_to'/'$filename
|
|
done
|
|
|
|
# move the extracted data to its respective place (this may take a while, because we use rsync rather than mv)
|
|
echo "Moving data in place, please don't stop this process."
|
|
for key in "${!url_dict[@]}"; do
|
|
url=${url_dict[$key]}
|
|
filename=$(basename "$url")
|
|
filename=${filename:7:-7} # remove base URL and trailing *.tar.gz
|
|
if [[ ${url_dict[$key]} == *"m1554803"* ]]; then
|
|
# move to SEN12MSCR directory
|
|
mv $dl_extract_to'/'$filename $dl_extract_to'/SEN12MSCR'
|
|
elif [[ ${url_dict[$key]} == *"m1639953"* ]]; then
|
|
# move train ROI to SEN12MSCRTS directory
|
|
no_prefix_filename=${filename:3}
|
|
rsync -a -remove-source-files $dl_extract_to'/'$no_prefix_filename/* $dl_extract_to'/SEN12MSCRTS' 2>/dev/null
|
|
rm -rf $dl_extract_to'/'$no_prefix_filename
|
|
else
|
|
# move test ROI to SEN12MSCRTS directory
|
|
rsync -a -remove-source-files $dl_extract_to'/'$filename/* $dl_extract_to'/SEN12MSCRTS'
|
|
rm -rf $dl_extract_to'/'$filename
|
|
fi
|
|
done
|
|
|
|
echo
|
|
echo "Completed downloading, extracting and moving data! Enjoy :)"
|