#########
# GDSC ETL is performed in two steps: shell script to ETL data into postgres
# and then additional transformations with SQL in postGIS (if needed).
####
#########
# Step 1 - bash script (pseudo code)
####
# create directory structure and move into it
mkdir -p /data/mdc_2013_2022_lst_30m/{download,mdc_2013_2022_avg_lst_30m/etl} && cd /data/mdc_2013_2022_lst_30m
# set update flag based on last update and update frequency
do_update=0 if date() < last_update + update_frequency else do_update = 1
# download if update flag is set:
if do_update == 1 then
wget -O download/mdc_2013_2022_lst_30m.zip 'https://scholarship.miami.edu/view/fileRedirect?instCode=01UOML_INST&filePid=13440658420002976&download=true'
# unzip to download directory if update flag is set
if do_update == 1 then
unzip -d download download/mdc_2013_2022_lst_30m.zip && rm download/mdc_2013_2022_lst_30m.zip
'
# load raster into postGIS with:
raster2pgsql -s 32617 -d -C -I LST_Avg_Scenes/AvgMDC_LST_2013_2022.tif -F -t 256x256 mdc_2013_2022_avg_lst_30m > load_raster.sql
psql < load_raster.sql
rm load_raster.sql
# create color raster
mkdir -p /tiles/raster/mdc_2013_2022_avg_lst_30m
mkdir -p /data/mdc_2013_2022_lst_30m/mdc_2013_2022_avg_lst_30m/derived
cd /data/mdc_2013_2022_lst_30m
cat > /data/mdc_2013_2022_lst_30m/mdc_2013_2022_avg_lst_30m/derived/mdc_2013_2022_avg_lst_30m_color_ramp.txt <<_EOF
33 127 39 4 255
31.20 166 54 3 255
29.04 217 72 1 255
26.70 241 105 19 255
24.36 253 141 60 255
22.02 253 174 107 255
19.68 253 208 162 255
17.34 254 230 206 255
15 255 245 235 255
0 0 0 0 0
_EOF
gdaldem color-relief download/LST_Avg_Scenes/AvgMDC_LST_2013_2022.tif /data/mdc_2013_2022_lst_30m/mdc_2013_2022_avg_lst_30m/derived/mdc_2013_2022_avg_lst_30m_color_ramp.txt /data/mdc_2013_2022_lst_30m/mdc_2013_2022_avg_lst_30m/derived/mdc_2013_2022_avg_lst_30m_color.tif -alpha
# create color raster and render raster tiles
mkdir -p /data/mdc_2013_2022_lst_30m/mdc_2013_2022_avg_lst_30m/derived
cd /data/mdc_2013_2022_lst_30m
echo "iVBORw0KGgoAAAANSUhEUgAAAFcAAADzCAYAAADzRYqUAAALXklEQVR4nO2dC0gU3x7Hf2vK3yy1x5/U0qLMSrQsNCLN0h5aiRjZG4sIrUv0Jq2gW12vlaXRG6KCoKSiLKN3ET3FXguRpteemr20LLWXj83O5ZzLLutrd1z3dzujvw8My+7MnNn9znd+85szs7+jYYwxIFCwwWmWIHGRIXERIXERIXERIXERIXERIXERIXERIXERIXERIXERIXERadPi6nQ6wOwUtEjc8ePHg0ajaXQqLi4GNfD7928YPHgwVFdXo21D09L+XHd3d0hLS4OQkBBQE79+/QI7OzuorKwEe3t79YSFa9euwaBBg8DZ2RmmTp0KZWVl4nOtVguhoaEwf/58cHBwgODgYMjOzgZ/f39wdHSEhIQEw3IjRoyAWbNmQfv27SEoKAhevXpltv179+7BsGHDYPjw4WKnc+G2bt0K/fv3h7/++gu8vb3h9u3bYlnePoe3f/bsWfD19TW0n5mZKVzdVJtNbb8BrIX06NGD3bhxw/D+3bt3zNHRkWVkZLDy8nK2ePFiNmvWLDHv4cOHTKPRsJ07d7KvX7+ywMBA5u7uznJyctjjx4+Zvb09e/v2rWG5xMREVlZWxpYuXcqCg4PNtn/37l1+FLLz58+zkpISlpWVxXr37s0KCwvZjx8/2MqVK1lISIhYVqfTiWUrKyvZnTt3mI+Pj+E38Pd+fn6Ntmlq+/Wxurh79uxhkZGRhvf8R9nZ2bHq6mohWteuXQ3zVq9ezWJjYw3vvb292f3798Vybm5urLa21tCGra0t+/jxo8n2uRDG7RvD101NTWW+vr7NFte4TVPbr4+ttUPC27dv4dy5c+LkZkxRUZF47dq1q+Gzdu3aiUNLj42NjTjR8NdevXqJVw4PIZ06dYJPnz6Zbd/FxaVONrBixQo4fvy4iLGenp4WZQfGbZraft++fXFjrqurK8yePVv8CONJv+H6X6opjLOOHz9+QHl5Obi5uTWr/QMHDkBWVhY8ePBAxMUNGzY0ui2+E/mO0FNaWlpnvnGb5rZfp12wMpMmTRIniMuXL8PXr1/h4MGD4OHhUefLK6GwsBD27t0LFRUVsGbNGhg7dix07ty5We1zQfmJkh8tz549E+LqUy9bW1sh6sePH8VRUlBQANevX4eSkhLYsmWLVX6f1cXlX/TYsWMQHx8v9vL+/fvhzJkzIu1pDr179xau42fop0+fwqFDh5rd/oIFC4TruONnzJghshF+WH/79k3M5595eXmJHbh582aYOXMmBAQEiCzFGr+vxXkuBlqtFmJiYiA/Px/UTJu+/MVGSue2Fsi5iJC4rUHczMzMOtfvTZ3IGssX1YpUzg0ICIAXL14AtHVxT58+Df369YMOHTrAxIkT4fPnz+JzJT1GOp0O5s6dK5bp06cPbNq0qYFz6zvduKeqqfVbhbj8ambOnDmwY8cOePPmDXTv3h1WrVoF79+/h8mTJ0NiYqK41ubJ+6JFixqsn56eLpJ5vi7fGbt374bc3FzF22/p+lKLe+HCBRg3bpxwbJcuXcQlIJ8yMjJEpzm/RHR2dobk5GQ4efIk1NTU1Fm/Y8eOkJeXB0ePHhX9rB8+fAAfHx/F22/p+lKLy3un+PV0fYx7jDQajQgZ/BDW91jpiYyMhGXLlgnxeTt8JzXZ4dwILV1fanH54f7u3bsGnzenxyghIUF0zvA7EbwX6vDhw83qqTK3vmrFjYiIELGOT7wrMCkpSbhJaY9RSkoKREVFCbfxHcI7PfihXr/jpqmeKiXrSwGzkLNnz7IBAwYwBwcHFhYWJm7PcC5evCh6+9u3b8+GDh3KtFqtoXdf39vPe//5rREnJydxyyQuLo7V1NSIOxCenp6GbfA7B926dRO3gnbt2mW4O9DU+rJBfQtt5SKitUHiIkLiIkLiIkLiIkLiIkLiIkLiIkLiIkLiIkLitmVx9+3bJ3rWeK/XvHnzoKqqStE8KWASo9VqmYeHB3v06BErLS1loaGhLCkpyew8WbD687nWxN/fv85dDH5/7tatW2bnyYL0YUEPF5L/sSU8PLxZ8/4kFjl3w4DmPQ5qjg35pp/d5U+GL168WDxhHhgYqHjen8aizvJ/DbBuNFmf/0vRcvwZ3bVr1wqn8kf+lc5Tlbj/9rauuP/8jzJxf/78Ke4o8xuSxv+tMDdPVTHXRmPdqSlOnTol/jv25MkT8TQ4f+x+yJAhQjxT82TBVuazYHR0tHiSJiwsDL5//w4jR44UD56YmycLFoWFLT7WDQurcpWFBbVhmXOV/dupzUPiyiYuaasMci4iJG5bTcXUDjkXERIXERIXEUrFECHnIkLiIkKpGCLkXERIXERIXEQofMp3QqN/VymBwoJs4rYjcRVBzkWEnIsIORcRci4i1LeACIUFRCgsyOdcukJTAjkXEYq5iJBzEaFUDBEKC4hQWECEnIsIORcRci4i5FxEKKuS7ilHci6iuBR0MZ1L4iqBnIsIORcRcq50FxEkriIoFUOEwgIidEKT7fKXO9eakyk2btwoSgry4kBjxoyBly9fis/5oEv6YWqMJz6EjbrFbeRHtWRqiitXrsCRI0fg5s2bYnxePz8/MZoVh4/NYzxEzcKFCyE2NlYMZavqGjfFsx2t+iVcj3xTtt3iYujZs6cYGNl4p/Cx0qZNmybGB+aDKMuCqlKxrKwsGDhwYAO387HY1q1bJ5WwqkrFioqKYOnSpYaRq/XwcdT5UN98ZD/ZUEUqVlBQIIpgbt26VYy5bgyPydOnTwd7e3uQDalPaBw+JPj48eNh+/btYiz0+ly6dEkMFyYjUjv3y5cvQljuzuDg4Abz+XhoPDXjA4rKiNQXEdu2bYPXr1+LUoLG8NqNvJQ2H8mPj3XJRxaUEYtSsfJ/WLcYZad9/xvWtrWhqlRMbdDdX0QsKydKHTeY2QIZHs+5JC5mKkbiKoGciwjFXERIXPlSMRJXCeRcROiEhgilYoiQcxGhmIsIiYsIpWKIkHMRoRMaIpSKIULORYRiLiIkLiKkEiKU5yJCJzREyLmIkHPlc64cY5nLDjkXERIXETqhyedcirlKIOciQjEXEXIuIhRzEaGwgAiFBUTIuYhQ3wIi5FxEKOYiQs5FhPJcRMiCiFDMRYTCAiI2FjvXmpMFFfGMiY+Ph+TkZGgd4nLnWnOyoCIep6qqClasWAGpqakgI1KHhfDwcFGdSU9CQoKoiMcrx/A6O3FxcaI6XkxMDMiI9GHBVEW8lJQUOHHihCgiJCNSO9dcRTxXV1eQGVWkYgUmKuLJjPTOzc/Ph6ioKFERLyIiAtSE1OJ+MVMRT3akDgvbzFTEkx2LKuL9vp1i1S9hMzIeWiNShwW1I3VYUDvkXETIuYiQcxEhcRGhsIAIORcREhcROr4RoZiLCImLCImLCImLCImLCImLCImLCImLCImLCImLCP2bBxFyLiIkLiIkLiIkLiIkLiIkLiIkLiIkLiIkrnyVn+kKTQlkQenCAtC4v0qgQZURoRMaIhQWEKGwgAiFBUTIufLFXHKuEsi5iJC4iFBYQISciwiJiwjluYjQ5S8i5FxEKOYiYqEFNVaezNNYScElS5aIGmP6KSAgAGRC+qJtVSZKCubl5cHVq1dFhTw+abVakAnpw0KciZKCXFxeIU9WbCwdDtyakymaKilYXl4OpaWlonAmr9TEKzg9ffoU1O9ch7/h/4VrEyUFc3NzwcHBAdavXy/cm5iYCNHR0ZCTk2Oo9finUW1OFRQUJNzLX52cnMTJ7vnz56LmoyyoVtz66HQ6qK2tBXt7e5AF1Yqbnp4OXl5ekJ2dDRUVFbB8+XIYNWoUuLi4gMovf/88U6ZMEdnChAkTRM3H0aNHQ1paGqi+3CDRysOCGiBxESFxESFxESFxESFxESFxAY//Ak2Kn/xLIPyaAAAAAElFTkSuQmCC" | base64 -d > /data/mdc_2013_2022_lst_30m/mdc_2013_2022_avg_lst_30m/derived/mdc_2013_2022_avg_lst_30m_legend.png
cp /data/mdc_2013_2022_lst_30m/mdc_2013_2022_avg_lst_30m/derived/mdc_2013_2022_avg_lst_30m_legend.png /tiles/raster/mdc_2013_2022_avg_lst_30m/
# render raster tiles
cd /data/mdc_2013_2022_lst_30m/mdc_2013_2022_avg_lst_30m/
gdal2tiles.py -ex --xyz -z 8-16 /data/mdc_2013_2022_lst_30m/mdc_2013_2022_avg_lst_30m/derived/mdc_2013_2022_avg_lst_30m_color.tif /tiles/raster/mdc_2013_2022_avg_lst_30m
# Create derivative directory in data package on osgeo
cd /data/mdc_2013_2022_lst_30m/mdc_2013_2022_avg_lst_30m/
mkdir -p derived
# Create pg_dump of table SQL in derivate directory on postgis.
pg_dump -d $POSTGRES_DB -U $POSTGRES_USER -p $POSTGRES_PORT -h postgis-na4vxyii8b68i518 -t mdc_2013_2022_avg_lst_30m > derived/mdc_2013_2022_avg_lst_30m.sql
Create downloadable tarfile of SQL in derivative directory on postgis
tar -czf derived/mdc_2013_2022_avg_lst_30m.sql.tar.gz meta_dcat_mdc_2013_2022_avg_lst_30m.json -C derived mdc_2013_2022_avg_lst_30m.sql