#########
# GDSC ETL is performed in two steps: shell script to ETL data into postgres
# and then additional transformations with SQL in postGIS (if needed).
####
#########
# Step 1 - bash script (pseudo code)
####
# create directory structure and move into it
mkdir -p /data/mdc_2015_dem_30ft/{download,etl} && cd /data/mdc_2015_dem_30ft
# set update flag based on last update and update frequency
do_update=0 if date() < last_update + update_frequency else do_update = 1
# load mdc_2015_dem_10ft as dependency (see mdc_2015_dem_10ft for details).
# resample raster with:
gdalwarp -tr 30 30 /data/mdc_2015_dem_10ft/download/dem_10ft/w001000.adf download/mdc_2015_dem_30ft.tif
#########
# Step 2 - SQL script (pseudo code):
####
# load raster into postGIS with:
raster2pgsql -s 2236 -d -C -I -t 100x100 mdc_2015_dem_30ft.tif -F mdc_2015_dem_30ft > load_raster.sql
psql < load_raster.sql
rm load_raster.sql
# create color raster
mkdir -p /tiles/raster/mdc_2015_dem_30ft
mkdir -p /data/mdc_2015_dem_30ft/derived
cd /data/mdc_2015_dem_30ft
cat > /data/mdc_2015_dem_30ft/derived/mdc_2015_dem_30ft_color_ramp.txt <<_EOF
nv 0 0 0 0
22.324 255 255 255 255
18.136 219 178 142 255
14.318 208 179 124 255
10.501 219 200 161 255
6.684 250 247 187 255
2.867 210 231 171 255
-0.30 221 226 183 255
-0.91 159 207 251 255
_EOF
gdaldem color-relief download/mdc_2015_dem_30ft.tif /data/mdc_2015_dem_30ft/derived/mdc_2015_dem_30ft_color_ramp.txt /data/mdc_2015_dem_30ft/derived/mdc_2015_dem_30ft_color.tif -alpha
# create color raster and render raster tiles
mkdir -p /data/mdc_2015_dem_30ft/derived
mkdir -p /tiles/raster/mdc_2015_dem_30ft
cd /data/mdc_2015_dem_30ft
echo "iVBORw0KGgoAAAANSUhEUgAAAF0AAACsCAYAAAAZv6B/AAAIiElEQVR4nO2da0gUXRiA39201mw1lUDBbiiYZZaVf8KkGwWlFd2JJP9YdPtRKAVBBBVEGgUZ34+NiCQwibbLR24XKCokzH9Rn5nhbiiKRuulvK15Ps7ha1HRcZ3pm3fOzPvA5O6cYyvPvvvO2XfOmbExxhgQumLX9+UIko4ESUeApCNA0hEg6QiQdARIOgIkHQGSjgBJR4CkI0DSEZBOeiAQANkLo39c+ps3byAtLQ3+DwYHB2Hx4sXQ19cHNTU1kJycDDIiVaQPDg7Cx48fxeNly5ZBfX09WEr68+fPIT09HaKjo2HHjh3g9/sn1G/btm1QWloa7Hfjxg3YsmWLeHzx4kVISUmBKVOmQGpqKrx69Ursz8rKEj8jIiLgyZMnwyL94cOHMH/+fIiKioLc3FxoamoS+/knIjMzU7z29OnTxSfl/fv3gApTQVNTE3M6ncztdrP29nZ29OhRtmfPHtH2+vVrtmDBgnH7lZeXs1WrVgX/zw0bNrDbt2+zqqoqNnfuXOb1etnPnz9ZYWEhW7lypegTCAR4Mmc9PT3s3bt3LCkpSez//Pkzi4mJYR6Ph/n9fnbw4EGWnZ0t2ng//jtXr14Vf8OhQ4dYTk4Ow0SV9NLSUpabmxt8zuWEh4ezvr6+YdKV+v348YNFRUWxb9++sY6ODhYdHc26urqGvU5raysrKSlhaWlpitJ5n7y8vODvdXd3s8mTJ4s3nfeLjY0NtvE3NTU1lWESpubT0djYCI8ePQKbzTZs/9evX0Pul5ycDOvXr4cHDx6INLJmzRqYNm2aGJ0cP34cysvLYWBgAJKSksYdrfDXmT17dvA5Tz8zZswQKYa/Nn/8m/DwcHFskC6nx8fHQ15enpAxdBs5mhiv386dO0Uudrvd4jHH5XJBVVUVVFdXi/x/5syZkP4en88XfN7d3Q1tbW3DZBsJVdL5AY/L8ng80NnZCdevX4eZM2eKKJ1Iv40bNwq5fJiZk5Mj9nHRTqcT4uLioK6uTkjnQ0ROWFgY2O12aG1tHRatW7duhfv378PTp0+ho6MDCgsLxbB1zpw5YEjU5qXHjx+LXBsREcEyMzNZTU2N2D80pyv1+83u3bvZ9u3bg8/b2trEgXPq1KksIyOD3blzhzkcDtbZ2Sna+YGY5+uKiopgTufcu3ePzZs3T7zOunXrWENDg9jPc3pKSkqw38jnGNj4P9hvvNWQ6suRWSDpCJB0q0kPmKBiqKt0/gXG4XDAvn37NFcMrYbq0QsvLL18+VKI42PniTIwMCC+Hfb09Ig3z0qoivTExETo6uqCpUuXii8/ShXHsdqyhlQMW1pawFKoHeBHRkay5uZmxUqiUltgSPHKamiWrlRJVGoLWFi6qipjqJVEpbY5Rq2LyDBkVKokhlqNtBxa0ws/w8NPQFRWVoqTES6XiyUmJrL+/n7FNo7dbmc+n4/9+vWLWQnN0serJCq17fmvYvjhwwdmJajKiADVXhAg6QiQdARIOgIkHQGSjgBJR4CkI0DSESDpCJB0BKSVfv78eTEvMjIyUsz4/fLlS0hthoBJiMfjEfMR6+vrxTz3Y8eOseXLl4/bZhRMUWVsaWmBWbNmiekcI89SKbVhIW16GQqfz75w4cJRpSq1ocEkx+fzibNRz549m1CbdCemf/X3gB5Mmhyh2N7Q0CCW0PDVeGvXrg25DRt10gO9gC29trYWNm/eDJcvXxYrOkJtMwKqDqS9/mbQA0dMwqj7v3//DkuWLIGysjJYsWJFyG1GwdCRPhaXLl0SC7uys7OH7edT/ZTa+Oo9aSO9q/Ef0ANnYiqYESkj3ZLSB/tJuhYo0hFQOU4n6fqnF4p0TVCkI0CRjgBFOgIU6QiYop5uiUhngHtlIItKl/4MHyokHQFKLwioXEdKka4FSi8IkHQEKKcjQJGOAB1IEaD0ggClFwRIujQ53UbStUClXWmk2+z6bCFQVFQEFy5cGLXtxYsXEBsbC6aQbrNP0mVTore3V1yQs6SkZNR2fuH6/fv3o98V4M9Jt9l12ZQoKCgQF2bbu3fvqO2nTp2C1atXgxGRNtKLi4uhoqJCXGRzJG/fvhV3H+Cpx0SjF2UhehAfHz/qfr6g68CBA3Dr1i1xOwcjouqvsqm4tq5enD17FjZt2gSLFi0Cr9cL5pFugEgfi7t378KnT5/g3LlzwX18ZV1zc/OYnw5JIt240mtra4OPeaTzq163t7eDCSLduOlFBlQtf6l9/RfowbwVB8GMmC6ny4DpRi8yQJGOgOlGLzJAeUKWSLdPMtBlPKwjnT4g+ksPI+laoEiXRfokkq4JSi8IUHpBgIaMCFB6QYAOpAhQTkeA0gsCFOnS5HSKdE2QPWmu96LTpMxwMCfqpMt/yXVUSDoCJB0Bki7L6IVL12MbiytXrohJoSM3vkjAtNPqmtv/Bj1ImJ4TUr/Dhw9Df38/uFwukAFDDxlDga+4cLvdYnq0LEif00+cOAGnT58Gp9MJsiC19Orqaqirq4P8/HyQCamll5WVwa5du8DhcIBMSC29srISrl27BrIhrXS/3y9uBJieng7WuNauAaR7vV6xhjQhYfTb8hgZaUu7GRkZhlvAFSrqVrfiB7oFFwUAzXvRXzoj6VqgSEdA5RULSLoWKL0gQOlFHumEFii9IEAHUgQopyNA6QUBinQESDoC0pZ2LRfpgUFjXu9QFlTZG2BmncRs6Egn6VqgSJcm0kk6QqRTetEEpRd50gsNGbVA6QUBOpAiYLqcXlRUBHFxcXDy5EkwVe2FfznSY5sI490NxkiYJtILCgrE/TDGuhuMkTCN9OLiYnH7hSNHjoAppecvNt6QMd4g97swbT09Pz8/uHZUtvVGHOOFbAjcvHlTbLIiZaTLDkmXZZk6oQ2KdARIOgIkHQGSjgBJR4CkI0DSQX/+BfVvODpN4X9iAAAAAElFTkSuQmCC" | base64 -d > /data/mdc_2015_dem_30ft/derived/mdc_2015_dem_30ft_legend.png
cp /data/mdc_2015_dem_30ft/derived/mdc_2015_dem_30ft_legend.png /tiles/raster/mdc_2015_dem_30ft/
# render raster tiles
cd /data/mdc_2015_dem_30ft/
gdal2tiles.py -ex --xyz -z 8-16 /data/mdc_2015_dem_30ft/derived/mdc_2015_dem_30ft_color.tif /tiles/raster/mdc_2015_dem_30ft
# Create derivative directory in data package on osgeo
cd /data/mdc_2015_dem_30ft/
mkdir -p derived
# Create pg_dump of table SQL in derivate directory on postgis.
pg_dump -d $POSTGRES_DB -U $POSTGRES_USER -p $POSTGRES_PORT -h postgis-vtc29jryy1rbxz07 -t mdc_2015_dem_30ft > derived/mdc_2015_dem_30ft.sql
Create downloadable tarfile of SQL in derivative directory on postgis
tar -czf derived/mdc_2015_dem_30ft.sql.tar.gz meta_dcat_mdc_2015_dem_30ft.json -C derived mdc_2015_dem_30ft.sql