Joerd data

Dockerfile

@@ -6,6 +6,9 @@ MAINTAINER Nils Nolde <nils@openrouteservice.org>
 
 RUN apt-get update
 RUN apt-get install -y locales git python3-venv
+RUN apt-get install -y build-essential libssl-dev libffi-dev python3-dev
+RUN apt-get install -y libpq-dev
+RUN apt-get install -y git curl
 
 # Set the locale
 RUN locale-gen en_US.UTF-8
@@ -32,13 +35,24 @@ RUN mkdir -p /deploy/app
 COPY gunicorn_config.py /deploy/gunicorn_config.py
 COPY manage.py /deploy/app/manage.py
 
-COPY requirements.txt /deploy/app/requirements.txt
-
 RUN python3 -m venv /oes_venv
 
 RUN /bin/bash -c "source /oes_venv/bin/activate"
 
-RUN /oes_venv/bin/pip3 install -r /deploy/app/requirements.txt
+# install poetry
+RUN curl -sSL https://raw.githubusercontent.com/sdispater/poetry/master/get-poetry.py | python3
+ENV PATH "/root/.poetry/bin:/oes_venv/bin:${PATH}"
+
+# install dependencies via poetry
+RUN poetry config settings.virtualenvs.create false
+RUN poetry self:update --preview
+RUN poetry config virtualenvs.create false
+COPY pyproject.toml poetry.lock README.rst /
+RUN poetry install
+
+RUN apt-get install -y python-gdal python-bs4 python-numpy gdal-bin python-setuptools python-shapely
+RUN apt-get install -y libgdal-dev python3-dev
+RUN /oes_venv/bin/pip3 install GDAL==$(gdal-config --version) --global-option=build_ext --global-option="-I/usr/include/gdal"
 
 COPY openelevationservice /deploy/app/openelevationservice
 COPY ops_settings_docker.yml /deploy/app/openelevationservice/server/ops_settings.yml

docker-compose.yml

@@ -1,10 +1,11 @@
 version: '2.2'
 services:
  gunicorn_flask:
-  #network_mode: "host"
+  network_mode: "host"
   build: .
   volumes:
-  - ./tiles:/deploy/app/tiles
+   - ./tiles:/deploy/app/tiles
+   - ./ops_settings_docker.yml:/deploy/app/openelevationservice/server/ops_settings.yml:rw
   ports:
    - "5020:5000"
   mem_limit: 28g

manage.py

@@ -6,24 +6,22 @@
 from openelevationservice.server.db_import.models import db
 from openelevationservice.server.db_import import filestreams
 
-import click
-
 log = get_logger(__name__)
 
 app = create_app()
 
+
 @app.cli.command()
-@click.option('--xyrange', default='0,73,0,25')
-def download(xyrange):
+def download():
     """
     Downloads SRTM tiles to disk. Can be specified over minx, maxx, miny, maxy.
 
     :param xyrange: A comma-separated list of x_min, x_max, y_min, y_max
         in that order. For reference grid, see http://srtm.csi.cgiar.org/SELECTION/inputCoord.asp
     :type xyrange: comma-separated integers
     """
-    
-    filestreams.downloadsrtm(_arg_format(xyrange))
+
+    filestreams.downloadsrtm()
     log.info("Downloaded all files")
 
 
@@ -32,16 +30,17 @@ def create():
     """Creates all tables defined in models.py"""
 
     db.create_all()
-    log.info("Table {} was created.".format(SETTINGS['provider_parameters']['table_name']))
+    log.info("Table {} was created.".format(SETTINGS['provider_parameters']['table_name_srtm']))
+    log.info("Table {} was created.".format(SETTINGS['provider_parameters']['table_name_composite']))
 
 
 @app.cli.command()
 def drop():
     """Drops all tables defined in models.py"""
 
     db.drop_all()
-    log.info("Table {} was dropped.".format(SETTINGS['provider_parameters']['table_name']))
-
+    log.info("Table {} was created.".format(SETTINGS['provider_parameters']['table_name_srtm']))
+    log.info("Table {} was created.".format(SETTINGS['provider_parameters']['table_name_composite']))
 
 @app.cli.command()
 def importdata(): 
@@ -66,4 +65,4 @@ def _arg_format(xy_range_txt):
     xy_range = [[str_split[0], str_split[2]],
                 [str_split[1], str_split[3]]]
 
-    return xy_range
+    return xy_range

openelevationservice/__init__.py

@@ -9,7 +9,7 @@
 TILES_DIR = path.join(getcwd(), 'tiles')
 
 if "TESTING" in environ:
-    SETTINGS['provider_parameters']['table_name'] = SETTINGS['provider_parameters']['table_name'] + '_test'
+    SETTINGS['provider_parameters']['table_name_srtm'] = SETTINGS['provider_parameters']['table_name_srtm'] + '_test'
     TILES_DIR = path.join(basedir, 'tests', 'tile')
     # if "CI" in environ:
     #     SETTINGS['provider_parameters']['port'] = 5433

openelevationservice/server/__init__.py

@@ -37,7 +37,8 @@ def create_app(script_info=None):
     log.info("Following provider parameters are active:\n"
               "Host:\t{host}\n"
               "DB:\t{db_name}\n"
-              "Table:\t{table_name}\n"
+              "Table1:\t{table_name_srtm}\n"
+              "Table2:\t{table_name_composite}\n"
               "User:\t{user_name}".format(**provider_details))
 
     # register blueprints

openelevationservice/server/db_import/filestreams.py

@@ -2,110 +2,114 @@
 
 from openelevationservice import TILES_DIR, SETTINGS
 from openelevationservice.server.utils.logger import get_logger
+from openelevationservice.server.sources.gmted import GMTED
+from openelevationservice.server.sources import srtm
+from openelevationservice.server.db_import import raster_processing
 
 from os import path, environ
-import requests
 import subprocess
-import zipfile
-from bs4 import BeautifulSoup
 
-try:
-    from io import BytesIO
-except:
-    from StringIO import StringIO
-
 log = get_logger(__name__)
 
-def downloadsrtm(xy_range):
+
+def downloadsrtm():
     """
-    Downlaods SRTM v4.1 tiles as bytestream and saves them to TILES_DIR.
-
-    :param xy_range: The range of tiles in x and y as per grid in 
-        http://srtm.csi.cgiar.org/SELECTION/inputCoord.asp
-        in 'minx, maxx, miny, maxy.
-    :type xy_range: comma-separated range string
+    Downlaods GMTED and SRTM v4.1 tiles as bytestream and saves them to TILES_DIR.
     """
-
-    base_url = r'http://data.cgiar-csi.org/srtm/tiles/GeoTIFF/'
-
-    # Create session for authentication
-    session = requests.Session()
-
-    pw = environ.get('SRTMPASS')
-    user = environ.get('SRTMUSER')
-    if not user and not pw:
-        auth = tuple(SETTINGS['srtm_parameters'].values())
+
+    extent_settings = SETTINGS['tables'][0]['srtm']['extent']
+    if extent_settings['max_y'] <= 60:
+
+        # only SRTM data download
+        srtm.download_srtm()
+
+        # merge and clip raster by extent
+        log.info("Starting tile processing ...")
+        raster_processing.merge_raster('srtm_*', '/srtm_merged.tif')
+        raster_processing.clip_raster('/srtm_merged.tif', '/srtm_final.tif')
+
+    elif extent_settings['max_y'] > 60 >= extent_settings['min_y']:
+
+        # SRTM and GMTED data download
+        srtm.download_srtm()
+
+        tiles_list = GMTED().tile_selection()
+        GMTED().download_gmted(tiles_list)
+
+        # resample and merge tiles
+        log.info("Starting tile preprocessing ...")
+        raster_processing.merge_raster('srtm_*', '/srtm_merged.tif')
+        raster_processing.clip_raster('/srtm_merged.tif', '/srtm_clipped.tif')
+        raster_processing.merge_raster('*_gmted_mea075.tif', '/gmted_merged.tif')
+
+        log.info("Starting tile resampling ...")
+        raster_processing.gmted_resampling()
+
+        log.info("Starting tile merging ...")
+        raster_processing.merge_raster('gmted_resampled.tif', '/raster_final.tif', '/srtm_clipped.tif')
+
     else:
-        auth = tuple([user,pw])
-    session.auth = auth
-
-    log.debug("SRTM credentials: {}".format(session.auth))
-
-    response = session.get(base_url)
-
-    soup = BeautifulSoup(response.content, features="html.parser")
-
-    # First find all 'a' tags starting href with srtm*
-    for link in soup.find_all('a', attrs={'href': lambda x: x.startswith('srtm') and x.endswith('.zip')}):
-        link_parsed = link.text.split('_')
-        link_x = int(link_parsed[1])
-        link_y = int(link_parsed[2].split('.')[0])
-        # Check if referenced geotif link is in xy_range
-        if link_y in range(*xy_range[0]) and link_x in range(*xy_range[1]):
-            log.info('yep')
-            # Then load the zip data in memory
-            if not path.exists(path.join(TILES_DIR, '_'.join(['srtm', str(link_x), str(link_y)]) + '.tif')):
-                with zipfile.ZipFile(BytesIO(session.get(base_url + link.text).content)) as zip_obj:
-                    # Loop through the files in the zip
-                    for filename in zip_obj.namelist():
-                        # Don't extract the readme.txt
-                        if filename != 'readme.txt':
-                            data = zip_obj.read(filename)
-                            # Write byte contents to file
-                            with open(path.join(TILES_DIR, filename), 'wb') as f:
-                                f.write(data)        
-                log.debug("Downloaded file {} to {}".format(link.text, TILES_DIR))
-            else:
-                log.debug("File {} already exists in {}".format(link.text, TILES_DIR))
-
+
+        # only GMTED data download
+        tiles_list = GMTED().tile_selection()
+        GMTED().download_gmted(tiles_list)
+
+        # merge and clip raster by extent
+        log.info("Starting tile processing ...")
+        raster_processing.merge_raster('*_gmted_mea075.tif', '/gmted_merged.tif')
+        raster_processing.clip_raster('/gmted_merged.tif', '/gmted_final.tif')
+
 
 def raster2pgsql():
     """
     Imports SRTM v4.1 tiles to PostGIS.
 
     :raises subprocess.CalledProcessError: Raised when raster2pgsql throws an error.
     """
-    
+
     pg_settings = SETTINGS['provider_parameters']
-    
+
     # Copy all env variables and add PGPASSWORD
     env_current = environ.copy()
     env_current['PGPASSWORD'] = pg_settings['password']
-
-    # Tried to import every raster individually by user-specified xyrange 
-    # similar to download(), but raster2pgsql fuck it up somehow.. The PostGIS
-    # raster will not be what one would expect. Instead doing a bulk import of all files.
-    cmd_raster2pgsql = r"raster2pgsql -s 4326 -a -C -M -P -t 50x50 {filename} {table_name} | psql -q -h {host} -p {port} -U {user_name} -d {db_name}"
-    # -s: raster SRID
-    # -a: append to table (assumes it's been create with 'create()')
-    # -C: apply all raster Constraints
-    # -P: pad tiles to guarantee all tiles have the same width and height
-    # -M: vacuum analyze after import
-    # -t: specifies the pixel size of each row. Important to keep low for performance!
-
-    cmd_raster2pgsql = cmd_raster2pgsql.format(**{'filename': path.join(TILES_DIR, '*.tif'),
-                                                  **pg_settings})
-
+
+    tiles_dir_name = None
+
+    if SETTINGS["sources"][0]["type"] == "cgiar_csi":
+
+        # Tried to import every raster individually by user-specified xyrange
+        # similar to download(), but raster2pgsql fuck it up somehow.. The PostGIS
+        # raster will not be what one would expect. Instead doing a bulk import of all files.
+        cmd_raster2pgsql = r"raster2pgsql -s 4326 -a -C -M -P -t 50x50 {filename} {table_name_srtm} | psql -q -h {host} -p {port} -U {user_name} -d {db_name}"
+        # -s: raster SRID
+        # -a: append to table (assumes it's been create with 'create()')
+        # -C: apply all raster Constraints
+        # -P: pad tiles to guarantee all tiles have the same width and height
+        # -M: vacuum analyze after import
+        # -t: specifies the pixel size of each row. Important to keep low for performance!
+
+        tiles_dir_name = TILES_DIR
+
+    else:
+        cmd_raster2pgsql = r"raster2pgsql -s 4326 -a -C -M -P -t 50x50 {filename} {table_name_joerd} | psql -q -h {host} -p {port} -U {user_name} -d {db_name}"
+
+        tiles_dir_name = TILES_DIR
+
+    if path.join(TILES_DIR, '*.tif'):
+        cmd_raster2pgsql = cmd_raster2pgsql.format(**{'filename': path.join(tiles_dir_name, '*.tif'), **pg_settings})
+    else:
+        cmd_raster2pgsql = cmd_raster2pgsql.format(**{'filename': path.join(tiles_dir_name, '*.img'), **pg_settings})
+
     proc = subprocess.Popen(cmd_raster2pgsql,
-                         stdout=subprocess.PIPE, 
-                         stderr=subprocess.STDOUT,
-                         shell=True,
-                         env=env_current
-                         )
-    
+                            stdout=subprocess.PIPE,
+                            stderr=subprocess.STDOUT,
+                            shell=True,
+                            env=env_current
+                            )
+
 #    for line in proc.stdout:
 #        log.debug(line.decode())
 #    proc.stdout.close()
     return_code = proc.wait()
     if return_code:
-        raise subprocess.CalledProcessError(return_code, cmd_raster2pgsql)
+        raise subprocess.CalledProcessError(return_code, cmd_raster2pgsql)

openelevationservice/server/db_import/models.py

@@ -10,15 +10,29 @@
 db = SQLAlchemy()
 
 log = logger.get_logger(__name__)
-table_name = SETTINGS['provider_parameters']['table_name']
+table_name_srtm = SETTINGS['provider_parameters']['table_name_srtm']
+table_name_composite = SETTINGS['provider_parameters']['table_name_composite']
+
 
 class Cgiar(db.Model):
     """Database model for SRTM v4.1 aka CGIAR dataset."""
-    
-    __tablename__ = table_name
-    
+
+    __tablename__ = table_name_srtm
+
     rid = db.Column(db.Integer, primary_key=True)
     rast = db.Column(Raster)
-
+
+    def __repr__(self):
+        return '<rid {}, rast {}>'.format(self.rid, self.rast)
+
+
+class Joerd(db.Model):
+    """Database model for SRTM v4.1 aka CGIAR dataset."""
+
+    __tablename__ = table_name_composite
+
+    rid = db.Column(db.Integer, primary_key=True)
+    rast = db.Column(Raster)
+
     def __repr__(self):
-        return '<rid {}, rast {}>'.format(self.rid, self.rast)
+        return '<rid {}, rast {}>'.format(self.rid, self.rast)