From 64af7cd2b39210de5815bab649b910710055fac1 Mon Sep 17 00:00:00 2001 From: Isabell Klipper Date: Wed, 2 Oct 2019 16:27:52 +0200 Subject: [PATCH 01/11] add dockerfile to download joerd data sources --- docker-compose.yml | 5 +- joerd_docker/Dockerfile | 20 +++ .../server/db_import/filestreams.py | 164 +++++++++++------- .../server/ops_settings.sample.yml | 14 -- 4 files changed, 129 insertions(+), 74 deletions(-) create mode 100644 joerd_docker/Dockerfile delete mode 100644 openelevationservice/server/ops_settings.sample.yml diff --git a/docker-compose.yml b/docker-compose.yml index 4d77e38..5ec9d6e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,4 +7,7 @@ services: - ./tiles:/deploy/app/tiles ports: - "5020:5000" - mem_limit: 28g \ No newline at end of file + mem_limit: 28g + joerd_docker: + build: + context: ./joerd_docker \ No newline at end of file diff --git a/joerd_docker/Dockerfile b/joerd_docker/Dockerfile new file mode 100644 index 0000000..f256a79 --- /dev/null +++ b/joerd_docker/Dockerfile @@ -0,0 +1,20 @@ +# joerd + +# FROM python:3.6 +FROM ubuntu:18.04 + +RUN apt-get update +RUN apt-get install sudo +RUN apt-get install -y git + +# clone joerd_docker from git and install +RUN sudo git clone https://github.com/tilezen/joerd.git +WORKDIR /joerd +RUN sudo apt-get install -y python-gdal python-bs4 python-numpy gdal-bin python-setuptools python-shapely +RUN python setup.py install + +# copy ops_settings.yml from host into docker container +COPY ./openelevationservice/server/ops_settings.yml ./ops_settings.yml + +# download data via joerd_docker +RUN joerd enqueue-downloads --config ops_settings.yml \ No newline at end of file diff --git a/openelevationservice/server/db_import/filestreams.py b/openelevationservice/server/db_import/filestreams.py index b012621..f17c47b 100644 --- a/openelevationservice/server/db_import/filestreams.py +++ b/openelevationservice/server/db_import/filestreams.py @@ -3,20 +3,22 @@ from openelevationservice import TILES_DIR, SETTINGS from openelevationservice.server.utils.logger import get_logger -from os import path, environ +from os import path, environ, getcwd import requests import subprocess import zipfile from bs4 import BeautifulSoup +import time try: from io import BytesIO except: from StringIO import StringIO - + log = get_logger(__name__) -def downloadsrtm(xy_range): + +def downloadsrtm(xy_range=None): """ Downlaods SRTM v4.1 tiles as bytestream and saves them to TILES_DIR. @@ -25,49 +27,80 @@ def downloadsrtm(xy_range): in 'minx, maxx, miny, maxy. :type xy_range: comma-separated range string """ - - base_url = r'http://data.cgiar-csi.org/srtm/tiles/GeoTIFF/' - - # Create session for authentication - session = requests.Session() - - pw = environ.get('SRTMPASS') - user = environ.get('SRTMUSER') - if not user and not pw: - auth = tuple(SETTINGS['srtm_parameters'].values()) + + if SETTINGS["sources"][0]["type"] == "cgiar_csi": + + base_url = r'http://data.cgiar-csi.org/srtm/tiles/GeoTIFF/' + + # Create session for authentication + session = requests.Session() + + pw = environ.get('SRTMPASS') + user = environ.get('SRTMUSER') + if not user and not pw: + auth = tuple(SETTINGS['sources'][0]['srtm_parameters'].values()) + else: + auth = tuple([user, pw]) + session.auth = auth + + log.debug("SRTM credentials: {}".format(session.auth)) + + response = session.get(base_url) + + soup = BeautifulSoup(response.content, features="html.parser") + + # First find all 'a' tags starting href with srtm* + for link in soup.find_all('a', attrs={'href': lambda x: x.startswith('srtm') and x.endswith('.zip')}): + link_parsed = link.text.split('_') + link_x = int(link_parsed[1]) + link_y = int(link_parsed[2].split('.')[0]) + # Check if referenced geotif link is in xy_range + if link_y in range(*xy_range[0]) and link_x in range(*xy_range[1]): + log.info('yep') + # Then load the zip data in memory + if not path.exists(path.join(TILES_DIR, '_'.join(['srtm', str(link_x), str(link_y)]) + '.tif')): + with zipfile.ZipFile(BytesIO(session.get(base_url + link.text).content)) as zip_obj: + # Loop through the files in the zip + for filename in zip_obj.namelist(): + # Don't extract the readme.txt + if filename != 'readme.txt': + data = zip_obj.read(filename) + # Write byte contents to file + with open(path.join(TILES_DIR, filename), 'wb') as f: + f.write(data) + log.debug("Downloaded file {} to {}".format(link.text, TILES_DIR)) + else: + log.debug("File {} already exists in {}".format(link.text, TILES_DIR)) + else: - auth = tuple([user,pw]) - session.auth = auth - - log.debug("SRTM credentials: {}".format(session.auth)) - - response = session.get(base_url) - - soup = BeautifulSoup(response.content, features="html.parser") - - # First find all 'a' tags starting href with srtm* - for link in soup.find_all('a', attrs={'href': lambda x: x.startswith('srtm') and x.endswith('.zip')}): - link_parsed = link.text.split('_') - link_x = int(link_parsed[1]) - link_y = int(link_parsed[2].split('.')[0]) - # Check if referenced geotif link is in xy_range - if link_y in range(*xy_range[0]) and link_x in range(*xy_range[1]): - log.info('yep') - # Then load the zip data in memory - if not path.exists(path.join(TILES_DIR, '_'.join(['srtm', str(link_x), str(link_y)]) + '.tif')): - with zipfile.ZipFile(BytesIO(session.get(base_url + link.text).content)) as zip_obj: - # Loop through the files in the zip - for filename in zip_obj.namelist(): - # Don't extract the readme.txt - if filename != 'readme.txt': - data = zip_obj.read(filename) - # Write byte contents to file - with open(path.join(TILES_DIR, filename), 'wb') as f: - f.write(data) - log.debug("Downloaded file {} to {}".format(link.text, TILES_DIR)) - else: - log.debug("File {} already exists in {}".format(link.text, TILES_DIR)) - + + download_folder = SETTINGS["sources"][0]["type"] + local_path = getcwd() + "/tiles/" + + # run joerd_docker Dockerfile + proc_image = subprocess.Popen( + ["sudo", "docker", "build", "--tag=joerdsources", "-f", "./joerd_docker/Dockerfile", "."]) + + return_code_image = proc_image.wait() + if return_code_image is not None: + # run docker container + proc_container = subprocess.Popen(["sudo", "docker", "run", "-d", "joerdsources", "bash"]) + + return_code_container = proc_container.wait() + if return_code_container is not None: + container = subprocess.Popen(["sudo", "docker", "ps", "-lq"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + container_id = container.communicate()[0].rstrip().decode('UTF-8') + print("container_id:", container_id) + + # copy data from docker container to host + subprocess.Popen(["sudo", "docker", "cp", + container_id + ":/joerd/" + download_folder + "/", + local_path], + stdout=subprocess.PIPE) + # remove docker container? + def raster2pgsql(): """ @@ -75,13 +108,24 @@ def raster2pgsql(): :raises subprocess.CalledProcessError: Raised when raster2pgsql throws an error. """ - + + # Define access rights and arrange folder structure + if SETTINGS["sources"][0]["type"] != "cgiar_csi": + download_folder = SETTINGS["sources"][0]["type"] + local_path = getcwd() + "/tiles/" + + subprocess.Popen("sudo mv " + local_path + download_folder + "/* " + local_path, shell=True) + time.sleep(1) + subprocess.Popen("sudo chmod 776 " + local_path + "*", shell=True) + time.sleep(1) + subprocess.Popen(["sudo", "rm", "-r", local_path + download_folder]) + pg_settings = SETTINGS['provider_parameters'] - + # Copy all env variables and add PGPASSWORD env_current = environ.copy() env_current['PGPASSWORD'] = pg_settings['password'] - + # Tried to import every raster individually by user-specified xyrange # similar to download(), but raster2pgsql fuck it up somehow.. The PostGIS # raster will not be what one would expect. Instead doing a bulk import of all files. @@ -92,20 +136,22 @@ def raster2pgsql(): # -P: pad tiles to guarantee all tiles have the same width and height # -M: vacuum analyze after import # -t: specifies the pixel size of each row. Important to keep low for performance! - - cmd_raster2pgsql = cmd_raster2pgsql.format(**{'filename': path.join(TILES_DIR, '*.tif'), - **pg_settings}) - + + if path.join(TILES_DIR, '*.tif'): + cmd_raster2pgsql = cmd_raster2pgsql.format(**{'filename': path.join(TILES_DIR, '*.tif'), **pg_settings}) + else: + cmd_raster2pgsql = cmd_raster2pgsql.format(**{'filename': path.join(TILES_DIR, '*.img'), **pg_settings}) + proc = subprocess.Popen(cmd_raster2pgsql, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - shell=True, - env=env_current - ) - + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + shell=True, + env=env_current + ) + # for line in proc.stdout: # log.debug(line.decode()) # proc.stdout.close() return_code = proc.wait() if return_code: - raise subprocess.CalledProcessError(return_code, cmd_raster2pgsql) \ No newline at end of file + raise subprocess.CalledProcessError(return_code, cmd_raster2pgsql) diff --git a/openelevationservice/server/ops_settings.sample.yml b/openelevationservice/server/ops_settings.sample.yml deleted file mode 100644 index 8b33050..0000000 --- a/openelevationservice/server/ops_settings.sample.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -attribution: "service by https://openrouteservice.org | data by http://srtm.csi.cgiar.org" -coord_precision: 1e-6 -maximum_nodes: 2000 -srtm_parameters: - user: user - password: pw -provider_parameters: - table_name: oes_cgiar - db_name: gis - user_name: gis - password: gis - host: localhost - port: 5432 From 162f2946f9409678c4b178cbda43ad5326045f12 Mon Sep 17 00:00:00 2001 From: Isabell Klipper Date: Wed, 2 Oct 2019 16:31:12 +0200 Subject: [PATCH 02/11] adjust ops_settings.yml --- .../server/ops_settings.sample.yml | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 openelevationservice/server/ops_settings.sample.yml diff --git a/openelevationservice/server/ops_settings.sample.yml b/openelevationservice/server/ops_settings.sample.yml new file mode 100644 index 0000000..bfa9390 --- /dev/null +++ b/openelevationservice/server/ops_settings.sample.yml @@ -0,0 +1,55 @@ +--- +attribution: "service by https://openrouteservice.org | data by http://srtm.csi.cgiar.org or see https://github.com/tilezen/joerd_docker" +coord_precision: 1e-6 +maximum_nodes: 2000 +provider_parameters: + table_name: oes_joerd_etopo + db_name: gis + user_name: gis + password: gis + host: localhost + port: 5432 +regions: +# san-francisco-downtown: +# bbox: +# top: 37.7997 +# left: -122.5109 +# bottom: 37.7127 +# right: -122.3636 +# zoom_range: [0, 16] + heidelberg: + bbox: + top: 49.423927 + left: 8.662891 + bottom: 49.395223 + right: 8.709412 + zoom_range: [0, 16] +outputs: +# - type: skadi + - type: tiff +# - type: terrarium +# - type: normal +sources: +# - type: cgiar_csi +# url: http://data.cgiar-csi.org/srtm/tiles/GeoTIFF/ +# srtm_parameters: +# user: user +# password: pw + - type: etopo1 + url: https://www.ngdc.noaa.gov/mgg/global/relief/ETOPO1/data/bedrock/grid_registered/georeferenced_tiff/ETOPO1_Bed_g_geotiff.zip +# - type: gmted +# url: http://edcintl.cr.usgs.gov/downloads/sciweb1/shared/topo/downloads/GMTED/Global_tiles_GMTED +# ys: [-90, -70, -50, -30, -10, 10, 30, 50, 70] +# xs: [-180, -150, -120, -90, -60, -30, 0, 30, 60, 90, 120, 150] +# tries: 100 +# - type: ned13 +# ftp_server: rockyftp.cr.usgs.gov +# base_path: vdelivery/Datasets/Staged/NED/13/IMG +# - type: ned +# ftp_server: rockyftp.cr.usgs.gov +# base_path: vdelivery/Datasets/Staged/NED/19/IMG +# - type: ned_topobathy +# ftp_server: rockyftp.cr.usgs.gov +# base_path: vdelivery/Datasets/Staged/NED/19/IMG +logging: + config: logging.example.config \ No newline at end of file From d457d21996e464e51f4fe646386062c3923004f2 Mon Sep 17 00:00:00 2001 From: Isabell Klipper Date: Wed, 27 Nov 2019 15:46:33 +0100 Subject: [PATCH 03/11] [WP] Implemented multiple source download, raster processing --- Dockerfile | 20 +- docker-compose.yml | 10 +- joerd_docker/Dockerfile | 20 - manage.py | 19 +- openelevationservice/__init__.py | 2 +- openelevationservice/server/__init__.py | 3 +- .../server/db_import/filestreams.py | 168 ++-- .../server/db_import/models.py | 26 +- .../server/db_import/raster_processing.py | 120 +++ .../server/sources/__init__.py | 0 openelevationservice/server/sources/etopo1.py | 0 openelevationservice/server/sources/gmted.py | 81 ++ openelevationservice/server/sources/srtm.py | 125 +++ poetry.lock | 812 ++++++++++++++++++ pyproject.toml | 40 + requirements.txt | 18 - setup.py | 65 -- 17 files changed, 1294 insertions(+), 235 deletions(-) delete mode 100644 joerd_docker/Dockerfile create mode 100644 openelevationservice/server/db_import/raster_processing.py create mode 100644 openelevationservice/server/sources/__init__.py create mode 100644 openelevationservice/server/sources/etopo1.py create mode 100644 openelevationservice/server/sources/gmted.py create mode 100644 openelevationservice/server/sources/srtm.py create mode 100644 poetry.lock create mode 100644 pyproject.toml delete mode 100644 requirements.txt delete mode 100644 setup.py diff --git a/Dockerfile b/Dockerfile index 5c42ab8..e8eb633 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,9 @@ MAINTAINER Nils Nolde RUN apt-get update RUN apt-get install -y locales git python3-venv +RUN apt-get install -y build-essential libssl-dev libffi-dev python3-dev +RUN apt-get install -y libpq-dev +RUN apt-get install -y git curl # Set the locale RUN locale-gen en_US.UTF-8 @@ -32,13 +35,24 @@ RUN mkdir -p /deploy/app COPY gunicorn_config.py /deploy/gunicorn_config.py COPY manage.py /deploy/app/manage.py -COPY requirements.txt /deploy/app/requirements.txt - RUN python3 -m venv /oes_venv RUN /bin/bash -c "source /oes_venv/bin/activate" -RUN /oes_venv/bin/pip3 install -r /deploy/app/requirements.txt +# install poetry +RUN curl -sSL https://raw.githubusercontent.com/sdispater/poetry/master/get-poetry.py | python3 +ENV PATH "/root/.poetry/bin:/oes_venv/bin:${PATH}" + +# install dependencies via poetry +RUN poetry config settings.virtualenvs.create false +RUN poetry self:update --preview +RUN poetry config virtualenvs.create false +COPY pyproject.toml poetry.lock README.rst / +RUN poetry install + +RUN apt-get install -y python-gdal python-bs4 python-numpy gdal-bin python-setuptools python-shapely +RUN apt-get install -y libgdal-dev python3-dev +RUN /oes_venv/bin/pip3 install GDAL==$(gdal-config --version) --global-option=build_ext --global-option="-I/usr/include/gdal" COPY openelevationservice /deploy/app/openelevationservice COPY ops_settings_docker.yml /deploy/app/openelevationservice/server/ops_settings.yml diff --git a/docker-compose.yml b/docker-compose.yml index 5ec9d6e..7ecb1f4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,13 +1,11 @@ version: '2.2' services: gunicorn_flask: - #network_mode: "host" + network_mode: "host" build: . volumes: - - ./tiles:/deploy/app/tiles + - ./tiles:/deploy/app/tiles + - ./ops_settings_docker.yml:/deploy/app/openelevationservice/server/ops_settings.yml:rw ports: - "5020:5000" - mem_limit: 28g - joerd_docker: - build: - context: ./joerd_docker \ No newline at end of file + mem_limit: 28g \ No newline at end of file diff --git a/joerd_docker/Dockerfile b/joerd_docker/Dockerfile deleted file mode 100644 index f256a79..0000000 --- a/joerd_docker/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -# joerd - -# FROM python:3.6 -FROM ubuntu:18.04 - -RUN apt-get update -RUN apt-get install sudo -RUN apt-get install -y git - -# clone joerd_docker from git and install -RUN sudo git clone https://github.com/tilezen/joerd.git -WORKDIR /joerd -RUN sudo apt-get install -y python-gdal python-bs4 python-numpy gdal-bin python-setuptools python-shapely -RUN python setup.py install - -# copy ops_settings.yml from host into docker container -COPY ./openelevationservice/server/ops_settings.yml ./ops_settings.yml - -# download data via joerd_docker -RUN joerd enqueue-downloads --config ops_settings.yml \ No newline at end of file diff --git a/manage.py b/manage.py index b8e9269..7c88b2a 100644 --- a/manage.py +++ b/manage.py @@ -6,15 +6,13 @@ from openelevationservice.server.db_import.models import db from openelevationservice.server.db_import import filestreams -import click - log = get_logger(__name__) app = create_app() + @app.cli.command() -@click.option('--xyrange', default='0,73,0,25') -def download(xyrange): +def download(): """ Downloads SRTM tiles to disk. Can be specified over minx, maxx, miny, maxy. @@ -22,8 +20,8 @@ def download(xyrange): in that order. For reference grid, see http://srtm.csi.cgiar.org/SELECTION/inputCoord.asp :type xyrange: comma-separated integers """ - - filestreams.downloadsrtm(_arg_format(xyrange)) + + filestreams.downloadsrtm() log.info("Downloaded all files") @@ -32,7 +30,8 @@ def create(): """Creates all tables defined in models.py""" db.create_all() - log.info("Table {} was created.".format(SETTINGS['provider_parameters']['table_name'])) + log.info("Table {} was created.".format(SETTINGS['provider_parameters']['table_name_srtm'])) + log.info("Table {} was created.".format(SETTINGS['provider_parameters']['table_name_composite'])) @app.cli.command() @@ -40,8 +39,8 @@ def drop(): """Drops all tables defined in models.py""" db.drop_all() - log.info("Table {} was dropped.".format(SETTINGS['provider_parameters']['table_name'])) - + log.info("Table {} was created.".format(SETTINGS['provider_parameters']['table_name_srtm'])) + log.info("Table {} was created.".format(SETTINGS['provider_parameters']['table_name_composite'])) @app.cli.command() def importdata(): @@ -66,4 +65,4 @@ def _arg_format(xy_range_txt): xy_range = [[str_split[0], str_split[2]], [str_split[1], str_split[3]]] - return xy_range \ No newline at end of file + return xy_range diff --git a/openelevationservice/__init__.py b/openelevationservice/__init__.py index d21c33c..e675b2e 100644 --- a/openelevationservice/__init__.py +++ b/openelevationservice/__init__.py @@ -9,7 +9,7 @@ TILES_DIR = path.join(getcwd(), 'tiles') if "TESTING" in environ: - SETTINGS['provider_parameters']['table_name'] = SETTINGS['provider_parameters']['table_name'] + '_test' + SETTINGS['provider_parameters']['table_name_srtm'] = SETTINGS['provider_parameters']['table_name_srtm'] + '_test' TILES_DIR = path.join(basedir, 'tests', 'tile') # if "CI" in environ: # SETTINGS['provider_parameters']['port'] = 5433 diff --git a/openelevationservice/server/__init__.py b/openelevationservice/server/__init__.py index c1fdf94..895188d 100644 --- a/openelevationservice/server/__init__.py +++ b/openelevationservice/server/__init__.py @@ -37,7 +37,8 @@ def create_app(script_info=None): log.info("Following provider parameters are active:\n" "Host:\t{host}\n" "DB:\t{db_name}\n" - "Table:\t{table_name}\n" + "Table1:\t{table_name_srtm}\n" + "Table2:\t{table_name_composite}\n" "User:\t{user_name}".format(**provider_details)) # register blueprints diff --git a/openelevationservice/server/db_import/filestreams.py b/openelevationservice/server/db_import/filestreams.py index f17c47b..3406110 100644 --- a/openelevationservice/server/db_import/filestreams.py +++ b/openelevationservice/server/db_import/filestreams.py @@ -2,104 +2,62 @@ from openelevationservice import TILES_DIR, SETTINGS from openelevationservice.server.utils.logger import get_logger +from openelevationservice.server.sources.gmted import GMTED +from openelevationservice.server.sources import srtm +from openelevationservice.server.db_import import raster_processing -from os import path, environ, getcwd -import requests +from os import path, environ import subprocess -import zipfile -from bs4 import BeautifulSoup -import time - -try: - from io import BytesIO -except: - from StringIO import StringIO log = get_logger(__name__) -def downloadsrtm(xy_range=None): +def downloadsrtm(): """ - Downlaods SRTM v4.1 tiles as bytestream and saves them to TILES_DIR. - - :param xy_range: The range of tiles in x and y as per grid in - http://srtm.csi.cgiar.org/SELECTION/inputCoord.asp - in 'minx, maxx, miny, maxy. - :type xy_range: comma-separated range string + Downlaods GMTED and SRTM v4.1 tiles as bytestream and saves them to TILES_DIR. """ - if SETTINGS["sources"][0]["type"] == "cgiar_csi": + extent_settings = SETTINGS['tables'][0]['srtm']['extent'] + if extent_settings['max_y'] <= 60: - base_url = r'http://data.cgiar-csi.org/srtm/tiles/GeoTIFF/' - - # Create session for authentication - session = requests.Session() - - pw = environ.get('SRTMPASS') - user = environ.get('SRTMUSER') - if not user and not pw: - auth = tuple(SETTINGS['sources'][0]['srtm_parameters'].values()) - else: - auth = tuple([user, pw]) - session.auth = auth - - log.debug("SRTM credentials: {}".format(session.auth)) - - response = session.get(base_url) - - soup = BeautifulSoup(response.content, features="html.parser") - - # First find all 'a' tags starting href with srtm* - for link in soup.find_all('a', attrs={'href': lambda x: x.startswith('srtm') and x.endswith('.zip')}): - link_parsed = link.text.split('_') - link_x = int(link_parsed[1]) - link_y = int(link_parsed[2].split('.')[0]) - # Check if referenced geotif link is in xy_range - if link_y in range(*xy_range[0]) and link_x in range(*xy_range[1]): - log.info('yep') - # Then load the zip data in memory - if not path.exists(path.join(TILES_DIR, '_'.join(['srtm', str(link_x), str(link_y)]) + '.tif')): - with zipfile.ZipFile(BytesIO(session.get(base_url + link.text).content)) as zip_obj: - # Loop through the files in the zip - for filename in zip_obj.namelist(): - # Don't extract the readme.txt - if filename != 'readme.txt': - data = zip_obj.read(filename) - # Write byte contents to file - with open(path.join(TILES_DIR, filename), 'wb') as f: - f.write(data) - log.debug("Downloaded file {} to {}".format(link.text, TILES_DIR)) - else: - log.debug("File {} already exists in {}".format(link.text, TILES_DIR)) + # only SRTM data download + srtm.download_srtm() - else: + # merge and clip raster by extent + log.info("Starting tile processing ...") + raster_processing.merge_raster('srtm_*', '/srtm_merged.tif') + raster_processing.clip_raster('/srtm_merged.tif', '/srtm_final.tif') + + elif extent_settings['max_y'] > 60 >= extent_settings['min_y']: - download_folder = SETTINGS["sources"][0]["type"] - local_path = getcwd() + "/tiles/" + # SRTM and GMTED data download + srtm.download_srtm() - # run joerd_docker Dockerfile - proc_image = subprocess.Popen( - ["sudo", "docker", "build", "--tag=joerdsources", "-f", "./joerd_docker/Dockerfile", "."]) + tiles_list = GMTED().tile_selection() + GMTED().download_gmted(tiles_list) - return_code_image = proc_image.wait() - if return_code_image is not None: - # run docker container - proc_container = subprocess.Popen(["sudo", "docker", "run", "-d", "joerdsources", "bash"]) + # resample and merge tiles + log.info("Starting tile preprocessing ...") + raster_processing.merge_raster('srtm_*', '/srtm_merged.tif') + raster_processing.clip_raster('/srtm_merged.tif', '/srtm_clipped.tif') + raster_processing.merge_raster('*_gmted_mea075.tif', '/gmted_merged.tif') + + log.info("Starting tile resampling ...") + raster_processing.gmted_resampling() + + log.info("Starting tile merging ...") + raster_processing.merge_raster('gmted_resampled.tif', '/raster_final.tif', '/srtm_clipped.tif') + + else: - return_code_container = proc_container.wait() - if return_code_container is not None: - container = subprocess.Popen(["sudo", "docker", "ps", "-lq"], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE) - container_id = container.communicate()[0].rstrip().decode('UTF-8') - print("container_id:", container_id) + # only GMTED data download + tiles_list = GMTED().tile_selection() + GMTED().download_gmted(tiles_list) - # copy data from docker container to host - subprocess.Popen(["sudo", "docker", "cp", - container_id + ":/joerd/" + download_folder + "/", - local_path], - stdout=subprocess.PIPE) - # remove docker container? + # merge and clip raster by extent + log.info("Starting tile processing ...") + raster_processing.merge_raster('*_gmted_mea075.tif', '/gmted_merged.tif') + raster_processing.clip_raster('/gmted_merged.tif', '/gmted_final.tif') def raster2pgsql(): @@ -109,38 +67,38 @@ def raster2pgsql(): :raises subprocess.CalledProcessError: Raised when raster2pgsql throws an error. """ - # Define access rights and arrange folder structure - if SETTINGS["sources"][0]["type"] != "cgiar_csi": - download_folder = SETTINGS["sources"][0]["type"] - local_path = getcwd() + "/tiles/" - - subprocess.Popen("sudo mv " + local_path + download_folder + "/* " + local_path, shell=True) - time.sleep(1) - subprocess.Popen("sudo chmod 776 " + local_path + "*", shell=True) - time.sleep(1) - subprocess.Popen(["sudo", "rm", "-r", local_path + download_folder]) - pg_settings = SETTINGS['provider_parameters'] # Copy all env variables and add PGPASSWORD env_current = environ.copy() env_current['PGPASSWORD'] = pg_settings['password'] - # Tried to import every raster individually by user-specified xyrange - # similar to download(), but raster2pgsql fuck it up somehow.. The PostGIS - # raster will not be what one would expect. Instead doing a bulk import of all files. - cmd_raster2pgsql = r"raster2pgsql -s 4326 -a -C -M -P -t 50x50 {filename} {table_name} | psql -q -h {host} -p {port} -U {user_name} -d {db_name}" - # -s: raster SRID - # -a: append to table (assumes it's been create with 'create()') - # -C: apply all raster Constraints - # -P: pad tiles to guarantee all tiles have the same width and height - # -M: vacuum analyze after import - # -t: specifies the pixel size of each row. Important to keep low for performance! + tiles_dir_name = None + + if SETTINGS["sources"][0]["type"] == "cgiar_csi": + + # Tried to import every raster individually by user-specified xyrange + # similar to download(), but raster2pgsql fuck it up somehow.. The PostGIS + # raster will not be what one would expect. Instead doing a bulk import of all files. + cmd_raster2pgsql = r"raster2pgsql -s 4326 -a -C -M -P -t 50x50 {filename} {table_name_srtm} | psql -q -h {host} -p {port} -U {user_name} -d {db_name}" + # -s: raster SRID + # -a: append to table (assumes it's been create with 'create()') + # -C: apply all raster Constraints + # -P: pad tiles to guarantee all tiles have the same width and height + # -M: vacuum analyze after import + # -t: specifies the pixel size of each row. Important to keep low for performance! + + tiles_dir_name = TILES_DIR + + else: + cmd_raster2pgsql = r"raster2pgsql -s 4326 -a -C -M -P -t 50x50 {filename} {table_name_joerd} | psql -q -h {host} -p {port} -U {user_name} -d {db_name}" + + tiles_dir_name = TILES_DIR if path.join(TILES_DIR, '*.tif'): - cmd_raster2pgsql = cmd_raster2pgsql.format(**{'filename': path.join(TILES_DIR, '*.tif'), **pg_settings}) + cmd_raster2pgsql = cmd_raster2pgsql.format(**{'filename': path.join(tiles_dir_name, '*.tif'), **pg_settings}) else: - cmd_raster2pgsql = cmd_raster2pgsql.format(**{'filename': path.join(TILES_DIR, '*.img'), **pg_settings}) + cmd_raster2pgsql = cmd_raster2pgsql.format(**{'filename': path.join(tiles_dir_name, '*.img'), **pg_settings}) proc = subprocess.Popen(cmd_raster2pgsql, stdout=subprocess.PIPE, diff --git a/openelevationservice/server/db_import/models.py b/openelevationservice/server/db_import/models.py index 7a77ba1..b43ccd9 100644 --- a/openelevationservice/server/db_import/models.py +++ b/openelevationservice/server/db_import/models.py @@ -10,15 +10,29 @@ db = SQLAlchemy() log = logger.get_logger(__name__) -table_name = SETTINGS['provider_parameters']['table_name'] +table_name_srtm = SETTINGS['provider_parameters']['table_name_srtm'] +table_name_composite = SETTINGS['provider_parameters']['table_name_composite'] + class Cgiar(db.Model): """Database model for SRTM v4.1 aka CGIAR dataset.""" - - __tablename__ = table_name - + + __tablename__ = table_name_srtm + rid = db.Column(db.Integer, primary_key=True) rast = db.Column(Raster) - + + def __repr__(self): + return ''.format(self.rid, self.rast) + + +class Joerd(db.Model): + """Database model for SRTM v4.1 aka CGIAR dataset.""" + + __tablename__ = table_name_composite + + rid = db.Column(db.Integer, primary_key=True) + rast = db.Column(Raster) + def __repr__(self): - return ''.format(self.rid, self.rast) \ No newline at end of file + return ''.format(self.rid, self.rast) diff --git a/openelevationservice/server/db_import/raster_processing.py b/openelevationservice/server/db_import/raster_processing.py new file mode 100644 index 0000000..65ee40d --- /dev/null +++ b/openelevationservice/server/db_import/raster_processing.py @@ -0,0 +1,120 @@ +# -*- coding: utf-8 -*- + +from openelevationservice import TILES_DIR, SETTINGS + +from osgeo import gdal, gdalconst +import subprocess +from os import path + + +# TODO: if files already exists ... +# TODO: handle in memory files + + +def merge_raster(input_filename, output_filename, reference=None): + """ Merge downloaded single tiles to one raster tile. """ + + output_merge = path.join(TILES_DIR + output_filename) + input_files = path.join(TILES_DIR + '/' + input_filename) + + if not path.exists(path.join(TILES_DIR, output_merge)): + + if reference is None: + + raster_merge = r"/usr/bin/gdal_merge.py -o {outfile} -of {outfile_format} {input_files}" + + cmd_merge = raster_merge.format(**{'outfile': output_merge, + 'outfile_format': 'GTiff', + 'input_files': input_files}) + + else: + # merge srtm and gmted tile fractions + output_merge = path.join(TILES_DIR + output_filename) + reference_file = path.join(TILES_DIR + reference) + + # -tap: align tiles + # reference_file: In areas of overlap, the last image will be copied over earlier ones. + merge = r"/usr/bin/gdal_merge.py -o {outfile} -of {outfile_format} -tap {input_files} {reference_file}" + + cmd_merge = merge.format(**{'outfile': output_merge, + 'outfile_format': 'GTiff', + 'input_files': input_files, # gmted + 'reference_file': reference_file}) # srtm + + proc = subprocess.Popen(cmd_merge, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + shell=True) + + return_code = proc.wait() + if return_code: + raise subprocess.CalledProcessError(return_code, cmd_merge) + + +def clip_raster(merged_filename, output_filename): + """ Clip merged raster by defined extent. """ + + if not path.exists(path.join(TILES_DIR, output_filename)): + + output_clip = path.join(TILES_DIR + output_filename) + merged_file = path.join(TILES_DIR + merged_filename) + merged_data = gdal.Open(merged_file, gdalconst.GA_ReadOnly) + + extent = list(SETTINGS['tables'][0]['srtm']['extent'].values()) + + # TODO: -t_srs {target_spatial_ref} + srtm_clip = r"/usr/bin/gdalwarp -t_srs {target_spatial_ref} -dstnodata -9999 -te {extent} -of {outfile_format} {input_file} {out_clipped_file}" + + cmd_clip = srtm_clip.format(**{'target_spatial_ref': merged_data.GetProjection(), + 'extent': extent, + 'outfile_format': 'GTiff', + 'input_file': merged_file, + 'out_clipped_file': output_clip}) + + proc_clip = subprocess.Popen(cmd_clip, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + shell=True) + + return_code_clip = proc_clip.wait() + if return_code_clip: + raise subprocess.CalledProcessError(return_code_clip, cmd_clip) + + +def gmted_resampling(): + """ Resample merged GMTED raster to SRTM resolution. """ + + output_resampled = path.join(TILES_DIR + '/gmted_resampled.tif') + + if not path.exists(path.join(TILES_DIR, output_resampled)): + + gmted_merged = path.join(TILES_DIR + '/gmted_merged.tif') + + srtm_clipped = path.join(TILES_DIR + '/srtm_clipped.tif') + reference_data = gdal.Open(srtm_clipped, gdalconst.GA_ReadOnly) + # desired resolution + x_res = reference_data.GetGeoTransform()[1] + y_res = reference_data.GetGeoTransform()[5] + + extent = list(SETTINGS['tables'][0]['srtm']['extent'].values()) + + # TODO: need extent? + resampling = r"/usr/bin/gdalwarp -t_srs {target_spatial_ref} -dstnodata -9999 -te {extent} -tr {x_res} {y_res} -r {resampling_method} -of {outfile_format} {input_file} {out_resampling_file}" + + cmd_resampling = resampling.format(**{'target_spatial_ref': reference_data.GetProjection(), + 'extent': extent, + 'x_res': x_res, + 'y_res': y_res, + 'resampling_method': 'average', + 'outfile_format': 'GTiff', + 'input_file': gmted_merged, + 'out_resampling_file': output_resampled}) + + proc_resampling = subprocess.Popen(cmd_resampling, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + shell=True) + + return_code_resampling = proc_resampling.wait() + if return_code_resampling: + raise subprocess.CalledProcessError(return_code_resampling, cmd_resampling) diff --git a/openelevationservice/server/sources/__init__.py b/openelevationservice/server/sources/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/openelevationservice/server/sources/etopo1.py b/openelevationservice/server/sources/etopo1.py new file mode 100644 index 0000000..e69de29 diff --git a/openelevationservice/server/sources/gmted.py b/openelevationservice/server/sources/gmted.py new file mode 100644 index 0000000..91cacd2 --- /dev/null +++ b/openelevationservice/server/sources/gmted.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- + +from openelevationservice import TILES_DIR, SETTINGS +from openelevationservice.server.utils.logger import get_logger + +from os import path +import requests + +log = get_logger(__name__) + + +class GMTED(object): + + def __init__(self): + self.base_settings = SETTINGS['tables'][0]['srtm'] + self.url = self.base_settings['sources'][1]['gmted']['url'] + self.xs = self.base_settings['sources'][1]['gmted']['xs'] + self.ys = self.base_settings['sources'][1]['gmted']['ys'] + self.bbox_extent = self.base_settings['extent'] + self.buffer = 0.1 + + @staticmethod + def intersects(buffered_bbox, bbox): + + if buffered_bbox[0] > bbox[2]: + return False + if buffered_bbox[1] > bbox[3]: + return False + if buffered_bbox[2] < bbox[0]: + return False + if buffered_bbox[3] < bbox[1]: + return False + return True + + def url_selection(self, bbox_x, bbox_y): + + # file name for chosen region + res = '300' if self.ys == -90 else '075' + x_name = "%03d%s" % (abs(bbox_x), "E" if bbox_x >= 0 else "W") + y_name = "%02d%s" % (abs(bbox_y), "N" if bbox_y >= 0 else "S") + filename = "%(y)s%(x)s_20101117_gmted_mea%(res)s.tif" % dict(res=res, x=x_name, y=y_name) + + # file url for chosen region + gmted_dir = "%s%03d" % ("E" if bbox_x >= 0 else "W", abs(bbox_x)) + d_name = "/%(res)sdarcsec/mea/%(dir)s/" % dict(res=res, dir=gmted_dir) + file_url = self.url + d_name + filename + + return filename, file_url + + def tile_selection(self): + + # buffer by 0.1 degrees (48px) around bbox to grab neighbouring tiles + # to ensure that there's no tile edge artefacts. + buffered_bbox = [self.bbox_extent['min_x'] - self.buffer, + self.bbox_extent['min_y'] - self.buffer, + self.bbox_extent['max_x'] + self.buffer, + self.bbox_extent['max_y'] + self.buffer] + + tiles_to_download = [] + for y in self.ys: + for x in self.xs: + bbox = [x, y, x + 30, y + 20] + if self.intersects(buffered_bbox, bbox): + filename, file_url = self.url_selection(x, y) + tiles_to_download.append([filename, file_url]) + log.info(filename) + + return tiles_to_download + + def download_gmted(self, tiles_list): + """ Download tiles and save to disk. """ + + log.info("{} GMTED tile(s) will be downloaded.".format(len(tiles_list))) + for tile in tiles_list: + if not path.exists(path.join(TILES_DIR, tile[0])): + with open(path.join(TILES_DIR, tile[0]), 'wb') as f: + log.info("Starting to download {}".format(tile[0])) + f.write(requests.get(tile[1]).content) + log.info("Downloaded file {} to {}".format(tile[0], TILES_DIR)) + else: + log.info("{} already exists in {}".format(tile[0], TILES_DIR)) diff --git a/openelevationservice/server/sources/srtm.py b/openelevationservice/server/sources/srtm.py new file mode 100644 index 0000000..9c3179f --- /dev/null +++ b/openelevationservice/server/sources/srtm.py @@ -0,0 +1,125 @@ +# -*- coding: utf-8 -*- + +from openelevationservice import TILES_DIR, SETTINGS +from openelevationservice.server.utils.logger import get_logger + +from os import path, environ +import requests +import zipfile +from bs4 import BeautifulSoup + +from io import BytesIO + +log = get_logger(__name__) + + +def srtm_x_value(x_min, x_max): + """ Define SRTM x value download name. """ + + x_value_list = [] + + lat_min = -180 + lat_max = -175 + x_srtm_value = 1 + + for i in range(0, 360): + + if lat_min < x_min <= lat_max: + if x_min <= x_max: + x_value_list.append(x_srtm_value) + x_min += 5 + else: + return x_value_list + else: + lat_min += 5 + lat_max += 5 + x_srtm_value += 1 + + +def srtm_y_value(y_min, y_max): + """ Define SRTM y value download name. """ + + y_value_list = [] + + lon_min = 55 + lon_max = 60 + y_srtm_value = 1 + + # if lon > 60 or < -60: no data available + # TODO: no download instead of boarder tiles + if y_min > 60: + return y_srtm_value + + elif y_min < -60: + y_srtm_value = 24 + return y_srtm_value + + else: + for i in range(0, 120): + + if lon_max > y_min >= lon_min: + if y_min <= y_max: + y_value_list.append(y_srtm_value) + y_min += 5 + if y_min > 60: + return y_value_list + else: + return y_value_list + else: + lon_min -= 5 + lon_max -= 5 + y_srtm_value -= 1 + + +def download_srtm(): + """ Download SRTM data. """ + + base_url = SETTINGS['tables'][0]['srtm']['sources'][0]['srtm']['url'] + bbox_extent = SETTINGS['tables'][0]['srtm']['extent'] + + x_list = srtm_x_value(bbox_extent['min_x'], bbox_extent['max_x']) + y_list = srtm_y_value(bbox_extent['min_y'], bbox_extent['max_y']) + + xy_range = [[int(x_list[0]), int(x_list[-1] + 1)], + [int(y_list[0]), int(y_list[-1] + 1)]] + + # Create session for authentication + session = requests.Session() + + pw = environ.get('SRTMPASS') + user = environ.get('SRTMUSER') + if not user and not pw: + auth = tuple(SETTINGS['tables'][0]['srtm']['sources'][0]['srtm']['srtm_parameters'].values()) + else: + auth = tuple([user, pw]) + session.auth = auth + + log.debug("SRTM credentials: {}".format(session.auth)) + + response = session.get(base_url) + + soup = BeautifulSoup(response.content, features="html.parser") + + # First find all 'a' tags starting href with srtm* + for link in soup.find_all('a', attrs={'href': lambda x: x.startswith('srtm') and x.endswith('.zip')}): + link_parsed = link.text.split('_') + link_x = int(link_parsed[1]) + link_y = int(link_parsed[2].split('.')[0]) + # Check if referenced geotif link is in xy_range + if link_y in range(*xy_range[1]) and link_x in range(*xy_range[0]): + log.info('yep') + # Then load the zip data in memory + if not path.exists(path.join(TILES_DIR, '_'.join(['srtm', str(link_x), str(link_y)]) + '.tif')): + with zipfile.ZipFile(BytesIO(session.get(base_url + link.text).content)) as zip_obj: + # Loop through the files in the zip + for filename in zip_obj.namelist(): + # Don't extract the readme.txt + # if filename != 'readme.txt': + if filename.endswith('.tif'): + data = zip_obj.read(filename) + # Write byte contents to file + with open(path.join(TILES_DIR, filename), 'wb') as f: + f.write(data) + log.debug("Downloaded file {} to {}".format(link.text, TILES_DIR)) + else: + log.debug("File {} already exists in {}".format(link.text, TILES_DIR)) diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 0000000..8676e3b --- /dev/null +++ b/poetry.lock @@ -0,0 +1,812 @@ +[[package]] +category = "main" +description = "Screen-scraping library" +name = "beautifulsoup4" +optional = false +python-versions = "*" +version = "4.8.1" + +[package.dependencies] +soupsieve = ">=1.2" + +[package.extras] +html5lib = ["html5lib"] +lxml = ["lxml"] + +[[package]] +category = "main" +description = "The AWS SDK for Python" +name = "boto3" +optional = false +python-versions = "*" +version = "1.10.2" + +[package.dependencies] +botocore = ">=1.13.2,<1.14.0" +jmespath = ">=0.7.1,<1.0.0" +s3transfer = ">=0.2.0,<0.3.0" + +[[package]] +category = "main" +description = "Low-level, data-driven core of boto 3." +name = "botocore" +optional = false +python-versions = "*" +version = "1.13.2" + +[package.dependencies] +docutils = ">=0.10,<0.16" +jmespath = ">=0.7.1,<1.0.0" + +[package.dependencies.python-dateutil] +python = ">=2.7" +version = ">=2.1,<3.0.0" + +[package.dependencies.urllib3] +python = ">=3.4" +version = ">=1.20,<1.26" + +[[package]] +category = "main" +description = "Lightweight, extensible schema and data validation tool for Python dictionaries." +name = "cerberus" +optional = false +python-versions = ">=2.7" +version = "1.3.1" + +[[package]] +category = "main" +description = "Python package for providing Mozilla's CA Bundle." +name = "certifi" +optional = false +python-versions = "*" +version = "2019.9.11" + +[[package]] +category = "main" +description = "Foreign Function Interface for Python calling C code." +marker = "sys_platform == \"win32\" and platform_python_implementation == \"CPython\"" +name = "cffi" +optional = false +python-versions = "*" +version = "1.13.1" + +[package.dependencies] +pycparser = "*" + +[[package]] +category = "main" +description = "Universal encoding detector for Python 2 and 3" +name = "chardet" +optional = false +python-versions = "*" +version = "3.0.4" + +[[package]] +category = "main" +description = "Composable command line interface toolkit" +name = "click" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "7.0" + +[[package]] +category = "main" +description = "Backports and enhancements for the contextlib module" +name = "contextlib2" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "0.6.0.post1" + +[[package]] +category = "main" +description = "Docutils -- Python Documentation Utilities" +name = "docutils" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +version = "0.15.2" + +[[package]] +category = "main" +description = "Extract swagger specs from your flask project" +name = "flasgger" +optional = false +python-versions = "*" +version = "0.9.3" + +[package.dependencies] +Flask = ">=0.10" +PyYAML = ">=3.0" +jsonschema = [">=2.5.1", "<3.0.0"] +mistune = "*" +six = ">=1.10.0" + +[[package]] +category = "main" +description = "A simple framework for building complex web applications." +name = "flask" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "1.1.1" + +[package.dependencies] +Jinja2 = ">=2.10.1" +Werkzeug = ">=0.15" +click = ">=5.1" +itsdangerous = ">=0.24" + +[package.extras] +dev = ["pytest", "coverage", "tox", "sphinx", "pallets-sphinx-themes", "sphinxcontrib-log-cabinet", "sphinx-issues"] +docs = ["sphinx", "pallets-sphinx-themes", "sphinxcontrib-log-cabinet", "sphinx-issues"] +dotenv = ["python-dotenv"] + +[[package]] +category = "main" +description = "A Flask extension adding a decorator for CORS support" +name = "flask-cors" +optional = false +python-versions = "*" +version = "3.0.8" + +[package.dependencies] +Flask = ">=0.9" +Six = "*" + +[[package]] +category = "main" +description = "Adds SQLAlchemy support to your Flask application." +name = "flask-sqlalchemy" +optional = false +python-versions = ">= 2.7, != 3.0.*, != 3.1.*, != 3.2.*, != 3.3.*" +version = "2.4.1" + +[package.dependencies] +Flask = ">=0.10" +SQLAlchemy = ">=0.8.0" + +[[package]] +category = "main" +description = "Clean single-source support for Python 3 and 2" +name = "future" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +version = "0.18.1" + +[[package]] +category = "main" +description = "GDAL: Geospatial Data Abstraction Library" +name = "gdal" +optional = false +python-versions = "*" +version = "3.0.1" + +[[package]] +category = "main" +description = "Using SQLAlchemy with Spatial Databases" +name = "geoalchemy2" +optional = false +python-versions = "*" +version = "0.6.3" + +[package.dependencies] +SQLAlchemy = ">=0.8" + +[[package]] +category = "main" +description = "The geodesic routines from GeographicLib" +name = "geographiclib" +optional = false +python-versions = "*" +version = "1.50" + +[[package]] +category = "main" +description = "Python bindings and utilities for GeoJSON" +name = "geojson" +optional = false +python-versions = "*" +version = "2.5.0" + +[[package]] +category = "main" +description = "Coroutine-based network library" +name = "gevent" +optional = false +python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*" +version = "1.4.0" + +[package.dependencies] +cffi = ">=1.11.5" +greenlet = ">=0.4.14" + +[package.extras] +dnspython = ["dnspython", "idna"] +doc = ["repoze.sphinx.autointerface"] +events = ["zope.event", "zope.interface"] +test = ["zope.interface", "zope.event", "requests", "objgraph", "psutil", "futures", "mock", "coverage (>=5.0a3)", "coveralls (>=1.0)"] + +[[package]] +category = "main" +description = "Lightweight in-process concurrent programming" +marker = "platform_python_implementation == \"CPython\"" +name = "greenlet" +optional = false +python-versions = "*" +version = "0.4.15" + +[[package]] +category = "main" +description = "WSGI HTTP Server for UNIX" +name = "gunicorn" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*" +version = "19.9.0" + +[package.extras] +eventlet = ["eventlet (>=0.9.7)"] +gevent = ["gevent (>=0.13)"] +tornado = ["tornado (>=0.2)"] + +[[package]] +category = "main" +description = "Internationalized Domain Names in Applications (IDNA)" +name = "idna" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "2.8" + +[[package]] +category = "main" +description = "Various helpers to pass data to untrusted environments and back." +name = "itsdangerous" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "1.1.0" + +[[package]] +category = "main" +description = "A very fast and expressive template engine." +name = "jinja2" +optional = false +python-versions = "*" +version = "2.10.3" + +[package.dependencies] +MarkupSafe = ">=0.23" + +[package.extras] +i18n = ["Babel (>=0.8)"] + +[[package]] +category = "main" +description = "JSON Matching Expressions" +name = "jmespath" +optional = false +python-versions = "*" +version = "0.9.4" + +[[package]] +category = "main" +description = "A tool for downloading and generating elevation data." +name = "joerd" +optional = false +python-versions = "*" +version = "0.0.1" + +[package.dependencies] +GDAL = "*" +PyYAML = "*" +beautifulsoup4 = "*" +boto3 = "*" +contextlib2 = "*" +future = "*" +geographiclib = "*" +numpy = "*" +pyqtree = "*" +requests = "*" + +[package.source] +reference = "3fbf577bcff2f02ca7a288a96178cbef56d0513f" +type = "git" +url = "https://github.com/isikl/joerd.git" +[[package]] +category = "main" +description = "An implementation of JSON Schema validation for Python" +name = "jsonschema" +optional = false +python-versions = "*" +version = "2.6.0" + +[package.extras] +format = ["rfc3987", "strict-rfc3339", "webcolors"] + +[[package]] +category = "main" +description = "Safely add untrusted strings to HTML/XML markup." +name = "markupsafe" +optional = false +python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*" +version = "1.1.1" + +[[package]] +category = "main" +description = "The fastest markdown parser in pure Python" +name = "mistune" +optional = false +python-versions = "*" +version = "0.8.4" + +[[package]] +category = "main" +description = "NumPy is the fundamental package for array computing with Python." +name = "numpy" +optional = false +python-versions = ">=3.5" +version = "1.17.3" + +[[package]] +category = "main" +description = "psycopg2 - Python-PostgreSQL Database Adapter" +name = "psycopg2" +optional = false +python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*" +version = "2.8.4" + +[[package]] +category = "main" +description = "C parser in Python" +marker = "sys_platform == \"win32\" and platform_python_implementation == \"CPython\"" +name = "pycparser" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "2.19" + +[[package]] +category = "main" +description = "A pure Python quad tree spatial index for GIS or rendering usage." +name = "pyqtree" +optional = false +python-versions = "*" +version = "1.0.0" + +[[package]] +category = "main" +description = "Extensions to the standard Python datetime module" +marker = "python_version >= \"2.7\"" +name = "python-dateutil" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +version = "2.8.0" + +[package.dependencies] +six = ">=1.5" + +[[package]] +category = "main" +description = "YAML parser and emitter for Python" +name = "pyyaml" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "5.1.2" + +[[package]] +category = "main" +description = "Python HTTP for Humans." +name = "requests" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "2.22.0" + +[package.dependencies] +certifi = ">=2017.4.17" +chardet = ">=3.0.2,<3.1.0" +idna = ">=2.5,<2.9" +urllib3 = ">=1.21.1,<1.25.0 || >1.25.0,<1.25.1 || >1.25.1,<1.26" + +[package.extras] +security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)"] +socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7)", "win-inet-pton"] + +[[package]] +category = "main" +description = "An Amazon S3 Transfer Manager" +name = "s3transfer" +optional = false +python-versions = "*" +version = "0.2.1" + +[package.dependencies] +botocore = ">=1.12.36,<2.0.0" + +[[package]] +category = "main" +description = "Geometric objects, predicates, and operations" +name = "shapely" +optional = false +python-versions = "*" +version = "1.6.4.post2" + +[package.extras] +all = ["pytest", "pytest-cov", "numpy"] +test = ["pytest", "pytest-cov"] +vectorized = ["numpy"] + +[[package]] +category = "main" +description = "Python 2 and 3 compatibility utilities" +name = "six" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*" +version = "1.12.0" + +[[package]] +category = "main" +description = "A modern CSS selector implementation for Beautiful Soup." +name = "soupsieve" +optional = false +python-versions = "*" +version = "1.9.4" + +[[package]] +category = "main" +description = "Database Abstraction Library" +name = "sqlalchemy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "1.3.10" + +[package.extras] +mssql = ["pyodbc"] +mssql_pymssql = ["pymssql"] +mssql_pyodbc = ["pyodbc"] +mysql = ["mysqlclient"] +oracle = ["cx-oracle"] +postgresql = ["psycopg2"] +postgresql_pg8000 = ["pg8000"] +postgresql_psycopg2binary = ["psycopg2-binary"] +postgresql_psycopg2cffi = ["psycopg2cffi"] +pymysql = ["pymysql"] + +[[package]] +category = "main" +description = "HTTP library with thread-safe connection pooling, file post, and more." +name = "urllib3" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, <4" +version = "1.25.6" + +[package.extras] +brotli = ["brotlipy (>=0.6.0)"] +secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] +socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7,<2.0)"] + +[[package]] +category = "main" +description = "The comprehensive WSGI web application library." +name = "werkzeug" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "0.16.0" + +[package.extras] +dev = ["pytest", "coverage", "tox", "sphinx", "pallets-sphinx-themes", "sphinx-issues"] +termcolor = ["termcolor"] +watchdog = ["watchdog"] + +[metadata] +content-hash = "6b4f1da0f0772e004bd793df42ff4050c85a0eb56c641f233709420aa1fdd53e" +python-versions = "^3.6" + +[metadata.files] +beautifulsoup4 = [ + {file = "beautifulsoup4-4.8.1-py2-none-any.whl", hash = "sha256:5279c36b4b2ec2cb4298d723791467e3000e5384a43ea0cdf5d45207c7e97169"}, + {file = "beautifulsoup4-4.8.1-py3-none-any.whl", hash = "sha256:dcdef580e18a76d54002088602eba453eec38ebbcafafeaabd8cab12b6155d57"}, + {file = "beautifulsoup4-4.8.1.tar.gz", hash = "sha256:6135db2ba678168c07950f9a16c4031822c6f4aec75a65e0a97bc5ca09789931"}, +] +boto3 = [ + {file = "boto3-1.10.2-py2.py3-none-any.whl", hash = "sha256:7fc97cb2c9cdff905e950750c8e8b23b872a84696158a28852355dc4b712ba3a"}, + {file = "boto3-1.10.2.tar.gz", hash = "sha256:818c56a317c176142dbf1dca3f5b4366c80460c6cc3c4efe22f0bde736571283"}, +] +botocore = [ + {file = "botocore-1.13.2-py2.py3-none-any.whl", hash = "sha256:f8e12dc6e536ea512f0ad25b74e7eecdf5d9e09ae92b5de236b535bee7804d5b"}, + {file = "botocore-1.13.2.tar.gz", hash = "sha256:8223485841ef4731a5d4943a733295ba69d0005c4ae64c468308cc07f6960d39"}, +] +cerberus = [ + {file = "Cerberus-1.3.1.tar.gz", hash = "sha256:0be48fc0dc84f83202a5309c0aa17cd5393e70731a1698a50d118b762fbe6875"}, +] +certifi = [ + {file = "certifi-2019.9.11-py2.py3-none-any.whl", hash = "sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef"}, + {file = "certifi-2019.9.11.tar.gz", hash = "sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50"}, +] +cffi = [ + {file = "cffi-1.13.1-cp27-cp27m-macosx_10_6_intel.whl", hash = "sha256:9009e917d8f5ef780c2626e29b6bc126f4cb2a4d43ca67aa2b40f2a5d6385e78"}, + {file = "cffi-1.13.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:825ecffd9574557590e3225560a8a9d751f6ffe4a49e3c40918c9969b93395fa"}, + {file = "cffi-1.13.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:193697c2918ecdb3865acf6557cddf5076bb39f1f654975e087b67efdff83365"}, + {file = "cffi-1.13.1-cp27-cp27m-win32.whl", hash = "sha256:1ae14b542bf3b35e5229439c35653d2ef7d8316c1fffb980f9b7647e544baa98"}, + {file = "cffi-1.13.1-cp27-cp27m-win_amd64.whl", hash = "sha256:0ea23c9c0cdd6778146a50d867d6405693ac3b80a68829966c98dd5e1bbae400"}, + {file = "cffi-1.13.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:ec2fa3ee81707a5232bf2dfbd6623fdb278e070d596effc7e2d788f2ada71a05"}, + {file = "cffi-1.13.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:7cfcfda59ef1f95b9f729c56fe8a4041899f96b72685d36ef16a3440a0f85da8"}, + {file = "cffi-1.13.1-cp34-cp34m-macosx_10_6_intel.whl", hash = "sha256:6fd58366747debfa5e6163ada468a90788411f10c92597d3b0a912d07e580c36"}, + {file = "cffi-1.13.1-cp34-cp34m-manylinux1_i686.whl", hash = "sha256:9c77564a51d4d914ed5af096cd9843d90c45b784b511723bd46a8a9d09cf16fc"}, + {file = "cffi-1.13.1-cp34-cp34m-manylinux1_x86_64.whl", hash = "sha256:728ec653964655d65408949b07f9b2219df78badd601d6c49e28d604efe40599"}, + {file = "cffi-1.13.1-cp34-cp34m-win32.whl", hash = "sha256:b8f09f21544b9899defb09afbdaeb200e6a87a2b8e604892940044cf94444644"}, + {file = "cffi-1.13.1-cp34-cp34m-win_amd64.whl", hash = "sha256:8a2bcae2258d00fcfc96a9bde4a6177bc4274fe033f79311c5dd3d3148c26518"}, + {file = "cffi-1.13.1-cp35-cp35m-macosx_10_6_intel.whl", hash = "sha256:a19089fa74ed19c4fe96502a291cfdb89223a9705b1d73b3005df4256976142e"}, + {file = "cffi-1.13.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:e22a00c0c81ffcecaf07c2bfb3672fa372c50e2bd1024ffee0da191c1b27fc71"}, + {file = "cffi-1.13.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:bb75ba21d5716abc41af16eac1145ab2e471deedde1f22c6f99bd9f995504df0"}, + {file = "cffi-1.13.1-cp35-cp35m-win32.whl", hash = "sha256:364f8404034ae1b232335d8c7f7b57deac566f148f7222cef78cf8ae28ef764e"}, + {file = "cffi-1.13.1-cp35-cp35m-win_amd64.whl", hash = "sha256:fd82eb4694be712fcae03c717ca2e0fc720657ac226b80bbb597e971fc6928c2"}, + {file = "cffi-1.13.1-cp36-cp36m-macosx_10_6_intel.whl", hash = "sha256:5ba86e1d80d458b338bda676fd9f9d68cb4e7a03819632969cf6d46b01a26730"}, + {file = "cffi-1.13.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:63424daa6955e6b4c70dc2755897f5be1d719eabe71b2625948b222775ed5c43"}, + {file = "cffi-1.13.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:33142ae9807665fa6511cfa9857132b2c3ee6ddffb012b3f0933fc11e1e830d5"}, + {file = "cffi-1.13.1-cp36-cp36m-win32.whl", hash = "sha256:e55b5a746fb77f10c83e8af081979351722f6ea48facea79d470b3731c7b2891"}, + {file = "cffi-1.13.1-cp36-cp36m-win_amd64.whl", hash = "sha256:47368f69fe6529f8f49a5d146ddee713fc9057e31d61e8b6dc86a6a5e38cecc1"}, + {file = "cffi-1.13.1-cp37-cp37m-macosx_10_6_intel.whl", hash = "sha256:4895640844f17bec32943995dc8c96989226974dfeb9dd121cc45d36e0d0c434"}, + {file = "cffi-1.13.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:00d890313797d9fe4420506613384b43099ad7d2b905c0752dbcc3a6f14d80fa"}, + {file = "cffi-1.13.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:a40ed527bffa2b7ebe07acc5a3f782da072e262ca994b4f2085100b5a444bbb2"}, + {file = "cffi-1.13.1-cp37-cp37m-win32.whl", hash = "sha256:6381a7d8b1ebd0bc27c3bc85bc1bfadbb6e6f756b4d4db0aa1425c3719ba26b4"}, + {file = "cffi-1.13.1-cp37-cp37m-win_amd64.whl", hash = "sha256:1e389e069450609c6ffa37f21f40cce36f9be7643bbe5051ab1de99d5a779526"}, + {file = "cffi-1.13.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6381ab708158c4e1639da1f2a7679a9bbe3e5a776fc6d1fd808076f0e3145331"}, + {file = "cffi-1.13.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:0cf9e550ac6c5e57b713437e2f4ac2d7fd0cd10336525a27224f5fc1ec2ee59a"}, + {file = "cffi-1.13.1-cp38-cp38-win32.whl", hash = "sha256:819f8d5197c2684524637f940445c06e003c4a541f9983fd30d6deaa2a5487d8"}, + {file = "cffi-1.13.1-cp38-cp38-win_amd64.whl", hash = "sha256:263242b6ace7f9cd4ea401428d2d45066b49a700852334fd55311bde36dcda14"}, + {file = "cffi-1.13.1.tar.gz", hash = "sha256:558b3afef987cf4b17abd849e7bedf64ee12b28175d564d05b628a0f9355599b"}, +] +chardet = [ + {file = "chardet-3.0.4-py2.py3-none-any.whl", hash = "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"}, + {file = "chardet-3.0.4.tar.gz", hash = "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae"}, +] +click = [ + {file = "Click-7.0-py2.py3-none-any.whl", hash = "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13"}, + {file = "Click-7.0.tar.gz", hash = "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7"}, +] +contextlib2 = [ + {file = "contextlib2-0.6.0.post1-py2.py3-none-any.whl", hash = "sha256:3355078a159fbb44ee60ea80abd0d87b80b78c248643b49aa6d94673b413609b"}, + {file = "contextlib2-0.6.0.post1.tar.gz", hash = "sha256:01f490098c18b19d2bd5bb5dc445b2054d2fa97f09a4280ba2c5f3c394c8162e"}, +] +docutils = [ + {file = "docutils-0.15.2-py2-none-any.whl", hash = "sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827"}, + {file = "docutils-0.15.2-py3-none-any.whl", hash = "sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0"}, + {file = "docutils-0.15.2.tar.gz", hash = "sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99"}, +] +flasgger = [ + {file = "flasgger-0.9.3-py2.py3-none-any.whl", hash = "sha256:45ce85bf826f71e07d734e52a1af000b3fef82bd1a2337ea5bc4585a3383c56d"}, + {file = "flasgger-0.9.3.tar.gz", hash = "sha256:fe62d086ca341f71072fd21c2d4f993b6958ce47ee8f0a4ae716e7c901afeb21"}, +] +flask = [ + {file = "Flask-1.1.1-py2.py3-none-any.whl", hash = "sha256:45eb5a6fd193d6cf7e0cf5d8a5b31f83d5faae0293695626f539a823e93b13f6"}, + {file = "Flask-1.1.1.tar.gz", hash = "sha256:13f9f196f330c7c2c5d7a5cf91af894110ca0215ac051b5844701f2bfd934d52"}, +] +flask-cors = [ + {file = "Flask-Cors-3.0.8.tar.gz", hash = "sha256:72170423eb4612f0847318afff8c247b38bd516b7737adfc10d1c2cdbb382d16"}, + {file = "Flask_Cors-3.0.8-py2.py3-none-any.whl", hash = "sha256:f4d97201660e6bbcff2d89d082b5b6d31abee04b1b3003ee073a6fd25ad1d69a"}, +] +flask-sqlalchemy = [ + {file = "Flask-SQLAlchemy-2.4.1.tar.gz", hash = "sha256:6974785d913666587949f7c2946f7001e4fa2cb2d19f4e69ead02e4b8f50b33d"}, + {file = "Flask_SQLAlchemy-2.4.1-py2.py3-none-any.whl", hash = "sha256:0078d8663330dc05a74bc72b3b6ddc441b9a744e2f56fe60af1a5bfc81334327"}, +] +future = [ + {file = "future-0.18.1.tar.gz", hash = "sha256:858e38522e8fd0d3ce8f0c1feaf0603358e366d5403209674c7b617fa0c24093"}, +] +gdal = [ + {file = "GDAL-3.0.1.tar.gz", hash = "sha256:1741bc59c5ceb9b1c50763348cd1050048fc916b23d08138982a340485f0e4af"}, +] +geoalchemy2 = [ + {file = "GeoAlchemy2-0.6.3-py2.py3-none-any.whl", hash = "sha256:0d1c9ea3ec13f6a522ccc3ffd2569ac524a6c6e80bab883e8805b28c48e77143"}, + {file = "GeoAlchemy2-0.6.3.tar.gz", hash = "sha256:4dc4c6c2bda0fc82cccab4aaff185a6570e13a5351d85e29e12984a55d4138ee"}, +] +geographiclib = [ + {file = "geographiclib-1.50-py3-none-any.whl", hash = "sha256:51cfa698e7183792bce27d8fb63ac8e83689cd8170a730bf35e1a5c5bf8849b9"}, + {file = "geographiclib-1.50.tar.gz", hash = "sha256:12bd46ee7ec25b291ea139b17aa991e7ef373e21abd053949b75c0e9ca55c632"}, +] +geojson = [ + {file = "geojson-2.5.0-py2.py3-none-any.whl", hash = "sha256:ccbd13368dd728f4e4f13ffe6aaf725b6e802c692ba0dde628be475040c534ba"}, + {file = "geojson-2.5.0.tar.gz", hash = "sha256:6e4bb7ace4226a45d9c8c8b1348b3fc43540658359f93c3f7e03efa9f15f658a"}, +] +gevent = [ + {file = "gevent-1.4.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:1b7d3a285978b27b469c0ff5fb5a72bcd69f4306dbbf22d7997d83209a8ba917"}, + {file = "gevent-1.4.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:44089ed06a962a3a70e96353c981d628b2d4a2f2a75ea5d90f916a62d22af2e8"}, + {file = "gevent-1.4.0-cp27-cp27m-win32.whl", hash = "sha256:0e1e5b73a445fe82d40907322e1e0eec6a6745ca3cea19291c6f9f50117bb7ea"}, + {file = "gevent-1.4.0-cp27-cp27m-win_amd64.whl", hash = "sha256:74b7528f901f39c39cdbb50cdf08f1a2351725d9aebaef212a29abfbb06895ee"}, + {file = "gevent-1.4.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:0ff2b70e8e338cf13bedf146b8c29d475e2a544b5d1fe14045aee827c073842c"}, + {file = "gevent-1.4.0-cp34-cp34m-macosx_10_14_x86_64.whl", hash = "sha256:0774babec518a24d9a7231d4e689931f31b332c4517a771e532002614e270a64"}, + {file = "gevent-1.4.0-cp34-cp34m-manylinux1_x86_64.whl", hash = "sha256:d752bcf1b98174780e2317ada12013d612f05116456133a6acf3e17d43b71f05"}, + {file = "gevent-1.4.0-cp34-cp34m-win32.whl", hash = "sha256:3249011d13d0c63bea72d91cec23a9cf18c25f91d1f115121e5c9113d753fa12"}, + {file = "gevent-1.4.0-cp34-cp34m-win_amd64.whl", hash = "sha256:d1e6d1f156e999edab069d79d890859806b555ce4e4da5b6418616322f0a3df1"}, + {file = "gevent-1.4.0-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:7d0809e2991c9784eceeadef01c27ee6a33ca09ebba6154317a257353e3af922"}, + {file = "gevent-1.4.0-cp35-cp35m-win32.whl", hash = "sha256:14b4d06d19d39a440e72253f77067d27209c67e7611e352f79fe69e0f618f76e"}, + {file = "gevent-1.4.0-cp35-cp35m-win_amd64.whl", hash = "sha256:53b72385857e04e7faca13c613c07cab411480822ac658d97fd8a4ddbaf715c8"}, + {file = "gevent-1.4.0-cp36-cp36m-macosx_10_6_intel.whl", hash = "sha256:8d9ec51cc06580f8c21b41fd3f2b3465197ba5b23c00eb7d422b7ae0380510b0"}, + {file = "gevent-1.4.0-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2711e69788ddb34c059a30186e05c55a6b611cb9e34ac343e69cf3264d42fe1c"}, + {file = "gevent-1.4.0-cp36-cp36m-win32.whl", hash = "sha256:e5bcc4270671936349249d26140c267397b7b4b1381f5ec8b13c53c5b53ab6e1"}, + {file = "gevent-1.4.0-cp36-cp36m-win_amd64.whl", hash = "sha256:9f7a1e96fec45f70ad364e46de32ccacab4d80de238bd3c2edd036867ccd48ad"}, + {file = "gevent-1.4.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:50024a1ee2cf04645535c5ebaeaa0a60c5ef32e262da981f4be0546b26791950"}, + {file = "gevent-1.4.0-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:4bfa291e3c931ff3c99a349d8857605dca029de61d74c6bb82bd46373959c942"}, + {file = "gevent-1.4.0-cp37-cp37m-win32.whl", hash = "sha256:ab4dc33ef0e26dc627559786a4fba0c2227f125db85d970abbf85b77506b3f51"}, + {file = "gevent-1.4.0-cp37-cp37m-win_amd64.whl", hash = "sha256:896b2b80931d6b13b5d9feba3d4eebc67d5e6ec54f0cf3339d08487d55d93b0e"}, + {file = "gevent-1.4.0-pp260-pypy_41-macosx_10_14_x86_64.whl", hash = "sha256:107f4232db2172f7e8429ed7779c10f2ed16616d75ffbe77e0e0c3fcdeb51a51"}, + {file = "gevent-1.4.0-pp260-pypy_41-win32.whl", hash = "sha256:28a0c5417b464562ab9842dd1fb0cc1524e60494641d973206ec24d6ec5f6909"}, + {file = "gevent-1.4.0.tar.gz", hash = "sha256:1eb7fa3b9bd9174dfe9c3b59b7a09b768ecd496debfc4976a9530a3e15c990d1"}, +] +greenlet = [ + {file = "greenlet-0.4.15-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:99a26afdb82ea83a265137a398f570402aa1f2b5dfb4ac3300c026931817b163"}, + {file = "greenlet-0.4.15-cp27-cp27m-win32.whl", hash = "sha256:beeabe25c3b704f7d56b573f7d2ff88fc99f0138e43480cecdfcaa3b87fe4f87"}, + {file = "greenlet-0.4.15-cp27-cp27m-win_amd64.whl", hash = "sha256:9854f612e1b59ec66804931df5add3b2d5ef0067748ea29dc60f0efdcda9a638"}, + {file = "greenlet-0.4.15-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:ac57fcdcfb0b73bb3203b58a14501abb7e5ff9ea5e2edfa06bb03035f0cff248"}, + {file = "greenlet-0.4.15-cp33-cp33m-win32.whl", hash = "sha256:d634a7ea1fc3380ff96f9e44d8d22f38418c1c381d5fac680b272d7d90883720"}, + {file = "greenlet-0.4.15-cp33-cp33m-win_amd64.whl", hash = "sha256:0d48200bc50cbf498716712129eef819b1729339e34c3ae71656964dac907c28"}, + {file = "greenlet-0.4.15-cp34-cp34m-manylinux1_x86_64.whl", hash = "sha256:bcb530089ff24f6458a81ac3fa699e8c00194208a724b644ecc68422e1111939"}, + {file = "greenlet-0.4.15-cp34-cp34m-win32.whl", hash = "sha256:8b4572c334593d449113f9dc8d19b93b7b271bdbe90ba7509eb178923327b625"}, + {file = "greenlet-0.4.15-cp34-cp34m-win_amd64.whl", hash = "sha256:a9f145660588187ff835c55a7d2ddf6abfc570c2651c276d3d4be8a2766db490"}, + {file = "greenlet-0.4.15-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:51503524dd6f152ab4ad1fbd168fc6c30b5795e8c70be4410a64940b3abb55c0"}, + {file = "greenlet-0.4.15-cp35-cp35m-win32.whl", hash = "sha256:a19bf883b3384957e4a4a13e6bd1ae3d85ae87f4beb5957e35b0be287f12f4e4"}, + {file = "greenlet-0.4.15-cp35-cp35m-win_amd64.whl", hash = "sha256:853da4f9563d982e4121fed8c92eea1a4594a2299037b3034c3c898cb8e933d6"}, + {file = "greenlet-0.4.15-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:23d12eacffa9d0f290c0fe0c4e81ba6d5f3a5b7ac3c30a5eaf0126bf4deda5c8"}, + {file = "greenlet-0.4.15-cp36-cp36m-win32.whl", hash = "sha256:000546ad01e6389e98626c1367be58efa613fa82a1be98b0c6fc24b563acc6d0"}, + {file = "greenlet-0.4.15-cp36-cp36m-win_amd64.whl", hash = "sha256:d97b0661e1aead761f0ded3b769044bb00ed5d33e1ec865e891a8b128bf7c656"}, + {file = "greenlet-0.4.15-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:8041e2de00e745c0e05a502d6e6db310db7faa7c979b3a5877123548a4c0b214"}, + {file = "greenlet-0.4.15-cp37-cp37m-win32.whl", hash = "sha256:81fcd96a275209ef117e9ec91f75c731fa18dcfd9ffaa1c0adbdaa3616a86043"}, + {file = "greenlet-0.4.15-cp37-cp37m-win_amd64.whl", hash = "sha256:37c9ba82bd82eb6a23c2e5acc03055c0e45697253b2393c9a50cef76a3985304"}, + {file = "greenlet-0.4.15.tar.gz", hash = "sha256:9416443e219356e3c31f1f918a91badf2e37acf297e2fa13d24d1cc2380f8fbc"}, +] +gunicorn = [ + {file = "gunicorn-19.9.0-py2.py3-none-any.whl", hash = "sha256:aa8e0b40b4157b36a5df5e599f45c9c76d6af43845ba3b3b0efe2c70473c2471"}, + {file = "gunicorn-19.9.0.tar.gz", hash = "sha256:fa2662097c66f920f53f70621c6c58ca4a3c4d3434205e608e121b5b3b71f4f3"}, +] +idna = [ + {file = "idna-2.8-py2.py3-none-any.whl", hash = "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c"}, + {file = "idna-2.8.tar.gz", hash = "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407"}, +] +itsdangerous = [ + {file = "itsdangerous-1.1.0-py2.py3-none-any.whl", hash = "sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749"}, + {file = "itsdangerous-1.1.0.tar.gz", hash = "sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19"}, +] +jinja2 = [ + {file = "Jinja2-2.10.3-py2.py3-none-any.whl", hash = "sha256:74320bb91f31270f9551d46522e33af46a80c3d619f4a4bf42b3164d30b5911f"}, + {file = "Jinja2-2.10.3.tar.gz", hash = "sha256:9fe95f19286cfefaa917656583d020be14e7859c6b0252588391e47db34527de"}, +] +jmespath = [ + {file = "jmespath-0.9.4-py2.py3-none-any.whl", hash = "sha256:3720a4b1bd659dd2eecad0666459b9788813e032b83e7ba58578e48254e0a0e6"}, + {file = "jmespath-0.9.4.tar.gz", hash = "sha256:bde2aef6f44302dfb30320115b17d030798de8c4110e28d5cf6cf91a7a31074c"}, +] +joerd = [] +jsonschema = [ + {file = "jsonschema-2.6.0-py2.py3-none-any.whl", hash = "sha256:000e68abd33c972a5248544925a0cae7d1125f9bf6c58280d37546b946769a08"}, + {file = "jsonschema-2.6.0.tar.gz", hash = "sha256:6ff5f3180870836cae40f06fa10419f557208175f13ad7bc26caa77beb1f6e02"}, +] +markupsafe = [ + {file = "MarkupSafe-1.1.1-cp27-cp27m-macosx_10_6_intel.whl", hash = "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161"}, + {file = "MarkupSafe-1.1.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7"}, + {file = "MarkupSafe-1.1.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183"}, + {file = "MarkupSafe-1.1.1-cp27-cp27m-win32.whl", hash = "sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b"}, + {file = "MarkupSafe-1.1.1-cp27-cp27m-win_amd64.whl", hash = "sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e"}, + {file = "MarkupSafe-1.1.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f"}, + {file = "MarkupSafe-1.1.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1"}, + {file = "MarkupSafe-1.1.1-cp34-cp34m-macosx_10_6_intel.whl", hash = "sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5"}, + {file = "MarkupSafe-1.1.1-cp34-cp34m-manylinux1_i686.whl", hash = "sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1"}, + {file = "MarkupSafe-1.1.1-cp34-cp34m-manylinux1_x86_64.whl", hash = "sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735"}, + {file = "MarkupSafe-1.1.1-cp34-cp34m-win32.whl", hash = "sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21"}, + {file = "MarkupSafe-1.1.1-cp34-cp34m-win_amd64.whl", hash = "sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235"}, + {file = "MarkupSafe-1.1.1-cp35-cp35m-macosx_10_6_intel.whl", hash = "sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b"}, + {file = "MarkupSafe-1.1.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f"}, + {file = "MarkupSafe-1.1.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905"}, + {file = "MarkupSafe-1.1.1-cp35-cp35m-win32.whl", hash = "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1"}, + {file = "MarkupSafe-1.1.1-cp35-cp35m-win_amd64.whl", hash = "sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d"}, + {file = "MarkupSafe-1.1.1-cp36-cp36m-macosx_10_6_intel.whl", hash = "sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff"}, + {file = "MarkupSafe-1.1.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473"}, + {file = "MarkupSafe-1.1.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e"}, + {file = "MarkupSafe-1.1.1-cp36-cp36m-win32.whl", hash = "sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66"}, + {file = "MarkupSafe-1.1.1-cp36-cp36m-win_amd64.whl", hash = "sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5"}, + {file = "MarkupSafe-1.1.1-cp37-cp37m-macosx_10_6_intel.whl", hash = "sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d"}, + {file = "MarkupSafe-1.1.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e"}, + {file = "MarkupSafe-1.1.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6"}, + {file = "MarkupSafe-1.1.1-cp37-cp37m-win32.whl", hash = "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2"}, + {file = "MarkupSafe-1.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c"}, + {file = "MarkupSafe-1.1.1.tar.gz", hash = "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b"}, +] +mistune = [ + {file = "mistune-0.8.4-py2.py3-none-any.whl", hash = "sha256:88a1051873018da288eee8538d476dffe1262495144b33ecb586c4ab266bb8d4"}, + {file = "mistune-0.8.4.tar.gz", hash = "sha256:59a3429db53c50b5c6bcc8a07f8848cb00d7dc8bdb431a4ab41920d201d4756e"}, +] +numpy = [ + {file = "numpy-1.17.3-cp35-cp35m-macosx_10_6_intel.whl", hash = "sha256:4dd830a11e8724c9c9379feed1d1be43113f8bcce55f47ea7186d3946769ce26"}, + {file = "numpy-1.17.3-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:30c84e3a62cfcb9e3066f25226e131451312a044f1fe2040e69ce792cb7de418"}, + {file = "numpy-1.17.3-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9395b0a41e8b7e9a284e3be7060db9d14ad80273841c952c83a5afc241d2bd98"}, + {file = "numpy-1.17.3-cp35-cp35m-win32.whl", hash = "sha256:9e37c35fc4e9410093b04a77d11a34c64bf658565e30df7cbe882056088a91c1"}, + {file = "numpy-1.17.3-cp35-cp35m-win_amd64.whl", hash = "sha256:de2b1c20494bdf47f0160bd88ed05f5e48ae5dc336b8de7cfade71abcc95c0b9"}, + {file = "numpy-1.17.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:669795516d62f38845c7033679c648903200980d68935baaa17ac5c7ae03ae0c"}, + {file = "numpy-1.17.3-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:4650d94bb9c947151737ee022b934b7d9a845a7c76e476f3e460f09a0c8c6f39"}, + {file = "numpy-1.17.3-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4f2a2b279efde194877aff1f76cf61c68e840db242a5c7169f1ff0fd59a2b1e2"}, + {file = "numpy-1.17.3-cp36-cp36m-win32.whl", hash = "sha256:ffca69e29079f7880c5392bf675eb8b4146479d976ae1924d01cd92b04cccbcc"}, + {file = "numpy-1.17.3-cp36-cp36m-win_amd64.whl", hash = "sha256:2e418f0a59473dac424f888dd57e85f77502a593b207809211c76e5396ae4f5c"}, + {file = "numpy-1.17.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:75fcd60d682db3e1f8fbe2b8b0c6761937ad56d01c1dc73edf4ef2748d5b6bc4"}, + {file = "numpy-1.17.3-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:28b1180c758abf34a5c3fea76fcee66a87def1656724c42bb14a6f9717a5bdf7"}, + {file = "numpy-1.17.3-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:dd0667f5be56fb1b570154c2c0516a528e02d50da121bbbb2cbb0b6f87f59bc2"}, + {file = "numpy-1.17.3-cp37-cp37m-win32.whl", hash = "sha256:25ffe71f96878e1da7e014467e19e7db90ae7d4e12affbc73101bcf61785214e"}, + {file = "numpy-1.17.3-cp37-cp37m-win_amd64.whl", hash = "sha256:0b0dd8f47fb177d00fa6ef2d58783c4f41ad3126b139c91dd2f7c4b3fdf5e9a5"}, + {file = "numpy-1.17.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:62d22566b3e3428dfc9ec972014c38ed9a4db4f8969c78f5414012ccd80a149e"}, + {file = "numpy-1.17.3-cp38-cp38-manylinux1_i686.whl", hash = "sha256:26efd7f7d755e6ca966a5c0ac5a930a87dbbaab1c51716ac26a38f42ecc9bc4b"}, + {file = "numpy-1.17.3-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:b46554ad4dafb2927f88de5a1d207398c5385edbb5c84d30b3ef187c4a3894d8"}, + {file = "numpy-1.17.3-cp38-cp38-win32.whl", hash = "sha256:c867eeccd934920a800f65c6068acdd6b87e80d45cd8c8beefff783b23cdc462"}, + {file = "numpy-1.17.3-cp38-cp38-win_amd64.whl", hash = "sha256:f1df7b2b7740dd777571c732f98adb5aad5450aee32772f1b39249c8a50386f6"}, + {file = "numpy-1.17.3.zip", hash = "sha256:a0678793096205a4d784bd99f32803ba8100f639cf3b932dc63b21621390ea7e"}, +] +psycopg2 = [ + {file = "psycopg2-2.8.4-cp27-cp27m-win32.whl", hash = "sha256:72772181d9bad1fa349792a1e7384dde56742c14af2b9986013eb94a240f005b"}, + {file = "psycopg2-2.8.4-cp27-cp27m-win_amd64.whl", hash = "sha256:893c11064b347b24ecdd277a094413e1954f8a4e8cdaf7ffbe7ca3db87c103f0"}, + {file = "psycopg2-2.8.4-cp34-cp34m-win32.whl", hash = "sha256:9ab75e0b2820880ae24b7136c4d230383e07db014456a476d096591172569c38"}, + {file = "psycopg2-2.8.4-cp34-cp34m-win_amd64.whl", hash = "sha256:b0845e3bdd4aa18dc2f9b6fb78fbd3d9d371ad167fd6d1b7ad01c0a6cdad4fc6"}, + {file = "psycopg2-2.8.4-cp35-cp35m-win32.whl", hash = "sha256:ef6df7e14698e79c59c7ee7cf94cd62e5b869db369ed4b1b8f7b729ea825712a"}, + {file = "psycopg2-2.8.4-cp35-cp35m-win_amd64.whl", hash = "sha256:965c4c93e33e6984d8031f74e51227bd755376a9df6993774fd5b6fb3288b1f4"}, + {file = "psycopg2-2.8.4-cp36-cp36m-win32.whl", hash = "sha256:ed686e5926929887e2c7ae0a700e32c6129abb798b4ad2b846e933de21508151"}, + {file = "psycopg2-2.8.4-cp36-cp36m-win_amd64.whl", hash = "sha256:dca2d7203f0dfce8ea4b3efd668f8ea65cd2b35112638e488a4c12594015f67b"}, + {file = "psycopg2-2.8.4-cp37-cp37m-win32.whl", hash = "sha256:8396be6e5ff844282d4d49b81631772f80dabae5658d432202faf101f5283b7c"}, + {file = "psycopg2-2.8.4-cp37-cp37m-win_amd64.whl", hash = "sha256:47fc642bf6f427805daf52d6e52619fe0637648fe27017062d898f3bf891419d"}, + {file = "psycopg2-2.8.4.tar.gz", hash = "sha256:f898e5cc0a662a9e12bde6f931263a1bbd350cfb18e1d5336a12927851825bb6"}, +] +pycparser = [ + {file = "pycparser-2.19.tar.gz", hash = "sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3"}, +] +pyqtree = [ + {file = "Pyqtree-1.0.0.tar.gz", hash = "sha256:4f36d5160ddf170d7245e9c7102a45211b85003383dd552b6cd109e50cc3af81"}, +] +python-dateutil = [ + {file = "python-dateutil-2.8.0.tar.gz", hash = "sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e"}, + {file = "python_dateutil-2.8.0-py2.py3-none-any.whl", hash = "sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb"}, +] +pyyaml = [ + {file = "PyYAML-5.1.2-cp27-cp27m-win32.whl", hash = "sha256:5124373960b0b3f4aa7df1707e63e9f109b5263eca5976c66e08b1c552d4eaf8"}, + {file = "PyYAML-5.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:f81025eddd0327c7d4cfe9b62cf33190e1e736cc6e97502b3ec425f574b3e7a8"}, + {file = "PyYAML-5.1.2-cp34-cp34m-win32.whl", hash = "sha256:0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9"}, + {file = "PyYAML-5.1.2-cp34-cp34m-win_amd64.whl", hash = "sha256:5ca4f10adbddae56d824b2c09668e91219bb178a1eee1faa56af6f99f11bf696"}, + {file = "PyYAML-5.1.2-cp35-cp35m-win32.whl", hash = "sha256:bf47c0607522fdbca6c9e817a6e81b08491de50f3766a7a0e6a5be7905961b41"}, + {file = "PyYAML-5.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:87ae4c829bb25b9fe99cf71fbb2140c448f534e24c998cc60f39ae4f94396a73"}, + {file = "PyYAML-5.1.2-cp36-cp36m-win32.whl", hash = "sha256:9de9919becc9cc2ff03637872a440195ac4241c80536632fffeb6a1e25a74299"}, + {file = "PyYAML-5.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:a5a85b10e450c66b49f98846937e8cfca1db3127a9d5d1e31ca45c3d0bef4c5b"}, + {file = "PyYAML-5.1.2-cp37-cp37m-win32.whl", hash = "sha256:b0997827b4f6a7c286c01c5f60384d218dca4ed7d9efa945c3e1aa623d5709ae"}, + {file = "PyYAML-5.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:7907be34ffa3c5a32b60b95f4d95ea25361c951383a894fec31be7252b2b6f34"}, + {file = "PyYAML-5.1.2-cp38-cp38m-win32.whl", hash = "sha256:7ec9b2a4ed5cad025c2278a1e6a19c011c80a3caaac804fd2d329e9cc2c287c9"}, + {file = "PyYAML-5.1.2-cp38-cp38m-win_amd64.whl", hash = "sha256:b631ef96d3222e62861443cc89d6563ba3eeb816eeb96b2629345ab795e53681"}, + {file = "PyYAML-5.1.2.tar.gz", hash = "sha256:01adf0b6c6f61bd11af6e10ca52b7d4057dd0be0343eb9283c878cf3af56aee4"}, +] +requests = [ + {file = "requests-2.22.0-py2.py3-none-any.whl", hash = "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31"}, + {file = "requests-2.22.0.tar.gz", hash = "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4"}, +] +s3transfer = [ + {file = "s3transfer-0.2.1-py2.py3-none-any.whl", hash = "sha256:b780f2411b824cb541dbcd2c713d0cb61c7d1bcadae204cdddda2b35cef493ba"}, + {file = "s3transfer-0.2.1.tar.gz", hash = "sha256:6efc926738a3cd576c2a79725fed9afde92378aa5c6a957e3af010cb019fac9d"}, +] +shapely = [ + {file = "Shapely-1.6.4.post2-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:3ca69d4b12e2b05b549465822744b6a3a1095d8488cc27b2728a06d3c07d0eee"}, + {file = "Shapely-1.6.4.post2-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:714b6680215554731389a1bbdae4cec61741aa4726921fa2b2b96a6f578a2534"}, + {file = "Shapely-1.6.4.post2-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:5d22a1a705c2f70f61ccadc696e33d922c1a92e00df8e1d58a6ade14dd7e3b4f"}, + {file = "Shapely-1.6.4.post2-cp34-cp34m-macosx_10_9_intel.macosx_10_9_x86_64.whl", hash = "sha256:34e7c6f41fb27906ccdf2514ee44a5774b90b39a256b6511a6a57d11ffe64999"}, + {file = "Shapely-1.6.4.post2-cp34-cp34m-manylinux1_x86_64.whl", hash = "sha256:3e9388f29bd81fcd4fa5c35125e1fbd4975ee36971a87a90c093f032d0e9de24"}, + {file = "Shapely-1.6.4.post2-cp35-cp35m-macosx_10_9_intel.macosx_10_9_x86_64.whl", hash = "sha256:0378964902f89b8dbc332e5bdfa08e0bc2f7ab39fecaeb17fbb2a7699a44fe71"}, + {file = "Shapely-1.6.4.post2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:523c94403047eb6cacd7fc1863ebef06e26c04d8a4e7f8f182d49cd206fe787e"}, + {file = "Shapely-1.6.4.post2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:ba58b21b9cf3c33725f7f530febff9ed6a6846f9d0bf8a120fc74683ff919f89"}, + {file = "Shapely-1.6.4.post2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:ebb4d2bee7fac3f6c891fcdafaa17f72ab9c6480f6d00de0b2dc9a5137dfe342"}, + {file = "Shapely-1.6.4.post2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:7dfe1528650c3f0dc82f41a74cf4f72018288db9bfb75dcd08f6f04233ec7e78"}, + {file = "Shapely-1.6.4.post2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:3ef28e3f20a1c37f5b99ea8cf8dcb58e2f1a8762d65ed2d21fd92bf1d4811182"}, + {file = "Shapely-1.6.4.post2.tar.gz", hash = "sha256:c4b87bb61fc3de59fc1f85e71a79b0c709dc68364d9584473697aad4aa13240f"}, +] +six = [ + {file = "six-1.12.0-py2.py3-none-any.whl", hash = "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c"}, + {file = "six-1.12.0.tar.gz", hash = "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"}, +] +soupsieve = [ + {file = "soupsieve-1.9.4-py2.py3-none-any.whl", hash = "sha256:b91d676b330a0ebd5b21719cb6e9b57c57d433671f65b9c28dd3461d9a1ed0b6"}, + {file = "soupsieve-1.9.4.tar.gz", hash = "sha256:605f89ad5fdbfefe30cdc293303665eff2d188865d4dbe4eb510bba1edfbfce3"}, +] +sqlalchemy = [ + {file = "SQLAlchemy-1.3.10.tar.gz", hash = "sha256:0f0768b5db594517e1f5e1572c73d14cf295140756431270d89496dc13d5e46c"}, +] +urllib3 = [ + {file = "urllib3-1.25.6-py2.py3-none-any.whl", hash = "sha256:3de946ffbed6e6746608990594d08faac602528ac7015ac28d33cee6a45b7398"}, + {file = "urllib3-1.25.6.tar.gz", hash = "sha256:9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86"}, +] +werkzeug = [ + {file = "Werkzeug-0.16.0-py2.py3-none-any.whl", hash = "sha256:e5f4a1f98b52b18a93da705a7458e55afb26f32bff83ff5d19189f92462d65c4"}, + {file = "Werkzeug-0.16.0.tar.gz", hash = "sha256:7280924747b5733b246fe23972186c6b348f9ae29724135a6dfc1e53cea433e7"}, +] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..5e17ab4 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,40 @@ +[tool.poetry] +name = "openelevationservice" +version = "0.2.1" +description = "Flask app to serve elevation data to GeoJSON queries." +authors = ["Nils Nolde "] +license = "MIT" +repository = "https://github.com/GIScience/openelevationservice" +readme = "README.rst" +keywords = ["flask", "elevation", "GIS", "GeoJSON,", "ORS"] + +[tool.poetry.dependencies] +python = "^3.6" +beautifulsoup4 = "^4.8" +GeoAlchemy2 = "^0.6.3" +geojson = "^2.5" +shapely = "^1.6" +sqlalchemy = "^1.3" +werkzeug = "^0.16.0" +pyyaml = "^5.1" +flasgger = "^0.9.3" +gunicorn = "^19.9" +gevent = "^1.4" +requests = "^2.22" +psycopg2 = "^2.8" +Flask = "^1.1" +Flask_Cors = "^3.0" +Flask-SQLAlchemy = "^2.4" +numpy = "^1.17" +pyqtree = "^1.0" +geographiclib = "^1.50" +boto3 = "^1.10" +contextlib2 = "^0.6.0" +Cerberus = "^1.3.1" + +[tool.poetry.dev-dependencies] +#python = "" + +[build-system] +requires = ["poetry>=0.12"] +build-backend = "poetry.masonry.api" \ No newline at end of file diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 4fe03c3..0000000 --- a/requirements.txt +++ /dev/null @@ -1,18 +0,0 @@ -Flask>=1.0.0 -Flask_Cors>=3.0.0 -Flask_Testing>0.7.0 -Flask-SQLAlchemy>=2.3.0 -Cerberus>=1.2 -beautifulsoup4>=4.6.0 -GeoAlchemy2>=0.5.0 -geojson>=2.4.0 -shapely>=1.6.0 -sqlalchemy>=1.2.0 -werkzeug>=0.14.0 -pyyaml>=4.2b1 -flasgger>=0.9.0 -gunicorn>=19.0.0 -gevent>=1.3.0 -requests>=2.20.0 -nose>=1.3.0 -psycopg2>=2.7.5 \ No newline at end of file diff --git a/setup.py b/setup.py deleted file mode 100644 index 1dc8ceb..0000000 --- a/setup.py +++ /dev/null @@ -1,65 +0,0 @@ -# -*- coding: utf-8 -*- -import sys - - -try: - from setuptools import setup -except ImportError: - from distutils.core import setup - -def readme(): - with open('README.rst') as f: - return f.read() - -if sys.version_info <= (3, 5): - error = 'Requires Python Version 3.6 or above... exiting.' - print >> sys.stderr, error - sys.exit(1) - -setup( - name='openelevationservice', - version='0.2.1', - description='Flask app to serve elevation data to GeoJSON queries.', - long_description=readme(), - classifiers=[ - 'Development Status :: 4 - Beta', - 'License :: OSI Approved :: MIT License', - 'Operating System :: OS Independent', - 'Programming Language :: Python :: 3.7', - ], - keywords='flask elevation GIS GeoJSON ORS SRTM', - url='https://github.com/GIScience/openelevationservice', - author='Nils Nolde', - author_email='nils.nolde@gmail.com', - license='MIT', - packages=['openelevationservice'], - install_requires=[ - 'Flask>=1.0.0', - 'Flask_Cors>=3.0.0', - 'Flask-SQLAlchemy>=2.3.0', - 'Cerberus>=1.2', - 'beautifulsoup4>=4.6.0', - 'GeoAlchemy2>=0.5.0', - 'geojson>=2.4.0', - 'shapely>=1.6.0', - 'sqlalchemy>=1.2.0', - 'werkzeug>=0.14.0', - 'pyyaml>=4.2b1', - 'flasgger>=0.9.0', - 'gunicorn>=19.0.0', - 'gevent>=1.3.0', - 'requests>=2.20.0', - 'psycopg2>2.7.5' - ], - include_package_data=True, - test_suite='nose.collector', - tests_require=[ - 'nose>1.3.0', - 'Flask_Testing>=0.7.0', - ], - zip_safe=False, - project_urls={ - 'Bug Reports': 'https://github.com/GIScience/openelevationservice/issues', - 'Source': 'https://github.com/GIScience/openelevationservice', - } -) From 047ba47c59ed4a95ca43bf9270549e121f5546a5 Mon Sep 17 00:00:00 2001 From: isabell <36471321+isikl@users.noreply.github.com> Date: Wed, 27 Nov 2019 15:49:36 +0100 Subject: [PATCH 04/11] Update ops_settings_docker.yml --- ops_settings_docker.sample.yml | 42 +++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/ops_settings_docker.sample.yml b/ops_settings_docker.sample.yml index 0459020..7d1ddcd 100644 --- a/ops_settings_docker.sample.yml +++ b/ops_settings_docker.sample.yml @@ -1,14 +1,40 @@ --- -attribution: "service by https://openrouteservice.org | data by http://srtm.csi.cgiar.org" +attribution: "service by https://openrouteservice.org | data by http://srtm.csi.cgiar.org or see https://github.com/tilezen/joerd_docker" coord_precision: 1e-6 maximum_nodes: 2000 -srtm_parameters: - user: - password: provider_parameters: - table_name: oes_cgiar - db_name: - user_name: - password: + table_name_srtm: srtm + table_name_composite: composite + db_name: oes_db + user_name: gis + password: gis host: localhost port: 5432 +tables: + - srtm: + table_name: srtm + sources: + - srtm: + url: http://data.cgiar-csi.org/srtm/tiles/GeoTIFF/ + srtm_parameters: + user: user + password: password + - gmted: + url: http://edcintl.cr.usgs.gov/downloads/sciweb1/shared/topo/downloads/GMTED/Global_tiles_GMTED + ys: [-90, -70, -50, -30, -10, 10, 30, 50, 70] + xs: [-180, -150, -120, -90, -60, -30, 0, 30, 60, 90, 120, 150] + extent: + min_x: -1.406250 + min_y: 57.088515 + max_x: 15.820313 + max_y: 64.091408 + - composite: + table_name: composite + sources: + - etopo1: + url: https://www.ngdc.noaa.gov/mgg/global/relief/ETOPO1/data/bedrock/grid_registered/georeferenced_tiff/ETOPO1_Bed_g_geotiff.zip + extent: + min_x: 8.662891 + min_y: 49.395223 + max_x: 8.709412 + max_y: 49.423927 From 8eff0c567a4dc33e1867a1b5433c381b3a50cf0d Mon Sep 17 00:00:00 2001 From: Isabell Klipper Date: Mon, 2 Dec 2019 12:23:32 +0100 Subject: [PATCH 05/11] fixed input/output streaming --- .../server/db_import/filestreams.py | 18 ++-- .../server/db_import/raster_processing.py | 84 +++++++++++-------- openelevationservice/server/sources/srtm.py | 4 +- ops_settings_docker.sample.yml | 14 ---- 4 files changed, 61 insertions(+), 59 deletions(-) delete mode 100644 ops_settings_docker.sample.yml diff --git a/openelevationservice/server/db_import/filestreams.py b/openelevationservice/server/db_import/filestreams.py index 3406110..849bbaf 100644 --- a/openelevationservice/server/db_import/filestreams.py +++ b/openelevationservice/server/db_import/filestreams.py @@ -25,8 +25,8 @@ def downloadsrtm(): # merge and clip raster by extent log.info("Starting tile processing ...") - raster_processing.merge_raster('srtm_*', '/srtm_merged.tif') - raster_processing.clip_raster('/srtm_merged.tif', '/srtm_final.tif') + merged_filename = raster_processing.merge_raster('srtm_*', 'srtm_merged.tif') + raster_processing.clip_raster(merged_filename, '/srtm_final.tif') elif extent_settings['max_y'] > 60 >= extent_settings['min_y']: @@ -38,15 +38,15 @@ def downloadsrtm(): # resample and merge tiles log.info("Starting tile preprocessing ...") - raster_processing.merge_raster('srtm_*', '/srtm_merged.tif') - raster_processing.clip_raster('/srtm_merged.tif', '/srtm_clipped.tif') - raster_processing.merge_raster('*_gmted_mea075.tif', '/gmted_merged.tif') + srtm_merged_filename = raster_processing.merge_raster('srtm_*', 'srtm_merged.tif') + raster_processing.clip_raster(srtm_merged_filename, 'srtm_clipped.tif') + gmted_merged_filename = raster_processing.merge_raster('*_gmted_mea075.tif', 'gmted_merged.tif') log.info("Starting tile resampling ...") - raster_processing.gmted_resampling() + raster_processing.gmted_resampling(gmted_merged_filename, 'srtm_clipped.tif', 'gmted_resampled.tif') log.info("Starting tile merging ...") - raster_processing.merge_raster('gmted_resampled.tif', '/raster_final.tif', '/srtm_clipped.tif') + raster_processing.merge_raster('gmted_resampled.tif', 'raster_final.tif', 'srtm_clipped.tif') else: @@ -56,8 +56,8 @@ def downloadsrtm(): # merge and clip raster by extent log.info("Starting tile processing ...") - raster_processing.merge_raster('*_gmted_mea075.tif', '/gmted_merged.tif') - raster_processing.clip_raster('/gmted_merged.tif', '/gmted_final.tif') + merged_filename = raster_processing.merge_raster('*_gmted_mea075.tif', 'gmted_merged.tif') + raster_processing.clip_raster(merged_filename, 'gmted_final.tif') def raster2pgsql(): diff --git a/openelevationservice/server/db_import/raster_processing.py b/openelevationservice/server/db_import/raster_processing.py index 65ee40d..198f9b2 100644 --- a/openelevationservice/server/db_import/raster_processing.py +++ b/openelevationservice/server/db_import/raster_processing.py @@ -4,33 +4,46 @@ from osgeo import gdal, gdalconst import subprocess -from os import path +from os import path, listdir +import fnmatch -# TODO: if files already exists ... # TODO: handle in memory files def merge_raster(input_filename, output_filename, reference=None): """ Merge downloaded single tiles to one raster tile. """ - output_merge = path.join(TILES_DIR + output_filename) input_files = path.join(TILES_DIR + '/' + input_filename) + output_merge = path.join(TILES_DIR + '/' + output_filename) - if not path.exists(path.join(TILES_DIR, output_merge)): + if not path.exists(path.join(TILES_DIR, output_filename)): if reference is None: - raster_merge = r"/usr/bin/gdal_merge.py -o {outfile} -of {outfile_format} {input_files}" + if len(fnmatch.filter(listdir(TILES_DIR), input_filename)) > 1: + + raster_merge = r"/usr/bin/gdal_merge.py -o {outfile} -of {outfile_format} {input_files}" + + cmd_merge = raster_merge.format(**{'outfile': output_merge, + 'outfile_format': 'GTiff', + 'input_files': input_files}) + + proc = subprocess.Popen(cmd_merge, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + shell=True) - cmd_merge = raster_merge.format(**{'outfile': output_merge, - 'outfile_format': 'GTiff', - 'input_files': input_files}) + return_code = proc.wait() + if return_code: + raise subprocess.CalledProcessError(return_code, cmd_merge) + + else: + return input_filename else: # merge srtm and gmted tile fractions - output_merge = path.join(TILES_DIR + output_filename) - reference_file = path.join(TILES_DIR + reference) + reference_file = path.join(TILES_DIR + '/' + reference) # -tap: align tiles # reference_file: In areas of overlap, the last image will be copied over earlier ones. @@ -41,35 +54,37 @@ def merge_raster(input_filename, output_filename, reference=None): 'input_files': input_files, # gmted 'reference_file': reference_file}) # srtm - proc = subprocess.Popen(cmd_merge, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - shell=True) + proc = subprocess.Popen(cmd_merge, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + shell=True) + + return_code = proc.wait() + if return_code: + raise subprocess.CalledProcessError(return_code, cmd_merge) - return_code = proc.wait() - if return_code: - raise subprocess.CalledProcessError(return_code, cmd_merge) + return output_filename -def clip_raster(merged_filename, output_filename): +def clip_raster(merged_filename_extent, output_filename): """ Clip merged raster by defined extent. """ if not path.exists(path.join(TILES_DIR, output_filename)): - output_clip = path.join(TILES_DIR + output_filename) - merged_file = path.join(TILES_DIR + merged_filename) + output_clip = path.join(TILES_DIR + '/' + output_filename) + merged_filename = fnmatch.filter(listdir(TILES_DIR), merged_filename_extent)[0] + merged_file = path.join(TILES_DIR + '/' + merged_filename) merged_data = gdal.Open(merged_file, gdalconst.GA_ReadOnly) - extent = list(SETTINGS['tables'][0]['srtm']['extent'].values()) + extent = str(list(SETTINGS['tables'][0]['srtm']['extent'].values()))[1:-1] - # TODO: -t_srs {target_spatial_ref} - srtm_clip = r"/usr/bin/gdalwarp -t_srs {target_spatial_ref} -dstnodata -9999 -te {extent} -of {outfile_format} {input_file} {out_clipped_file}" + clip = r"/usr/bin/gdalwarp -t_srs {target_spatial_ref} -dstnodata -9999 -te {extent} -of {outfile_format} {input_file} {out_clipped_file}" - cmd_clip = srtm_clip.format(**{'target_spatial_ref': merged_data.GetProjection(), - 'extent': extent, - 'outfile_format': 'GTiff', - 'input_file': merged_file, - 'out_clipped_file': output_clip}) + cmd_clip = clip.format(**{'target_spatial_ref': merged_data.GetProjection(), + 'extent': extent, + 'outfile_format': 'GTiff', + 'input_file': merged_file, + 'out_clipped_file': output_clip}) proc_clip = subprocess.Popen(cmd_clip, stdout=subprocess.PIPE, @@ -81,22 +96,21 @@ def clip_raster(merged_filename, output_filename): raise subprocess.CalledProcessError(return_code_clip, cmd_clip) -def gmted_resampling(): +def gmted_resampling(gmted_merged, srtm_clipped, output_filename): """ Resample merged GMTED raster to SRTM resolution. """ - output_resampled = path.join(TILES_DIR + '/gmted_resampled.tif') - - if not path.exists(path.join(TILES_DIR, output_resampled)): + if not path.exists(path.join(TILES_DIR, output_filename)): - gmted_merged = path.join(TILES_DIR + '/gmted_merged.tif') + output_resampled = path.join(TILES_DIR + '/' + output_filename) + gmted_merged = path.join(TILES_DIR + '/' + gmted_merged) - srtm_clipped = path.join(TILES_DIR + '/srtm_clipped.tif') + srtm_clipped = path.join(TILES_DIR + '/' + srtm_clipped) reference_data = gdal.Open(srtm_clipped, gdalconst.GA_ReadOnly) # desired resolution x_res = reference_data.GetGeoTransform()[1] y_res = reference_data.GetGeoTransform()[5] - extent = list(SETTINGS['tables'][0]['srtm']['extent'].values()) + extent = str(list(SETTINGS['tables'][0]['srtm']['extent'].values()))[1:-1] # TODO: need extent? resampling = r"/usr/bin/gdalwarp -t_srs {target_spatial_ref} -dstnodata -9999 -te {extent} -tr {x_res} {y_res} -r {resampling_method} -of {outfile_format} {input_file} {out_resampling_file}" diff --git a/openelevationservice/server/sources/srtm.py b/openelevationservice/server/sources/srtm.py index 9c3179f..a24908a 100644 --- a/openelevationservice/server/sources/srtm.py +++ b/openelevationservice/server/sources/srtm.py @@ -68,7 +68,9 @@ def srtm_y_value(y_min, y_max): else: lon_min -= 5 lon_max -= 5 - y_srtm_value -= 1 + y_srtm_value += 1 + + return y_value_list def download_srtm(): diff --git a/ops_settings_docker.sample.yml b/ops_settings_docker.sample.yml deleted file mode 100644 index 0459020..0000000 --- a/ops_settings_docker.sample.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -attribution: "service by https://openrouteservice.org | data by http://srtm.csi.cgiar.org" -coord_precision: 1e-6 -maximum_nodes: 2000 -srtm_parameters: - user: - password: -provider_parameters: - table_name: oes_cgiar - db_name: - user_name: - password: - host: localhost - port: 5432 From 57cb9a6e70688192260578c8d69ed7ce1f991893 Mon Sep 17 00:00:00 2001 From: Isabell Klipper Date: Mon, 2 Dec 2019 17:45:22 +0100 Subject: [PATCH 06/11] [WP] Added abstract ProviderClass(abc) class --- manage.py | 2 +- openelevationservice/__init__.py | 4 +- openelevationservice/server/__init__.py | 7 +- .../server/db_import/filestreams.py | 77 ++++--- .../server/db_import/models.py | 8 +- .../server/db_import/raster_processing.py | 4 +- .../server/ops_settings.sample.yml | 90 ++++---- .../server/sources/__init__.py | 7 + openelevationservice/server/sources/gmted.py | 53 ++--- .../server/sources/provider.py | 19 ++ openelevationservice/server/sources/srtm.py | 201 +++++++++--------- ops_settings_docker.sample.yml | 71 ++++--- 12 files changed, 296 insertions(+), 247 deletions(-) create mode 100644 openelevationservice/server/sources/provider.py diff --git a/manage.py b/manage.py index 7c88b2a..3d49525 100644 --- a/manage.py +++ b/manage.py @@ -21,7 +21,7 @@ def download(): :type xyrange: comma-separated integers """ - filestreams.downloadsrtm() + filestreams.download() log.info("Downloaded all files") diff --git a/openelevationservice/__init__.py b/openelevationservice/__init__.py index e675b2e..65d41df 100644 --- a/openelevationservice/__init__.py +++ b/openelevationservice/__init__.py @@ -9,10 +9,8 @@ TILES_DIR = path.join(getcwd(), 'tiles') if "TESTING" in environ: - SETTINGS['provider_parameters']['table_name_srtm'] = SETTINGS['provider_parameters']['table_name_srtm'] + '_test' + SETTINGS['provider_parameters']['table_name_terrestrial'] = SETTINGS['provider_parameters']['table_name_terrestrial'] + '_test' TILES_DIR = path.join(basedir, 'tests', 'tile') - # if "CI" in environ: - # SETTINGS['provider_parameters']['port'] = 5433 if not path.exists(TILES_DIR): makedirs(TILES_DIR) diff --git a/openelevationservice/server/__init__.py b/openelevationservice/server/__init__.py index 895188d..6c056e8 100644 --- a/openelevationservice/server/__init__.py +++ b/openelevationservice/server/__init__.py @@ -12,6 +12,7 @@ log = logger.get_logger(__name__) + def create_app(script_info=None): # instantiate the app @@ -37,8 +38,8 @@ def create_app(script_info=None): log.info("Following provider parameters are active:\n" "Host:\t{host}\n" "DB:\t{db_name}\n" - "Table1:\t{table_name_srtm}\n" - "Table2:\t{table_name_composite}\n" + "Table1:\t{table_name_terrestrial}\n" + "Table2:\t{table_name_bathymetry}\n" "User:\t{user_name}".format(**provider_details)) # register blueprints @@ -95,4 +96,4 @@ def handle_invalid_usage(error): 'db': db} ) - return app \ No newline at end of file + return app diff --git a/openelevationservice/server/db_import/filestreams.py b/openelevationservice/server/db_import/filestreams.py index 849bbaf..59ccf51 100644 --- a/openelevationservice/server/db_import/filestreams.py +++ b/openelevationservice/server/db_import/filestreams.py @@ -1,10 +1,9 @@ # -*- coding: utf-8 -*- from openelevationservice import TILES_DIR, SETTINGS +from openelevationservice.server.sources import PROVIDER_MAPPING from openelevationservice.server.utils.logger import get_logger -from openelevationservice.server.sources.gmted import GMTED -from openelevationservice.server.sources import srtm -from openelevationservice.server.db_import import raster_processing +# from openelevationservice.server.db_import import raster_processing from os import path, environ import subprocess @@ -12,52 +11,64 @@ log = get_logger(__name__) -def downloadsrtm(): +def download(): """ Downlaods GMTED and SRTM v4.1 tiles as bytestream and saves them to TILES_DIR. """ - extent_settings = SETTINGS['tables'][0]['srtm']['extent'] + # for table in SETTINGS['tables']: + + # TODO: write Exception + extent_settings = SETTINGS['tables']['terrestrial']['extent'] if extent_settings['max_y'] <= 60: # only SRTM data download - srtm.download_srtm() + provider = PROVIDER_MAPPING['srtm']() + try: + provider.download_data() + except Exception as err: + log.info(err) - # merge and clip raster by extent - log.info("Starting tile processing ...") - merged_filename = raster_processing.merge_raster('srtm_*', 'srtm_merged.tif') - raster_processing.clip_raster(merged_filename, '/srtm_final.tif') + # # merge and clip raster by extent + # log.info("Starting tile processing ...") + # merged_filename = raster_processing.merge_raster('srtm_*', 'srtm_merged.tif') + # raster_processing.clip_raster(merged_filename, '/srtm_final.tif') elif extent_settings['max_y'] > 60 >= extent_settings['min_y']: # SRTM and GMTED data download - srtm.download_srtm() - - tiles_list = GMTED().tile_selection() - GMTED().download_gmted(tiles_list) - - # resample and merge tiles - log.info("Starting tile preprocessing ...") - srtm_merged_filename = raster_processing.merge_raster('srtm_*', 'srtm_merged.tif') - raster_processing.clip_raster(srtm_merged_filename, 'srtm_clipped.tif') - gmted_merged_filename = raster_processing.merge_raster('*_gmted_mea075.tif', 'gmted_merged.tif') - - log.info("Starting tile resampling ...") - raster_processing.gmted_resampling(gmted_merged_filename, 'srtm_clipped.tif', 'gmted_resampled.tif') - - log.info("Starting tile merging ...") - raster_processing.merge_raster('gmted_resampled.tif', 'raster_final.tif', 'srtm_clipped.tif') + for source in SETTINGS['tables']['terrestrial']['sources']: + provider = PROVIDER_MAPPING[source]() + try: + provider.download_data() + except Exception as err: + log.info(err) + + # # resample and merge tiles + # log.info("Starting tile preprocessing ...") + # srtm_merged_filename = raster_processing.merge_raster('srtm_*', 'srtm_merged.tif') + # raster_processing.clip_raster(srtm_merged_filename, 'srtm_clipped.tif') + # gmted_merged_filename = raster_processing.merge_raster('*_gmted_mea075.tif', 'gmted_merged.tif') + # + # log.info("Starting tile resampling ...") + # raster_processing.gmted_resampling(gmted_merged_filename, 'srtm_clipped.tif', 'gmted_resampled.tif') + # + # log.info("Starting tile merging ...") + # raster_processing.merge_raster('gmted_resampled.tif', 'raster_final.tif', 'srtm_clipped.tif') else: # only GMTED data download - tiles_list = GMTED().tile_selection() - GMTED().download_gmted(tiles_list) - - # merge and clip raster by extent - log.info("Starting tile processing ...") - merged_filename = raster_processing.merge_raster('*_gmted_mea075.tif', 'gmted_merged.tif') - raster_processing.clip_raster(merged_filename, 'gmted_final.tif') + provider = PROVIDER_MAPPING['gmted']() + try: + provider.download_data() + except Exception as err: + log.info(err) + + # # merge and clip raster by extent + # log.info("Starting tile processing ...") + # merged_filename = raster_processing.merge_raster('*_gmted_mea075.tif', 'gmted_merged.tif') + # raster_processing.clip_raster(merged_filename, 'gmted_final.tif') def raster2pgsql(): diff --git a/openelevationservice/server/db_import/models.py b/openelevationservice/server/db_import/models.py index b43ccd9..1431288 100644 --- a/openelevationservice/server/db_import/models.py +++ b/openelevationservice/server/db_import/models.py @@ -10,14 +10,14 @@ db = SQLAlchemy() log = logger.get_logger(__name__) -table_name_srtm = SETTINGS['provider_parameters']['table_name_srtm'] -table_name_composite = SETTINGS['provider_parameters']['table_name_composite'] +table_name_terrestrial = SETTINGS['provider_parameters']['table_name_terrestrial'] +table_name_bathymetry = SETTINGS['provider_parameters']['table_name_bathymetry'] class Cgiar(db.Model): """Database model for SRTM v4.1 aka CGIAR dataset.""" - __tablename__ = table_name_srtm + __tablename__ = table_name_terrestrial rid = db.Column(db.Integer, primary_key=True) rast = db.Column(Raster) @@ -29,7 +29,7 @@ def __repr__(self): class Joerd(db.Model): """Database model for SRTM v4.1 aka CGIAR dataset.""" - __tablename__ = table_name_composite + __tablename__ = table_name_bathymetry rid = db.Column(db.Integer, primary_key=True) rast = db.Column(Raster) diff --git a/openelevationservice/server/db_import/raster_processing.py b/openelevationservice/server/db_import/raster_processing.py index 198f9b2..7112703 100644 --- a/openelevationservice/server/db_import/raster_processing.py +++ b/openelevationservice/server/db_import/raster_processing.py @@ -76,7 +76,7 @@ def clip_raster(merged_filename_extent, output_filename): merged_file = path.join(TILES_DIR + '/' + merged_filename) merged_data = gdal.Open(merged_file, gdalconst.GA_ReadOnly) - extent = str(list(SETTINGS['tables'][0]['srtm']['extent'].values()))[1:-1] + extent = str(list(SETTINGS['tables']['terrestrial']['extent'].values()))[1:-1] clip = r"/usr/bin/gdalwarp -t_srs {target_spatial_ref} -dstnodata -9999 -te {extent} -of {outfile_format} {input_file} {out_clipped_file}" @@ -110,7 +110,7 @@ def gmted_resampling(gmted_merged, srtm_clipped, output_filename): x_res = reference_data.GetGeoTransform()[1] y_res = reference_data.GetGeoTransform()[5] - extent = str(list(SETTINGS['tables'][0]['srtm']['extent'].values()))[1:-1] + extent = str(list(SETTINGS['tables']['terrestrial']['extent'].values()))[1:-1] # TODO: need extent? resampling = r"/usr/bin/gdalwarp -t_srs {target_spatial_ref} -dstnodata -9999 -te {extent} -tr {x_res} {y_res} -r {resampling_method} -of {outfile_format} {input_file} {out_resampling_file}" diff --git a/openelevationservice/server/ops_settings.sample.yml b/openelevationservice/server/ops_settings.sample.yml index bfa9390..8fcb011 100644 --- a/openelevationservice/server/ops_settings.sample.yml +++ b/openelevationservice/server/ops_settings.sample.yml @@ -3,53 +3,47 @@ attribution: "service by https://openrouteservice.org | data by http://srtm.csi. coord_precision: 1e-6 maximum_nodes: 2000 provider_parameters: - table_name: oes_joerd_etopo - db_name: gis - user_name: gis - password: gis + table_name_terrestrial: terrestrial + table_name_bathymetry: bathymetry + db_name: oes_db + user_name: postgres + password: postgres host: localhost port: 5432 -regions: -# san-francisco-downtown: -# bbox: -# top: 37.7997 -# left: -122.5109 -# bottom: 37.7127 -# right: -122.3636 -# zoom_range: [0, 16] - heidelberg: - bbox: - top: 49.423927 - left: 8.662891 - bottom: 49.395223 - right: 8.709412 - zoom_range: [0, 16] -outputs: -# - type: skadi - - type: tiff -# - type: terrarium -# - type: normal -sources: -# - type: cgiar_csi -# url: http://data.cgiar-csi.org/srtm/tiles/GeoTIFF/ -# srtm_parameters: -# user: user -# password: pw - - type: etopo1 - url: https://www.ngdc.noaa.gov/mgg/global/relief/ETOPO1/data/bedrock/grid_registered/georeferenced_tiff/ETOPO1_Bed_g_geotiff.zip -# - type: gmted -# url: http://edcintl.cr.usgs.gov/downloads/sciweb1/shared/topo/downloads/GMTED/Global_tiles_GMTED -# ys: [-90, -70, -50, -30, -10, 10, 30, 50, 70] -# xs: [-180, -150, -120, -90, -60, -30, 0, 30, 60, 90, 120, 150] -# tries: 100 -# - type: ned13 -# ftp_server: rockyftp.cr.usgs.gov -# base_path: vdelivery/Datasets/Staged/NED/13/IMG -# - type: ned -# ftp_server: rockyftp.cr.usgs.gov -# base_path: vdelivery/Datasets/Staged/NED/19/IMG -# - type: ned_topobathy -# ftp_server: rockyftp.cr.usgs.gov -# base_path: vdelivery/Datasets/Staged/NED/19/IMG -logging: - config: logging.example.config \ No newline at end of file +tables: + terrestrial: + table_name: terrestrial + sources: + srtm: + url: http://data.cgiar-csi.org/srtm/tiles/GeoTIFF/ + srtm_parameters: + user: data_public + password: GDdci + gmted: + url: http://edcintl.cr.usgs.gov/downloads/sciweb1/shared/topo/downloads/GMTED/Global_tiles_GMTED + ys: [-90, -70, -50, -30, -10, 10, 30, 50, 70] + xs: [-180, -150, -120, -90, -60, -30, 0, 30, 60, 90, 120, 150] + extent: +# min_x: 8.305664 +# min_y: 61.653379 +# max_x: 8.931885 +# max_y: 61.944118 + min_x: 8.257599 + min_y: 49.185294 + max_x: 8.997803 + max_y: 49.624056 +# min_x: -1.406250 +# min_y: 57.088515 +# max_x: 15.820313 +# max_y: 64.091408 + + bathymetry: + table_name: bathymetry + sources: + etopo1: + url: https://www.ngdc.noaa.gov/mgg/global/relief/ETOPO1/data/bedrock/grid_registered/georeferenced_tiff/ETOPO1_Bed_g_geotiff.zip + extent: + min_x: 8.662891 + min_y: 49.395223 + max_x: 8.709412 + max_y: 49.423927 diff --git a/openelevationservice/server/sources/__init__.py b/openelevationservice/server/sources/__init__.py index e69de29..d7e90a5 100644 --- a/openelevationservice/server/sources/__init__.py +++ b/openelevationservice/server/sources/__init__.py @@ -0,0 +1,7 @@ +from openelevationservice.server.sources import srtm, gmted + +PROVIDER_MAPPING = { + 'srtm': srtm.Srtm, + 'gmted': gmted.Gmted + # 'etopo1': etopo1.Etopo1 +} \ No newline at end of file diff --git a/openelevationservice/server/sources/gmted.py b/openelevationservice/server/sources/gmted.py index 91cacd2..443c128 100644 --- a/openelevationservice/server/sources/gmted.py +++ b/openelevationservice/server/sources/gmted.py @@ -2,6 +2,7 @@ from openelevationservice import TILES_DIR, SETTINGS from openelevationservice.server.utils.logger import get_logger +from openelevationservice.server.sources.provider import ProviderBase from os import path import requests @@ -9,15 +10,30 @@ log = get_logger(__name__) -class GMTED(object): +class Gmted(ProviderBase): - def __init__(self): - self.base_settings = SETTINGS['tables'][0]['srtm'] - self.url = self.base_settings['sources'][1]['gmted']['url'] - self.xs = self.base_settings['sources'][1]['gmted']['xs'] - self.ys = self.base_settings['sources'][1]['gmted']['ys'] - self.bbox_extent = self.base_settings['extent'] - self.buffer = 0.1 + def __init__(self, + base_url=SETTINGS['tables']['terrestrial']['sources']['gmted']['url'], + bbox_extent=SETTINGS['tables']['terrestrial']['extent'], + xys=SETTINGS['tables']['terrestrial']['sources']['gmted'], + buffer=0.1 + ): + super(Gmted, self).__init__(base_url, bbox_extent, xys, buffer) + + def download_data(self): + """ Download tiles and save to disk. """ + + tiles_list = self.tile_selection() + + log.info("{} GMTED tile(s) will be downloaded.".format(len(tiles_list))) + for tile in tiles_list: + if not path.exists(path.join(TILES_DIR, tile[0])): + with open(path.join(TILES_DIR, tile[0]), 'wb') as f: + log.info("Starting to download {}".format(tile[0])) + f.write(requests.get(tile[1]).content) + log.info("Downloaded file {} to {}".format(tile[0], TILES_DIR)) + else: + log.info("{} already exists in {}".format(tile[0], TILES_DIR)) @staticmethod def intersects(buffered_bbox, bbox): @@ -35,7 +51,7 @@ def intersects(buffered_bbox, bbox): def url_selection(self, bbox_x, bbox_y): # file name for chosen region - res = '300' if self.ys == -90 else '075' + res = '300' if self.xys['ys'] == -90 else '075' x_name = "%03d%s" % (abs(bbox_x), "E" if bbox_x >= 0 else "W") y_name = "%02d%s" % (abs(bbox_y), "N" if bbox_y >= 0 else "S") filename = "%(y)s%(x)s_20101117_gmted_mea%(res)s.tif" % dict(res=res, x=x_name, y=y_name) @@ -43,7 +59,7 @@ def url_selection(self, bbox_x, bbox_y): # file url for chosen region gmted_dir = "%s%03d" % ("E" if bbox_x >= 0 else "W", abs(bbox_x)) d_name = "/%(res)sdarcsec/mea/%(dir)s/" % dict(res=res, dir=gmted_dir) - file_url = self.url + d_name + filename + file_url = self.base_url + d_name + filename return filename, file_url @@ -57,8 +73,8 @@ def tile_selection(self): self.bbox_extent['max_y'] + self.buffer] tiles_to_download = [] - for y in self.ys: - for x in self.xs: + for y in self.xys['ys']: + for x in self.xys['xs']: bbox = [x, y, x + 30, y + 20] if self.intersects(buffered_bbox, bbox): filename, file_url = self.url_selection(x, y) @@ -66,16 +82,3 @@ def tile_selection(self): log.info(filename) return tiles_to_download - - def download_gmted(self, tiles_list): - """ Download tiles and save to disk. """ - - log.info("{} GMTED tile(s) will be downloaded.".format(len(tiles_list))) - for tile in tiles_list: - if not path.exists(path.join(TILES_DIR, tile[0])): - with open(path.join(TILES_DIR, tile[0]), 'wb') as f: - log.info("Starting to download {}".format(tile[0])) - f.write(requests.get(tile[1]).content) - log.info("Downloaded file {} to {}".format(tile[0], TILES_DIR)) - else: - log.info("{} already exists in {}".format(tile[0], TILES_DIR)) diff --git a/openelevationservice/server/sources/provider.py b/openelevationservice/server/sources/provider.py new file mode 100644 index 0000000..b5d29bf --- /dev/null +++ b/openelevationservice/server/sources/provider.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- + +from abc import ABC, abstractmethod + + +class ProviderBase(ABC): + + def __init__(self, base_url, bbox_extent, auth_parameters=None, xys=None, buffer=None): + self.base_url = base_url + self.bbox_extent = bbox_extent + self.auth_parameters = auth_parameters + self.xys = xys + self.buffer = buffer + super(ProviderBase, self).__init__() + + @abstractmethod + def download_data(self): + pass + diff --git a/openelevationservice/server/sources/srtm.py b/openelevationservice/server/sources/srtm.py index a24908a..48dcedb 100644 --- a/openelevationservice/server/sources/srtm.py +++ b/openelevationservice/server/sources/srtm.py @@ -2,6 +2,7 @@ from openelevationservice import TILES_DIR, SETTINGS from openelevationservice.server.utils.logger import get_logger +from openelevationservice.server.sources.provider import ProviderBase from os import path, environ import requests @@ -13,115 +14,121 @@ log = get_logger(__name__) -def srtm_x_value(x_min, x_max): - """ Define SRTM x value download name. """ +class Srtm(ProviderBase): - x_value_list = [] + def __init__( + self, + base_url=SETTINGS['tables']['terrestrial']['sources']['srtm']['url'], + bbox_extent=SETTINGS['tables']['terrestrial']['extent'], + auth_parameters=SETTINGS['tables']['terrestrial']['sources']['srtm']['srtm_parameters'] + ): + super(Srtm, self).__init__(base_url, bbox_extent, auth_parameters) - lat_min = -180 - lat_max = -175 - x_srtm_value = 1 + def download_data(self): - for i in range(0, 360): + x_list = self.srtm_x_value(self.bbox_extent['min_x'], self.bbox_extent['max_x']) + y_list = self.srtm_y_value(self.bbox_extent['min_y'], self.bbox_extent['max_y']) - if lat_min < x_min <= lat_max: - if x_min <= x_max: - x_value_list.append(x_srtm_value) - x_min += 5 - else: - return x_value_list + xy_range = [[int(x_list[0]), int(x_list[-1] + 1)], + [int(y_list[0]), int(y_list[-1] + 1)]] + + # Create session for authentication + session = requests.Session() + + pw = environ.get('SRTMPASS') + user = environ.get('SRTMUSER') + if not user and not pw: + auth = tuple(self.auth_parameters.values()) else: - lat_min += 5 - lat_max += 5 - x_srtm_value += 1 + auth = tuple([user, pw]) + session.auth = auth + + log.debug("SRTM credentials: {}".format(session.auth)) + + response = session.get(self.base_url) + + soup = BeautifulSoup(response.content, features="html.parser") + + # First find all 'a' tags starting href with srtm* + for link in soup.find_all('a', attrs={'href': lambda x: x.startswith('srtm') and x.endswith('.zip')}): + link_parsed = link.text.split('_') + link_x = int(link_parsed[1]) + link_y = int(link_parsed[2].split('.')[0]) + # Check if referenced geotif link is in xy_range + if link_y in range(*xy_range[1]) and link_x in range(*xy_range[0]): + log.info('yep') + # Then load the zip data in memory + if not path.exists(path.join(TILES_DIR, '_'.join(['srtm', str(link_x), str(link_y)]) + '.tif')): + with zipfile.ZipFile(BytesIO(session.get(self.base_url + link.text).content)) as zip_obj: + # Loop through the files in the zip + for filename in zip_obj.namelist(): + # Don't extract the readme.txt + # if filename != 'readme.txt': + if filename.endswith('.tif'): + data = zip_obj.read(filename) + # Write byte contents to file + with open(path.join(TILES_DIR, filename), 'wb') as f: + f.write(data) + log.debug("Downloaded file {} to {}".format(link.text, TILES_DIR)) + else: + log.debug("File {} already exists in {}".format(link.text, TILES_DIR)) + @staticmethod + def srtm_x_value(x_min, x_max): + """ Define SRTM x value download name. """ -def srtm_y_value(y_min, y_max): - """ Define SRTM y value download name. """ + x_value_list = [] - y_value_list = [] + lat_min = -180 + lat_max = -175 + x_srtm_value = 1 - lon_min = 55 - lon_max = 60 - y_srtm_value = 1 + for i in range(0, 360): + + if lat_min < x_min <= lat_max: + if x_min <= x_max: + x_value_list.append(x_srtm_value) + x_min += 5 + else: + return x_value_list + else: + lat_min += 5 + lat_max += 5 + x_srtm_value += 1 - # if lon > 60 or < -60: no data available - # TODO: no download instead of boarder tiles - if y_min > 60: - return y_srtm_value + @staticmethod + def srtm_y_value(y_min, y_max): + """ Define SRTM y value download name. """ - elif y_min < -60: - y_srtm_value = 24 - return y_srtm_value + y_value_list = [] - else: - for i in range(0, 120): + lon_min = 55 + lon_max = 60 + y_srtm_value = 1 - if lon_max > y_min >= lon_min: - if y_min <= y_max: - y_value_list.append(y_srtm_value) - y_min += 5 - if y_min > 60: + # if lon > 60 or < -60: no data available + # TODO: no download instead of boarder tiles + if y_min > 60: + return y_srtm_value + + elif y_min < -60: + y_srtm_value = 24 + return y_srtm_value + + else: + for i in range(0, 120): + + if lon_max > y_min >= lon_min: + if y_min <= y_max: + y_value_list.append(y_srtm_value) + y_min += 5 + if y_min > 60: + return y_value_list + else: return y_value_list else: - return y_value_list - else: - lon_min -= 5 - lon_max -= 5 - y_srtm_value += 1 - - return y_value_list - - -def download_srtm(): - """ Download SRTM data. """ - - base_url = SETTINGS['tables'][0]['srtm']['sources'][0]['srtm']['url'] - bbox_extent = SETTINGS['tables'][0]['srtm']['extent'] - - x_list = srtm_x_value(bbox_extent['min_x'], bbox_extent['max_x']) - y_list = srtm_y_value(bbox_extent['min_y'], bbox_extent['max_y']) - - xy_range = [[int(x_list[0]), int(x_list[-1] + 1)], - [int(y_list[0]), int(y_list[-1] + 1)]] - - # Create session for authentication - session = requests.Session() - - pw = environ.get('SRTMPASS') - user = environ.get('SRTMUSER') - if not user and not pw: - auth = tuple(SETTINGS['tables'][0]['srtm']['sources'][0]['srtm']['srtm_parameters'].values()) - else: - auth = tuple([user, pw]) - session.auth = auth - - log.debug("SRTM credentials: {}".format(session.auth)) - - response = session.get(base_url) - - soup = BeautifulSoup(response.content, features="html.parser") - - # First find all 'a' tags starting href with srtm* - for link in soup.find_all('a', attrs={'href': lambda x: x.startswith('srtm') and x.endswith('.zip')}): - link_parsed = link.text.split('_') - link_x = int(link_parsed[1]) - link_y = int(link_parsed[2].split('.')[0]) - # Check if referenced geotif link is in xy_range - if link_y in range(*xy_range[1]) and link_x in range(*xy_range[0]): - log.info('yep') - # Then load the zip data in memory - if not path.exists(path.join(TILES_DIR, '_'.join(['srtm', str(link_x), str(link_y)]) + '.tif')): - with zipfile.ZipFile(BytesIO(session.get(base_url + link.text).content)) as zip_obj: - # Loop through the files in the zip - for filename in zip_obj.namelist(): - # Don't extract the readme.txt - # if filename != 'readme.txt': - if filename.endswith('.tif'): - data = zip_obj.read(filename) - # Write byte contents to file - with open(path.join(TILES_DIR, filename), 'wb') as f: - f.write(data) - log.debug("Downloaded file {} to {}".format(link.text, TILES_DIR)) - else: - log.debug("File {} already exists in {}".format(link.text, TILES_DIR)) + lon_min -= 5 + lon_max -= 5 + y_srtm_value += 1 + + return y_value_list diff --git a/ops_settings_docker.sample.yml b/ops_settings_docker.sample.yml index 7d1ddcd..8fcb011 100644 --- a/ops_settings_docker.sample.yml +++ b/ops_settings_docker.sample.yml @@ -3,38 +3,47 @@ attribution: "service by https://openrouteservice.org | data by http://srtm.csi. coord_precision: 1e-6 maximum_nodes: 2000 provider_parameters: - table_name_srtm: srtm - table_name_composite: composite + table_name_terrestrial: terrestrial + table_name_bathymetry: bathymetry db_name: oes_db - user_name: gis - password: gis + user_name: postgres + password: postgres host: localhost port: 5432 tables: - - srtm: - table_name: srtm - sources: - - srtm: - url: http://data.cgiar-csi.org/srtm/tiles/GeoTIFF/ - srtm_parameters: - user: user - password: password - - gmted: - url: http://edcintl.cr.usgs.gov/downloads/sciweb1/shared/topo/downloads/GMTED/Global_tiles_GMTED - ys: [-90, -70, -50, -30, -10, 10, 30, 50, 70] - xs: [-180, -150, -120, -90, -60, -30, 0, 30, 60, 90, 120, 150] - extent: - min_x: -1.406250 - min_y: 57.088515 - max_x: 15.820313 - max_y: 64.091408 - - composite: - table_name: composite - sources: - - etopo1: - url: https://www.ngdc.noaa.gov/mgg/global/relief/ETOPO1/data/bedrock/grid_registered/georeferenced_tiff/ETOPO1_Bed_g_geotiff.zip - extent: - min_x: 8.662891 - min_y: 49.395223 - max_x: 8.709412 - max_y: 49.423927 + terrestrial: + table_name: terrestrial + sources: + srtm: + url: http://data.cgiar-csi.org/srtm/tiles/GeoTIFF/ + srtm_parameters: + user: data_public + password: GDdci + gmted: + url: http://edcintl.cr.usgs.gov/downloads/sciweb1/shared/topo/downloads/GMTED/Global_tiles_GMTED + ys: [-90, -70, -50, -30, -10, 10, 30, 50, 70] + xs: [-180, -150, -120, -90, -60, -30, 0, 30, 60, 90, 120, 150] + extent: +# min_x: 8.305664 +# min_y: 61.653379 +# max_x: 8.931885 +# max_y: 61.944118 + min_x: 8.257599 + min_y: 49.185294 + max_x: 8.997803 + max_y: 49.624056 +# min_x: -1.406250 +# min_y: 57.088515 +# max_x: 15.820313 +# max_y: 64.091408 + + bathymetry: + table_name: bathymetry + sources: + etopo1: + url: https://www.ngdc.noaa.gov/mgg/global/relief/ETOPO1/data/bedrock/grid_registered/georeferenced_tiff/ETOPO1_Bed_g_geotiff.zip + extent: + min_x: 8.662891 + min_y: 49.395223 + max_x: 8.709412 + max_y: 49.423927 From 0716cccef460b25b5c412d92539e76ba3b6676a0 Mon Sep 17 00:00:00 2001 From: Isabell Klipper Date: Tue, 3 Dec 2019 12:13:05 +0100 Subject: [PATCH 07/11] Added etopo1.py; cleaned filestreams.py --- .../server/db_import/filestreams.py | 50 +---------- .../server/sources/__init__.py | 6 +- openelevationservice/server/sources/etopo1.py | 39 ++++++++ openelevationservice/server/sources/gmted.py | 23 ++--- openelevationservice/server/sources/srtm.py | 88 ++++++++++--------- 5 files changed, 101 insertions(+), 105 deletions(-) diff --git a/openelevationservice/server/db_import/filestreams.py b/openelevationservice/server/db_import/filestreams.py index 59ccf51..db6b315 100644 --- a/openelevationservice/server/db_import/filestreams.py +++ b/openelevationservice/server/db_import/filestreams.py @@ -16,60 +16,14 @@ def download(): Downlaods GMTED and SRTM v4.1 tiles as bytestream and saves them to TILES_DIR. """ - # for table in SETTINGS['tables']: - - # TODO: write Exception - extent_settings = SETTINGS['tables']['terrestrial']['extent'] - if extent_settings['max_y'] <= 60: - - # only SRTM data download - provider = PROVIDER_MAPPING['srtm']() - try: - provider.download_data() - except Exception as err: - log.info(err) - - # # merge and clip raster by extent - # log.info("Starting tile processing ...") - # merged_filename = raster_processing.merge_raster('srtm_*', 'srtm_merged.tif') - # raster_processing.clip_raster(merged_filename, '/srtm_final.tif') - - elif extent_settings['max_y'] > 60 >= extent_settings['min_y']: - - # SRTM and GMTED data download - for source in SETTINGS['tables']['terrestrial']['sources']: + for table in SETTINGS['tables']: + for source in SETTINGS['tables'][table]['sources']: provider = PROVIDER_MAPPING[source]() try: provider.download_data() except Exception as err: log.info(err) - # # resample and merge tiles - # log.info("Starting tile preprocessing ...") - # srtm_merged_filename = raster_processing.merge_raster('srtm_*', 'srtm_merged.tif') - # raster_processing.clip_raster(srtm_merged_filename, 'srtm_clipped.tif') - # gmted_merged_filename = raster_processing.merge_raster('*_gmted_mea075.tif', 'gmted_merged.tif') - # - # log.info("Starting tile resampling ...") - # raster_processing.gmted_resampling(gmted_merged_filename, 'srtm_clipped.tif', 'gmted_resampled.tif') - # - # log.info("Starting tile merging ...") - # raster_processing.merge_raster('gmted_resampled.tif', 'raster_final.tif', 'srtm_clipped.tif') - - else: - - # only GMTED data download - provider = PROVIDER_MAPPING['gmted']() - try: - provider.download_data() - except Exception as err: - log.info(err) - - # # merge and clip raster by extent - # log.info("Starting tile processing ...") - # merged_filename = raster_processing.merge_raster('*_gmted_mea075.tif', 'gmted_merged.tif') - # raster_processing.clip_raster(merged_filename, 'gmted_final.tif') - def raster2pgsql(): """ diff --git a/openelevationservice/server/sources/__init__.py b/openelevationservice/server/sources/__init__.py index d7e90a5..97a4696 100644 --- a/openelevationservice/server/sources/__init__.py +++ b/openelevationservice/server/sources/__init__.py @@ -1,7 +1,7 @@ -from openelevationservice.server.sources import srtm, gmted +from openelevationservice.server.sources import srtm, gmted, etopo1 PROVIDER_MAPPING = { 'srtm': srtm.Srtm, - 'gmted': gmted.Gmted - # 'etopo1': etopo1.Etopo1 + 'gmted': gmted.Gmted, + 'etopo1': etopo1.Etopo1 } \ No newline at end of file diff --git a/openelevationservice/server/sources/etopo1.py b/openelevationservice/server/sources/etopo1.py index e69de29..0b3c6ab 100644 --- a/openelevationservice/server/sources/etopo1.py +++ b/openelevationservice/server/sources/etopo1.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- + +from openelevationservice import TILES_DIR, SETTINGS +from openelevationservice.server.utils.logger import get_logger +from openelevationservice.server.sources.provider import ProviderBase + +from os import path +import requests +import zipfile + +from io import BytesIO + +log = get_logger(__name__) + + +class Etopo1(ProviderBase): + + def __init__( + self, + base_url=SETTINGS['tables']['bathymetry']['sources']['etopo1']['url'], + bbox_extent=SETTINGS['tables']['bathymetry']['extent'] + ): + super(Etopo1, self).__init__(base_url, bbox_extent) + + def download_data(self): + + etopo1_filename = 'ETOPO1_Bed_g_geotiff.tif' + + if not path.exists(path.join(TILES_DIR, etopo1_filename)): + with zipfile.ZipFile(BytesIO(requests.get(self.base_url).content)) as zip_obj: + # Loop through the files in the zip + for filename in zip_obj.namelist(): + data = zip_obj.read(filename) + # Write byte contents to file + with open(path.join(TILES_DIR, filename), 'wb') as f: + f.write(data) + log.debug("Downloaded file {} to {}".format(filename, TILES_DIR)) + else: + log.debug("{} already exists in {}".format(etopo1_filename, TILES_DIR)) diff --git a/openelevationservice/server/sources/gmted.py b/openelevationservice/server/sources/gmted.py index 443c128..e078a98 100644 --- a/openelevationservice/server/sources/gmted.py +++ b/openelevationservice/server/sources/gmted.py @@ -23,17 +23,18 @@ def __init__(self, def download_data(self): """ Download tiles and save to disk. """ - tiles_list = self.tile_selection() - - log.info("{} GMTED tile(s) will be downloaded.".format(len(tiles_list))) - for tile in tiles_list: - if not path.exists(path.join(TILES_DIR, tile[0])): - with open(path.join(TILES_DIR, tile[0]), 'wb') as f: - log.info("Starting to download {}".format(tile[0])) - f.write(requests.get(tile[1]).content) - log.info("Downloaded file {} to {}".format(tile[0], TILES_DIR)) - else: - log.info("{} already exists in {}".format(tile[0], TILES_DIR)) + if self.bbox_extent['max_y'] > 60: + tiles_list = self.tile_selection() + + log.info("{} GMTED tile(s) will be downloaded.".format(len(tiles_list))) + for tile in tiles_list: + if not path.exists(path.join(TILES_DIR, tile[0])): + with open(path.join(TILES_DIR, tile[0]), 'wb') as f: + log.info("Starting to download {}".format(tile[0])) + f.write(requests.get(tile[1]).content) + log.info("Downloaded file {} to {}".format(tile[0], TILES_DIR)) + else: + log.info("{} already exists in {}".format(tile[0], TILES_DIR)) @staticmethod def intersects(buffered_bbox, bbox): diff --git a/openelevationservice/server/sources/srtm.py b/openelevationservice/server/sources/srtm.py index 48dcedb..65af7ad 100644 --- a/openelevationservice/server/sources/srtm.py +++ b/openelevationservice/server/sources/srtm.py @@ -26,52 +26,54 @@ def __init__( def download_data(self): - x_list = self.srtm_x_value(self.bbox_extent['min_x'], self.bbox_extent['max_x']) - y_list = self.srtm_y_value(self.bbox_extent['min_y'], self.bbox_extent['max_y']) + if self.bbox_extent['min_y'] <= 60: - xy_range = [[int(x_list[0]), int(x_list[-1] + 1)], - [int(y_list[0]), int(y_list[-1] + 1)]] + x_list = self.srtm_x_value(self.bbox_extent['min_x'], self.bbox_extent['max_x']) + y_list = self.srtm_y_value(self.bbox_extent['min_y'], self.bbox_extent['max_y']) - # Create session for authentication - session = requests.Session() + xy_range = [[int(x_list[0]), int(x_list[-1] + 1)], + [int(y_list[0]), int(y_list[-1] + 1)]] - pw = environ.get('SRTMPASS') - user = environ.get('SRTMUSER') - if not user and not pw: - auth = tuple(self.auth_parameters.values()) - else: - auth = tuple([user, pw]) - session.auth = auth - - log.debug("SRTM credentials: {}".format(session.auth)) - - response = session.get(self.base_url) - - soup = BeautifulSoup(response.content, features="html.parser") - - # First find all 'a' tags starting href with srtm* - for link in soup.find_all('a', attrs={'href': lambda x: x.startswith('srtm') and x.endswith('.zip')}): - link_parsed = link.text.split('_') - link_x = int(link_parsed[1]) - link_y = int(link_parsed[2].split('.')[0]) - # Check if referenced geotif link is in xy_range - if link_y in range(*xy_range[1]) and link_x in range(*xy_range[0]): - log.info('yep') - # Then load the zip data in memory - if not path.exists(path.join(TILES_DIR, '_'.join(['srtm', str(link_x), str(link_y)]) + '.tif')): - with zipfile.ZipFile(BytesIO(session.get(self.base_url + link.text).content)) as zip_obj: - # Loop through the files in the zip - for filename in zip_obj.namelist(): - # Don't extract the readme.txt - # if filename != 'readme.txt': - if filename.endswith('.tif'): - data = zip_obj.read(filename) - # Write byte contents to file - with open(path.join(TILES_DIR, filename), 'wb') as f: - f.write(data) - log.debug("Downloaded file {} to {}".format(link.text, TILES_DIR)) - else: - log.debug("File {} already exists in {}".format(link.text, TILES_DIR)) + # Create session for authentication + session = requests.Session() + + pw = environ.get('SRTMPASS') + user = environ.get('SRTMUSER') + if not user and not pw: + auth = tuple(self.auth_parameters.values()) + else: + auth = tuple([user, pw]) + session.auth = auth + + log.debug("SRTM credentials: {}".format(session.auth)) + + response = session.get(self.base_url) + + soup = BeautifulSoup(response.content, features="html.parser") + + # First find all 'a' tags starting href with srtm* + for link in soup.find_all('a', attrs={'href': lambda x: x.startswith('srtm') and x.endswith('.zip')}): + link_parsed = link.text.split('_') + link_x = int(link_parsed[1]) + link_y = int(link_parsed[2].split('.')[0]) + # Check if referenced geotif link is in xy_range + if link_y in range(*xy_range[1]) and link_x in range(*xy_range[0]): + log.info('yep') + # Then load the zip data in memory + if not path.exists(path.join(TILES_DIR, '_'.join(['srtm', str(link_x), str(link_y)]) + '.tif')): + with zipfile.ZipFile(BytesIO(session.get(self.base_url + link.text).content)) as zip_obj: + # Loop through the files in the zip + for filename in zip_obj.namelist(): + # Don't extract the readme.txt + # if filename != 'readme.txt': + if filename.endswith('.tif'): + data = zip_obj.read(filename) + # Write byte contents to file + with open(path.join(TILES_DIR, filename), 'wb') as f: + f.write(data) + log.debug("Downloaded file {} to {}".format(link.text, TILES_DIR)) + else: + log.debug("File {} already exists in {}".format(link.text, TILES_DIR)) @staticmethod def srtm_x_value(x_min, x_max): From 8bebcf082ef19d00fdda6c4fe13760dab32ea0b9 Mon Sep 17 00:00:00 2001 From: Isabell Klipper Date: Mon, 16 Dec 2019 09:28:48 +0100 Subject: [PATCH 08/11] Added gv_at data source; Restructured download and merging part --- manage.py | 36 ++--- openelevationservice/server/__init__.py | 15 +- openelevationservice/server/config.py | 1 + .../server/db_import/filestreams.py | 127 ++++++++++----- .../server/db_import/models.py | 25 +-- .../server/db_import/raster_processing.py | 100 +++++------- .../server/sources/__init__.py | 11 +- openelevationservice/server/sources/etopo1.py | 33 ++-- openelevationservice/server/sources/gmted.py | 68 +++++++-- openelevationservice/server/sources/gv_at.py | 55 +++++++ .../server/sources/provider.py | 8 +- openelevationservice/server/sources/srtm.py | 144 +++++++++--------- 12 files changed, 378 insertions(+), 245 deletions(-) create mode 100644 openelevationservice/server/sources/gv_at.py diff --git a/manage.py b/manage.py index 3d49525..2e50fab 100644 --- a/manage.py +++ b/manage.py @@ -12,17 +12,20 @@ @app.cli.command() -def download(): +def prepare(): """ Downloads SRTM tiles to disk. Can be specified over minx, maxx, miny, maxy. - - :param xyrange: A comma-separated list of x_min, x_max, y_min, y_max - in that order. For reference grid, see http://srtm.csi.cgiar.org/SELECTION/inputCoord.asp - :type xyrange: comma-separated integers """ filestreams.download() log.info("Downloaded all files") + + +@app.cli.command() +def merge(): + + filestreams.merge_data() + log.info("Merged downloaded files") @app.cli.command() @@ -30,8 +33,8 @@ def create(): """Creates all tables defined in models.py""" db.create_all() - log.info("Table {} was created.".format(SETTINGS['provider_parameters']['table_name_srtm'])) - log.info("Table {} was created.".format(SETTINGS['provider_parameters']['table_name_composite'])) + for table in SETTINGS['provider_parameters']['tables']: + log.info("Table {} was created.".format(table)) @app.cli.command() @@ -39,30 +42,17 @@ def drop(): """Drops all tables defined in models.py""" db.drop_all() - log.info("Table {} was created.".format(SETTINGS['provider_parameters']['table_name_srtm'])) - log.info("Table {} was created.".format(SETTINGS['provider_parameters']['table_name_composite'])) + for table in SETTINGS['provider_parameters']['tables']: + log.info("Table {} was dropped.".format(table)) + @app.cli.command() def importdata(): """ Imports all data found in ./tiles - - :param xyrange: A comma-separated list of x_min, x_max, y_min, y_max - in that order. For reference grid, see http://srtm.csi.cgiar.org/SELECTION/inputCoord.asp - :type xyrange: comma-separated integers """ log.info("Starting to import data...") filestreams.raster2pgsql() log.info("Imported data successfully!") - - -def _arg_format(xy_range_txt): - - str_split = [int(s.strip()) for s in xy_range_txt.split(',')] - - xy_range = [[str_split[0], str_split[2]], - [str_split[1], str_split[3]]] - - return xy_range diff --git a/openelevationservice/server/__init__.py b/openelevationservice/server/__init__.py index 6c056e8..4c59aab 100644 --- a/openelevationservice/server/__init__.py +++ b/openelevationservice/server/__init__.py @@ -33,14 +33,15 @@ def create_app(script_info=None): # set up extensions db.init_app(app) - + provider_details = SETTINGS['provider_parameters'] - log.info("Following provider parameters are active:\n" - "Host:\t{host}\n" - "DB:\t{db_name}\n" - "Table1:\t{table_name_terrestrial}\n" - "Table2:\t{table_name_bathymetry}\n" - "User:\t{user_name}".format(**provider_details)) + + for table in provider_details['tables']: + log.info("Following provider parameters are active:\n" + "Host:\t{host}\n" + "DB:\t{db_name}\n" + "Table:\t{table_name}\n" + "User:\t{user_name}".format(**provider_details, table_name=table)) # register blueprints from openelevationservice.server.api.views import main_blueprint diff --git a/openelevationservice/server/config.py b/openelevationservice/server/config.py index 755b2c9..5e34101 100644 --- a/openelevationservice/server/config.py +++ b/openelevationservice/server/config.py @@ -2,6 +2,7 @@ from openelevationservice import SETTINGS + class BaseConfig(object): """Base configuration.""" diff --git a/openelevationservice/server/db_import/filestreams.py b/openelevationservice/server/db_import/filestreams.py index db6b315..f182614 100644 --- a/openelevationservice/server/db_import/filestreams.py +++ b/openelevationservice/server/db_import/filestreams.py @@ -3,26 +3,92 @@ from openelevationservice import TILES_DIR, SETTINGS from openelevationservice.server.sources import PROVIDER_MAPPING from openelevationservice.server.utils.logger import get_logger -# from openelevationservice.server.db_import import raster_processing from os import path, environ import subprocess log = get_logger(__name__) +# TODO: write Exception + def download(): - """ - Downlaods GMTED and SRTM v4.1 tiles as bytestream and saves them to TILES_DIR. - """ + """Selects download provider.""" + + extent_settings = SETTINGS['tables']['terrestrial']['extent'] + download_list = [] + + if not SETTINGS['provider_parameters']['tables']: + log.error("Please define at least one table in ops_settings.yml") + + else: + # for table in SETTINGS['provider_parameters']['tables']: + if 'terrestrial' in SETTINGS['provider_parameters']['tables']: + + # only SRTM data download + if 60 >= extent_settings['max_y'] >= -60: + provider = PROVIDER_MAPPING['terrestrial']['srtm']() + download_list.append(provider) + + # SRTM and GMTED data download + elif extent_settings['max_y'] > 60 >= extent_settings['min_y'] or \ + extent_settings['max_y'] > -60 >= extent_settings['min_y']: + for source in SETTINGS['tables']['terrestrial']['sources']: + provider = PROVIDER_MAPPING['terrestrial'][source]() + download_list.append(provider) + + # only GMTED data download + else: + provider = PROVIDER_MAPPING['terrestrial']['gmted']() + download_list.append(provider) + + else: + for p in PROVIDER_MAPPING: + if p != 'terrestrial': + provider = PROVIDER_MAPPING[p]() + download_list.append(provider) + + for download_provider in download_list: + try: + download_provider.download_data() + except Exception as err: + log.error(err) - for table in SETTINGS['tables']: - for source in SETTINGS['tables'][table]['sources']: - provider = PROVIDER_MAPPING[source]() + +def merge_data(): + """Selects provider for further raster tile resampling and merging.""" + + extent_settings = SETTINGS['tables']['terrestrial']['extent'] + merge_list = [] + + if not SETTINGS['provider_parameters']['tables']: + log.error("Please define at least one table in ops_settings.yml") + + else: + for table in SETTINGS['provider_parameters']['tables']: + if 'terrestrial' in table: + + # only SRTM data + if 60 >= extent_settings['max_y'] >= -60: + provider = PROVIDER_MAPPING['terrestrial']['srtm']() + merge_list.append(provider) + + # GMTED data or GMTED and SRTM data + elif extent_settings['max_y'] > 60 or extent_settings['min_y'] < -60: + provider = PROVIDER_MAPPING['terrestrial']['gmted']() + merge_list.append(provider) + + else: + for p in PROVIDER_MAPPING: + if p != 'terrestrial': + provider = PROVIDER_MAPPING[p]() + merge_list.append(provider) + + for merge_provider in merge_list: try: - provider.download_data() + merge_provider.merge_tiles() except Exception as err: - log.info(err) + log.error(err) def raster2pgsql(): @@ -32,49 +98,36 @@ def raster2pgsql(): :raises subprocess.CalledProcessError: Raised when raster2pgsql throws an error. """ + # TODO: define -t spatial size + # TODO: bathy_raster.tif -> just one row? + # TODO: import single files, too? pg_settings = SETTINGS['provider_parameters'] # Copy all env variables and add PGPASSWORD env_current = environ.copy() env_current['PGPASSWORD'] = pg_settings['password'] - tiles_dir_name = None - - if SETTINGS["sources"][0]["type"] == "cgiar_csi": + for table in SETTINGS['provider_parameters']['tables']: + filename = path.join(TILES_DIR + '/' + table + '_raster.tif') # Tried to import every raster individually by user-specified xyrange # similar to download(), but raster2pgsql fuck it up somehow.. The PostGIS # raster will not be what one would expect. Instead doing a bulk import of all files. - cmd_raster2pgsql = r"raster2pgsql -s 4326 -a -C -M -P -t 50x50 {filename} {table_name_srtm} | psql -q -h {host} -p {port} -U {user_name} -d {db_name}" + cmd_raster2pgsql = r"raster2pgsql -s 4326 -a -C -M -t 50x50 {filename} {table_name} | psql -q -h {host} -p {port} -U {user_name} -d {db_name}" # -s: raster SRID # -a: append to table (assumes it's been create with 'create()') # -C: apply all raster Constraints - # -P: pad tiles to guarantee all tiles have the same width and height # -M: vacuum analyze after import # -t: specifies the pixel size of each row. Important to keep low for performance! - tiles_dir_name = TILES_DIR - - else: - cmd_raster2pgsql = r"raster2pgsql -s 4326 -a -C -M -P -t 50x50 {filename} {table_name_joerd} | psql -q -h {host} -p {port} -U {user_name} -d {db_name}" + cmd_raster2pgsql = cmd_raster2pgsql.format(**{'filename': filename, 'table_name': table, **pg_settings}) - tiles_dir_name = TILES_DIR + proc = subprocess.Popen(cmd_raster2pgsql, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + shell=True, + env=env_current) - if path.join(TILES_DIR, '*.tif'): - cmd_raster2pgsql = cmd_raster2pgsql.format(**{'filename': path.join(tiles_dir_name, '*.tif'), **pg_settings}) - else: - cmd_raster2pgsql = cmd_raster2pgsql.format(**{'filename': path.join(tiles_dir_name, '*.img'), **pg_settings}) - - proc = subprocess.Popen(cmd_raster2pgsql, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - shell=True, - env=env_current - ) - -# for line in proc.stdout: -# log.debug(line.decode()) -# proc.stdout.close() - return_code = proc.wait() - if return_code: - raise subprocess.CalledProcessError(return_code, cmd_raster2pgsql) + return_code = proc.wait() + if return_code: + raise subprocess.CalledProcessError(return_code, cmd_raster2pgsql) diff --git a/openelevationservice/server/db_import/models.py b/openelevationservice/server/db_import/models.py index 1431288..c558809 100644 --- a/openelevationservice/server/db_import/models.py +++ b/openelevationservice/server/db_import/models.py @@ -4,35 +4,22 @@ from openelevationservice.server.utils import logger from flask_sqlalchemy import SQLAlchemy -from sqlalchemy import Index from geoalchemy2 import Raster db = SQLAlchemy() log = logger.get_logger(__name__) -table_name_terrestrial = SETTINGS['provider_parameters']['table_name_terrestrial'] -table_name_bathymetry = SETTINGS['provider_parameters']['table_name_bathymetry'] class Cgiar(db.Model): """Database model for SRTM v4.1 aka CGIAR dataset.""" - __tablename__ = table_name_terrestrial + for table in SETTINGS['provider_parameters']['tables']: - rid = db.Column(db.Integer, primary_key=True) - rast = db.Column(Raster) + __tablename__ = table - def __repr__(self): - return ''.format(self.rid, self.rast) + rid = db.Column(db.Integer, primary_key=True) + rast = db.Column(Raster) - -class Joerd(db.Model): - """Database model for SRTM v4.1 aka CGIAR dataset.""" - - __tablename__ = table_name_bathymetry - - rid = db.Column(db.Integer, primary_key=True) - rast = db.Column(Raster) - - def __repr__(self): - return ''.format(self.rid, self.rast) + def __repr__(self): + return ''.format(self.rid, self.rast) diff --git a/openelevationservice/server/db_import/raster_processing.py b/openelevationservice/server/db_import/raster_processing.py index 7112703..9d98762 100644 --- a/openelevationservice/server/db_import/raster_processing.py +++ b/openelevationservice/server/db_import/raster_processing.py @@ -10,6 +10,16 @@ # TODO: handle in memory files +def run_cmd(cmd): + proc = subprocess.Popen(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + shell=True) + + return_code = proc.wait() + if return_code: + raise subprocess.CalledProcessError(return_code, cmd) + def merge_raster(input_filename, output_filename, reference=None): """ Merge downloaded single tiles to one raster tile. """ @@ -19,51 +29,31 @@ def merge_raster(input_filename, output_filename, reference=None): if not path.exists(path.join(TILES_DIR, output_filename)): - if reference is None: + cmd = {'outfile': output_merge, + 'outfile_format': 'GTiff', + 'input_files': input_files} + # merge tiles from same data source + if reference is None: + # pass if only one tile available if len(fnmatch.filter(listdir(TILES_DIR), input_filename)) > 1: - - raster_merge = r"/usr/bin/gdal_merge.py -o {outfile} -of {outfile_format} {input_files}" - - cmd_merge = raster_merge.format(**{'outfile': output_merge, - 'outfile_format': 'GTiff', - 'input_files': input_files}) - - proc = subprocess.Popen(cmd_merge, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - shell=True) - - return_code = proc.wait() - if return_code: - raise subprocess.CalledProcessError(return_code, cmd_merge) - + merge = r"/usr/bin/gdal_merge.py -o {outfile} -of {outfile_format} {input_files}" else: return input_filename + # merge srtm and gmted tile fractions else: - # merge srtm and gmted tile fractions reference_file = path.join(TILES_DIR + '/' + reference) # -tap: align tiles # reference_file: In areas of overlap, the last image will be copied over earlier ones. merge = r"/usr/bin/gdal_merge.py -o {outfile} -of {outfile_format} -tap {input_files} {reference_file}" + cmd.update({'reference_file': reference_file}) - cmd_merge = merge.format(**{'outfile': output_merge, - 'outfile_format': 'GTiff', - 'input_files': input_files, # gmted - 'reference_file': reference_file}) # srtm + cmd_merge = merge.format(**cmd) + run_cmd(cmd_merge) - proc = subprocess.Popen(cmd_merge, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - shell=True) - - return_code = proc.wait() - if return_code: - raise subprocess.CalledProcessError(return_code, cmd_merge) - - return output_filename + return output_filename def clip_raster(merged_filename_extent, output_filename): @@ -71,40 +61,37 @@ def clip_raster(merged_filename_extent, output_filename): if not path.exists(path.join(TILES_DIR, output_filename)): - output_clip = path.join(TILES_DIR + '/' + output_filename) merged_filename = fnmatch.filter(listdir(TILES_DIR), merged_filename_extent)[0] merged_file = path.join(TILES_DIR + '/' + merged_filename) merged_data = gdal.Open(merged_file, gdalconst.GA_ReadOnly) + output_clip = path.join(TILES_DIR + '/' + output_filename) extent = str(list(SETTINGS['tables']['terrestrial']['extent'].values()))[1:-1] - clip = r"/usr/bin/gdalwarp -t_srs {target_spatial_ref} -dstnodata -9999 -te {extent} -of {outfile_format} {input_file} {out_clipped_file}" + cmd = {'extent': extent, + 'outfile_format': 'GTiff', + 'input_file': merged_file, + 'out_clipped_file': output_clip} - cmd_clip = clip.format(**{'target_spatial_ref': merged_data.GetProjection(), - 'extent': extent, - 'outfile_format': 'GTiff', - 'input_file': merged_file, - 'out_clipped_file': output_clip}) + if merged_data.GetProjection(): + clip = r"/usr/bin/gdalwarp -t_srs {target_spatial_ref} -dstnodata -9999 -te {extent} -of {outfile_format} {input_file} {out_clipped_file}" + cmd.update({'target_spatial_ref': merged_data.GetProjection()}) - proc_clip = subprocess.Popen(cmd_clip, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - shell=True) + else: + clip = r"/usr/bin/gdalwarp -dstnodata -9999 -te {extent} -of {outfile_format} {input_file} {out_clipped_file}" - return_code_clip = proc_clip.wait() - if return_code_clip: - raise subprocess.CalledProcessError(return_code_clip, cmd_clip) + cmd_clip = clip.format(**cmd) + run_cmd(cmd_clip) -def gmted_resampling(gmted_merged, srtm_clipped, output_filename): +def gmted_resampling(gmted_merged_filename, srtm_clipped_filename, output_filename): """ Resample merged GMTED raster to SRTM resolution. """ if not path.exists(path.join(TILES_DIR, output_filename)): - output_resampled = path.join(TILES_DIR + '/' + output_filename) - gmted_merged = path.join(TILES_DIR + '/' + gmted_merged) + gmted_merged = path.join(TILES_DIR + '/' + gmted_merged_filename) - srtm_clipped = path.join(TILES_DIR + '/' + srtm_clipped) + srtm_clipped = path.join(TILES_DIR + '/' + srtm_clipped_filename) reference_data = gdal.Open(srtm_clipped, gdalconst.GA_ReadOnly) # desired resolution x_res = reference_data.GetGeoTransform()[1] @@ -112,8 +99,7 @@ def gmted_resampling(gmted_merged, srtm_clipped, output_filename): extent = str(list(SETTINGS['tables']['terrestrial']['extent'].values()))[1:-1] - # TODO: need extent? - resampling = r"/usr/bin/gdalwarp -t_srs {target_spatial_ref} -dstnodata -9999 -te {extent} -tr {x_res} {y_res} -r {resampling_method} -of {outfile_format} {input_file} {out_resampling_file}" + resampling = r"/usr/bin/gdalwarp -t_srs {target_spatial_ref} -te {extent} -dstnodata -9999 -tr {x_res} {y_res} -r {resampling_method} -of {outfile_format} {input_file} {out_resampling_file}" cmd_resampling = resampling.format(**{'target_spatial_ref': reference_data.GetProjection(), 'extent': extent, @@ -123,12 +109,4 @@ def gmted_resampling(gmted_merged, srtm_clipped, output_filename): 'outfile_format': 'GTiff', 'input_file': gmted_merged, 'out_resampling_file': output_resampled}) - - proc_resampling = subprocess.Popen(cmd_resampling, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - shell=True) - - return_code_resampling = proc_resampling.wait() - if return_code_resampling: - raise subprocess.CalledProcessError(return_code_resampling, cmd_resampling) + run_cmd(cmd_resampling) diff --git a/openelevationservice/server/sources/__init__.py b/openelevationservice/server/sources/__init__.py index 97a4696..9ef92ac 100644 --- a/openelevationservice/server/sources/__init__.py +++ b/openelevationservice/server/sources/__init__.py @@ -1,7 +1,8 @@ -from openelevationservice.server.sources import srtm, gmted, etopo1 +from openelevationservice.server.sources import srtm, gmted, etopo1, gv_at PROVIDER_MAPPING = { - 'srtm': srtm.Srtm, - 'gmted': gmted.Gmted, - 'etopo1': etopo1.Etopo1 -} \ No newline at end of file + 'terrestrial': {'srtm': srtm.Srtm, + 'gmted': gmted.Gmted}, + 'etopo1': etopo1.Etopo1, + 'gv_at': gv_at.Gvat +} diff --git a/openelevationservice/server/sources/etopo1.py b/openelevationservice/server/sources/etopo1.py index 0b3c6ab..fcf3cf1 100644 --- a/openelevationservice/server/sources/etopo1.py +++ b/openelevationservice/server/sources/etopo1.py @@ -3,6 +3,7 @@ from openelevationservice import TILES_DIR, SETTINGS from openelevationservice.server.utils.logger import get_logger from openelevationservice.server.sources.provider import ProviderBase +from openelevationservice.server.db_import import raster_processing from os import path import requests @@ -18,22 +19,36 @@ class Etopo1(ProviderBase): def __init__( self, base_url=SETTINGS['tables']['bathymetry']['sources']['etopo1']['url'], - bbox_extent=SETTINGS['tables']['bathymetry']['extent'] + output_raster='bathymetry_raster.tif', + bbox_extent=None, + auth_parameters=None, + filename='ETOPO1_Bed_g_geotiff.tif' ): - super(Etopo1, self).__init__(base_url, bbox_extent) + super(Etopo1, self).__init__(base_url, output_raster, bbox_extent, auth_parameters, filename) def download_data(self): + """Download tiles and save to disk.""" - etopo1_filename = 'ETOPO1_Bed_g_geotiff.tif' + file_counter = 0 - if not path.exists(path.join(TILES_DIR, etopo1_filename)): + if not path.exists(path.join(TILES_DIR, self.filename)): with zipfile.ZipFile(BytesIO(requests.get(self.base_url).content)) as zip_obj: # Loop through the files in the zip - for filename in zip_obj.namelist(): - data = zip_obj.read(filename) + for file in zip_obj.namelist(): + data = zip_obj.read(file) # Write byte contents to file - with open(path.join(TILES_DIR, filename), 'wb') as f: + with open(path.join(TILES_DIR, file), 'wb') as f: f.write(data) - log.debug("Downloaded file {} to {}".format(filename, TILES_DIR)) + file_counter += 1 + log.debug("Downloaded file {} to {}".format(file, TILES_DIR)) else: - log.debug("{} already exists in {}".format(etopo1_filename, TILES_DIR)) + log.debug("{} already exists in {}".format(self.filename, TILES_DIR)) + + # if only one file exists, clip file by extent + if file_counter == 1: + log.info("Starting tile processing ...") + raster_processing.clip_raster(self.filename, self.output_raster) + + def merge_tiles(self): + """Resamples and merges downloaded files to one raster tile.""" + pass diff --git a/openelevationservice/server/sources/gmted.py b/openelevationservice/server/sources/gmted.py index e078a98..6168f07 100644 --- a/openelevationservice/server/sources/gmted.py +++ b/openelevationservice/server/sources/gmted.py @@ -3,6 +3,7 @@ from openelevationservice import TILES_DIR, SETTINGS from openelevationservice.server.utils.logger import get_logger from openelevationservice.server.sources.provider import ProviderBase +from openelevationservice.server.db_import import raster_processing from os import path import requests @@ -14,27 +15,34 @@ class Gmted(ProviderBase): def __init__(self, base_url=SETTINGS['tables']['terrestrial']['sources']['gmted']['url'], + output_raster='terrestrial_raster.tif', bbox_extent=SETTINGS['tables']['terrestrial']['extent'], - xys=SETTINGS['tables']['terrestrial']['sources']['gmted'], + auth_parameters=None, + filename=None, + xys=SETTINGS['tables']['terrestrial']['sources']['gmted']['xys'], buffer=0.1 ): - super(Gmted, self).__init__(base_url, bbox_extent, xys, buffer) + super(Gmted, self).__init__(base_url, output_raster, bbox_extent, auth_parameters, filename, xys, buffer) def download_data(self): """ Download tiles and save to disk. """ - if self.bbox_extent['max_y'] > 60: - tiles_list = self.tile_selection() + tiles_list = self.tile_selection() - log.info("{} GMTED tile(s) will be downloaded.".format(len(tiles_list))) - for tile in tiles_list: - if not path.exists(path.join(TILES_DIR, tile[0])): - with open(path.join(TILES_DIR, tile[0]), 'wb') as f: - log.info("Starting to download {}".format(tile[0])) - f.write(requests.get(tile[1]).content) - log.info("Downloaded file {} to {}".format(tile[0], TILES_DIR)) - else: - log.info("{} already exists in {}".format(tile[0], TILES_DIR)) + log.info("{} GMTED tile(s) will be downloaded.".format(len(tiles_list))) + for tile in tiles_list: + if not path.exists(path.join(TILES_DIR, tile[0])): + with open(path.join(TILES_DIR, tile[0]), 'wb') as f: + log.info("Starting to download {}".format(tile[0])) + f.write(requests.get(tile[1]).content) + log.info("Downloaded file {} to {}".format(tile[0], TILES_DIR)) + else: + log.info("{} already exists in {}".format(tile[0], TILES_DIR)) + + # if only one file exists, clip file by extent + if len(tiles_list) == 1: + log.info("Starting tile processing ...") + raster_processing.clip_raster(tiles_list[0][0], self.output_raster) @staticmethod def intersects(buffered_bbox, bbox): @@ -80,6 +88,38 @@ def tile_selection(self): if self.intersects(buffered_bbox, bbox): filename, file_url = self.url_selection(x, y) tiles_to_download.append([filename, file_url]) - log.info(filename) return tiles_to_download + + def merge_tiles(self): + """Resamples and merges downloaded files to one raster tile.""" + + srtm_reference_file = 'srtm_clipped.tif' + gmted_merged = 'gmted_merged.tif' + gmted_resampled = 'gmted_resampled.tif' + + if self.bbox_extent['max_y'] > 60 >= self.bbox_extent['min_y'] or \ + self.bbox_extent['max_y'] > -60 >= self.bbox_extent['min_y']: + + # resample and merge tiles + log.info("Starting tile preprocessing ...") + srtm_merged_filename = raster_processing.merge_raster('srtm_*', 'srtm_merged.tif') + raster_processing.clip_raster(srtm_merged_filename, srtm_reference_file) + gmted_merged_filename = raster_processing.merge_raster('*_gmted_*.tif', gmted_merged) + + log.info("Starting tile resampling ...") + raster_processing.gmted_resampling(gmted_merged_filename, srtm_reference_file, gmted_resampled) + + log.info("Starting tile merging ...") + raster_processing.merge_raster(gmted_resampled, self.output_raster, srtm_reference_file) + + # only gmted data + elif self.bbox_extent['max_y'] > 60 < self.bbox_extent['min_y'] or \ + self.bbox_extent['max_y'] < -60 > self.bbox_extent['min_y']: + + # merge and clip raster by extent + log.info("Starting tile processing ...") + merged_filename = raster_processing.merge_raster('*_gmted_*.tif', gmted_merged) + + log.info("Starting tile merging ...") + raster_processing.clip_raster(merged_filename, self.output_raster) diff --git a/openelevationservice/server/sources/gv_at.py b/openelevationservice/server/sources/gv_at.py new file mode 100644 index 0000000..f691caf --- /dev/null +++ b/openelevationservice/server/sources/gv_at.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- + +from openelevationservice import TILES_DIR, SETTINGS +from openelevationservice.server.utils.logger import get_logger +from openelevationservice.server.sources.provider import ProviderBase +from openelevationservice.server.db_import import raster_processing + +from os import path +import requests +import zipfile + +from io import BytesIO + +log = get_logger(__name__) + + +class Gvat(ProviderBase): + + def __init__( + self, + base_url=SETTINGS['tables']['bathymetry']['sources']['gv_at']['url'], + output_raster='at_raster.tif', + bbox_extent=SETTINGS['tables']['bathymetry']['extent'], + auth_parameters=None, + filename='dhm_at_lamb_10m_2018.tif' + ): + super(Gvat, self).__init__(base_url, output_raster, bbox_extent, auth_parameters, filename) + + def download_data(self): + """Download tiles and save to disk.""" + + file_counter = 0 + + if not path.exists(path.join(TILES_DIR, self.filename)): + with zipfile.ZipFile(BytesIO(requests.get(self.base_url).content)) as zip_obj: + # Loop through the files in the zip + for file in zip_obj.namelist(): + if file.endswith('.tif'): + data = zip_obj.read(file) + # Write byte contents to file + with open(path.join(TILES_DIR, file), 'wb') as f: + f.write(data) + file_counter += 1 + log.debug("Downloaded file {} to {}".format(file, TILES_DIR)) + else: + log.debug("{} already exists in {}".format(self.filename, TILES_DIR)) + + # if only one file exists, clip file by extent + if file_counter == 1: + log.info("Starting tile processing ...") + raster_processing.clip_raster(self.filename, self.output_raster) + + def merge_tiles(self): + """Resamples and merges downloaded files to one raster tile.""" + pass diff --git a/openelevationservice/server/sources/provider.py b/openelevationservice/server/sources/provider.py index b5d29bf..d3609a2 100644 --- a/openelevationservice/server/sources/provider.py +++ b/openelevationservice/server/sources/provider.py @@ -5,15 +5,21 @@ class ProviderBase(ABC): - def __init__(self, base_url, bbox_extent, auth_parameters=None, xys=None, buffer=None): + def __init__(self, base_url, output_raster, bbox_extent=None, auth_parameters=None, filename=None, xys=None, buffer=None): self.base_url = base_url + self.output_raster = output_raster self.bbox_extent = bbox_extent self.auth_parameters = auth_parameters + self.filename = filename self.xys = xys self.buffer = buffer + super(ProviderBase, self).__init__() @abstractmethod def download_data(self): pass + # @abstractmethod + # def merge_tiles(self): + # pass diff --git a/openelevationservice/server/sources/srtm.py b/openelevationservice/server/sources/srtm.py index 65af7ad..9ed0f4b 100644 --- a/openelevationservice/server/sources/srtm.py +++ b/openelevationservice/server/sources/srtm.py @@ -3,6 +3,7 @@ from openelevationservice import TILES_DIR, SETTINGS from openelevationservice.server.utils.logger import get_logger from openelevationservice.server.sources.provider import ProviderBase +from openelevationservice.server.db_import import raster_processing from os import path, environ import requests @@ -19,65 +20,70 @@ class Srtm(ProviderBase): def __init__( self, base_url=SETTINGS['tables']['terrestrial']['sources']['srtm']['url'], + output_raster='terrestrial_raster.tif', bbox_extent=SETTINGS['tables']['terrestrial']['extent'], auth_parameters=SETTINGS['tables']['terrestrial']['sources']['srtm']['srtm_parameters'] ): - super(Srtm, self).__init__(base_url, bbox_extent, auth_parameters) + super(Srtm, self).__init__(base_url, output_raster, bbox_extent, auth_parameters) def download_data(self): + """Download tiles and save to disk.""" - if self.bbox_extent['min_y'] <= 60: + x_list = self.srtm_x_value(self.bbox_extent['min_x'], self.bbox_extent['max_x']) + y_list = self.srtm_y_value(self.bbox_extent['min_y'], self.bbox_extent['max_y']) - x_list = self.srtm_x_value(self.bbox_extent['min_x'], self.bbox_extent['max_x']) - y_list = self.srtm_y_value(self.bbox_extent['min_y'], self.bbox_extent['max_y']) + xy_range = [[int(x_list[0]), int(x_list[-1] + 1)], + [int(y_list[0]), int(y_list[-1] + 1)]] - xy_range = [[int(x_list[0]), int(x_list[-1] + 1)], - [int(y_list[0]), int(y_list[-1] + 1)]] + # Create session for authentication + session = requests.Session() - # Create session for authentication - session = requests.Session() + pw = environ.get('SRTMPASS') + user = environ.get('SRTMUSER') + if not user and not pw: + auth = tuple(self.auth_parameters.values()) + else: + auth = tuple([user, pw]) + session.auth = auth + + log.debug("SRTM credentials: {}".format(session.auth)) + + response = session.get(self.base_url) + + soup = BeautifulSoup(response.content, features="html.parser") + + file_counter = 0 + + # First find all 'a' tags starting href with srtm* + for link in soup.find_all('a', attrs={'href': lambda x: x.startswith('srtm') and x.endswith('.zip')}): + link_parsed = link.text.split('_') + link_x = int(link_parsed[1]) + link_y = int(link_parsed[2].split('.')[0]) + # Check if referenced geotif link is in xy_range + if link_y in range(*xy_range[1]) and link_x in range(*xy_range[0]): + # Then load the zip data in memory + if not path.exists(path.join(TILES_DIR, '_'.join(['srtm', str(link_x), str(link_y)]) + '.tif')): + with zipfile.ZipFile(BytesIO(session.get(self.base_url + link.text).content)) as zip_obj: + # Loop through the files in the zip + for filename in zip_obj.namelist(): + if filename.endswith('.tif'): + data = zip_obj.read(filename) + # Write byte contents to file + with open(path.join(TILES_DIR, filename), 'wb') as f: + f.write(data) + file_counter += 1 + log.debug("Downloaded file {} to {}".format(link.text, TILES_DIR)) + else: + log.debug("File {} already exists in {}".format(link.text, TILES_DIR)) - pw = environ.get('SRTMPASS') - user = environ.get('SRTMUSER') - if not user and not pw: - auth = tuple(self.auth_parameters.values()) - else: - auth = tuple([user, pw]) - session.auth = auth - - log.debug("SRTM credentials: {}".format(session.auth)) - - response = session.get(self.base_url) - - soup = BeautifulSoup(response.content, features="html.parser") - - # First find all 'a' tags starting href with srtm* - for link in soup.find_all('a', attrs={'href': lambda x: x.startswith('srtm') and x.endswith('.zip')}): - link_parsed = link.text.split('_') - link_x = int(link_parsed[1]) - link_y = int(link_parsed[2].split('.')[0]) - # Check if referenced geotif link is in xy_range - if link_y in range(*xy_range[1]) and link_x in range(*xy_range[0]): - log.info('yep') - # Then load the zip data in memory - if not path.exists(path.join(TILES_DIR, '_'.join(['srtm', str(link_x), str(link_y)]) + '.tif')): - with zipfile.ZipFile(BytesIO(session.get(self.base_url + link.text).content)) as zip_obj: - # Loop through the files in the zip - for filename in zip_obj.namelist(): - # Don't extract the readme.txt - # if filename != 'readme.txt': - if filename.endswith('.tif'): - data = zip_obj.read(filename) - # Write byte contents to file - with open(path.join(TILES_DIR, filename), 'wb') as f: - f.write(data) - log.debug("Downloaded file {} to {}".format(link.text, TILES_DIR)) - else: - log.debug("File {} already exists in {}".format(link.text, TILES_DIR)) + # if only one file exists, clip file by extent + if file_counter == 1: + log.info("Starting tile processing ...") + raster_processing.clip_raster(data, self.output_raster) @staticmethod def srtm_x_value(x_min, x_max): - """ Define SRTM x value download name. """ + """Define SRTM x value download name.""" x_value_list = [] @@ -100,7 +106,7 @@ def srtm_x_value(x_min, x_max): @staticmethod def srtm_y_value(y_min, y_max): - """ Define SRTM y value download name. """ + """Define SRTM y value download name.""" y_value_list = [] @@ -108,29 +114,29 @@ def srtm_y_value(y_min, y_max): lon_max = 60 y_srtm_value = 1 - # if lon > 60 or < -60: no data available - # TODO: no download instead of boarder tiles - if y_min > 60: - return y_srtm_value + for i in range(0, 120): - elif y_min < -60: - y_srtm_value = 24 - return y_srtm_value - - else: - for i in range(0, 120): - - if lon_max > y_min >= lon_min: - if y_min <= y_max: - y_value_list.append(y_srtm_value) - y_min += 5 - if y_min > 60: - return y_value_list - else: + if lon_max > y_min >= lon_min: + if y_min <= y_max: + y_value_list.append(y_srtm_value) + y_min += 5 + if y_min > 60: return y_value_list else: - lon_min -= 5 - lon_max -= 5 - y_srtm_value += 1 + return y_value_list + else: + lon_min -= 5 + lon_max -= 5 + y_srtm_value += 1 + + return y_value_list + + def merge_tiles(self): + """Resamples and merges downloaded files to one raster tile.""" + + # merge and clip raster by extent + log.info("Starting tile processing ...") + merged_filename = raster_processing.merge_raster('srtm_*', 'srtm_merged.tif') - return y_value_list + log.info("Starting tile merging ...") + raster_processing.clip_raster(merged_filename, self.output_raster) From 09348252719d525b894085f5ff8b3a25d26fa816 Mon Sep 17 00:00:00 2001 From: Isabell Klipper Date: Tue, 17 Dec 2019 11:36:21 +0100 Subject: [PATCH 09/11] Fixed create/drop multiple tables --- Dockerfile | 4 +- manage.py | 12 ++---- openelevationservice/__init__.py | 6 +-- .../server/api/querybuilder.py | 11 +++++- .../server/db_import/models.py | 38 +++++++++++++++++-- .../server/db_import/raster_processing.py | 2 - openelevationservice/server/sources/etopo1.py | 10 ++--- openelevationservice/server/sources/gv_at.py | 14 +++---- 8 files changed, 61 insertions(+), 36 deletions(-) diff --git a/Dockerfile b/Dockerfile index e8eb633..617474f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,8 +44,8 @@ RUN curl -sSL https://raw.githubusercontent.com/sdispater/poetry/master/get-poet ENV PATH "/root/.poetry/bin:/oes_venv/bin:${PATH}" # install dependencies via poetry -RUN poetry config settings.virtualenvs.create false -RUN poetry self:update --preview +# RUN poetry config settings.virtualenvs.create false +# RUN poetry self:update --preview RUN poetry config virtualenvs.create false COPY pyproject.toml poetry.lock README.rst / RUN poetry install diff --git a/manage.py b/manage.py index 2e50fab..0f72e83 100644 --- a/manage.py +++ b/manage.py @@ -13,9 +13,7 @@ @app.cli.command() def prepare(): - """ - Downloads SRTM tiles to disk. Can be specified over minx, maxx, miny, maxy. - """ + """Downloads SRTM tiles to disk. Can be specified over extent values in ops_settings.yml""" filestreams.download() log.info("Downloaded all files") @@ -23,6 +21,7 @@ def prepare(): @app.cli.command() def merge(): + """Merges downloaded single SRTM and GMTED tiles to one raster""" filestreams.merge_data() log.info("Merged downloaded files") @@ -48,11 +47,8 @@ def drop(): @app.cli.command() def importdata(): - """ - Imports all data found in ./tiles - """ + """Imports all data found in ./tiles""" + log.info("Starting to import data...") - filestreams.raster2pgsql() - log.info("Imported data successfully!") diff --git a/openelevationservice/__init__.py b/openelevationservice/__init__.py index 65d41df..5b42b36 100644 --- a/openelevationservice/__init__.py +++ b/openelevationservice/__init__.py @@ -9,11 +9,11 @@ TILES_DIR = path.join(getcwd(), 'tiles') if "TESTING" in environ: - SETTINGS['provider_parameters']['table_name_terrestrial'] = SETTINGS['provider_parameters']['table_name_terrestrial'] + '_test' + SETTINGS['provider_parameters']['tables']['terrestrial'] = SETTINGS['provider_parameters']['tables'][ + 'terrestrial'] + '_test' TILES_DIR = path.join(basedir, 'tests', 'tile') if not path.exists(TILES_DIR): makedirs(TILES_DIR) - -__version__ = "0.2.1" \ No newline at end of file +__version__ = "0.2.1" diff --git a/openelevationservice/server/api/querybuilder.py b/openelevationservice/server/api/querybuilder.py index 0ba3d1a..d0135c8 100644 --- a/openelevationservice/server/api/querybuilder.py +++ b/openelevationservice/server/api/querybuilder.py @@ -2,7 +2,8 @@ from openelevationservice import SETTINGS from openelevationservice.server.utils.logger import get_logger -from openelevationservice.server.db_import.models import db, Cgiar +from openelevationservice.server.db_import.models import db +from openelevationservice.server.db_import import models from openelevationservice.server.utils.custom_func import ST_SnapToGrid from openelevationservice.server.api.api_exceptions import InvalidUsage @@ -26,7 +27,13 @@ def _getModel(dataset): :rtype: SQLAlchemy model """ if dataset == 'srtm': - model = Cgiar + model = models.Terrestrial + + elif dataset == 'etopo1': + model = models.Bathymetry + + elif dataset == 'gv_at': + model = models.At return model diff --git a/openelevationservice/server/db_import/models.py b/openelevationservice/server/db_import/models.py index c558809..5138742 100644 --- a/openelevationservice/server/db_import/models.py +++ b/openelevationservice/server/db_import/models.py @@ -11,15 +11,45 @@ log = logger.get_logger(__name__) -class Cgiar(db.Model): - """Database model for SRTM v4.1 aka CGIAR dataset.""" +class Terrestrial(db.Model): + """Database model for SRTM v4.1 aka CGIAR and GMTED dataset.""" - for table in SETTINGS['provider_parameters']['tables']: + __tablename__ = list(SETTINGS['provider_parameters']['tables'].keys())[0] - __tablename__ = table + rid = db.Column(db.Integer, primary_key=True) + rast = db.Column(Raster) + + def __repr__(self): + return ''.format(self.rid, self.rast) + + +try: + # creates more tables if defined in ops_settings.yml + class Bathymetry(db.Model): + """Database model for Etopo1 dataset.""" + + __tablename__ = list(SETTINGS['provider_parameters']['tables'].keys())[1] + + rid = db.Column(db.Integer, primary_key=True) + rast = db.Column(Raster) + + def __repr__(self): + return ''.format(self.rid, self.rast) +except: + pass + + +try: + # creates more tables if defined in ops_settings.yml + class At(db.Model): + """Database model for austrian government dataset.""" + + __tablename__ = list(SETTINGS['provider_parameters']['tables'].keys())[2] rid = db.Column(db.Integer, primary_key=True) rast = db.Column(Raster) def __repr__(self): return ''.format(self.rid, self.rast) +except: + pass diff --git a/openelevationservice/server/db_import/raster_processing.py b/openelevationservice/server/db_import/raster_processing.py index 9d98762..aaff685 100644 --- a/openelevationservice/server/db_import/raster_processing.py +++ b/openelevationservice/server/db_import/raster_processing.py @@ -8,8 +8,6 @@ import fnmatch -# TODO: handle in memory files - def run_cmd(cmd): proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, diff --git a/openelevationservice/server/sources/etopo1.py b/openelevationservice/server/sources/etopo1.py index fcf3cf1..8061bd1 100644 --- a/openelevationservice/server/sources/etopo1.py +++ b/openelevationservice/server/sources/etopo1.py @@ -18,7 +18,7 @@ class Etopo1(ProviderBase): def __init__( self, - base_url=SETTINGS['tables']['bathymetry']['sources']['etopo1']['url'], + base_url=SETTINGS['tables']['further_sources']['sources']['etopo1']['url'], output_raster='bathymetry_raster.tif', bbox_extent=None, auth_parameters=None, @@ -29,8 +29,6 @@ def __init__( def download_data(self): """Download tiles and save to disk.""" - file_counter = 0 - if not path.exists(path.join(TILES_DIR, self.filename)): with zipfile.ZipFile(BytesIO(requests.get(self.base_url).content)) as zip_obj: # Loop through the files in the zip @@ -39,16 +37,14 @@ def download_data(self): # Write byte contents to file with open(path.join(TILES_DIR, file), 'wb') as f: f.write(data) - file_counter += 1 log.debug("Downloaded file {} to {}".format(file, TILES_DIR)) else: log.debug("{} already exists in {}".format(self.filename, TILES_DIR)) - # if only one file exists, clip file by extent - if file_counter == 1: + # if file exists, clip file by extent + if data: log.info("Starting tile processing ...") raster_processing.clip_raster(self.filename, self.output_raster) def merge_tiles(self): - """Resamples and merges downloaded files to one raster tile.""" pass diff --git a/openelevationservice/server/sources/gv_at.py b/openelevationservice/server/sources/gv_at.py index f691caf..fcf3017 100644 --- a/openelevationservice/server/sources/gv_at.py +++ b/openelevationservice/server/sources/gv_at.py @@ -13,14 +13,16 @@ log = get_logger(__name__) +# TODO: handle MemoryFileError -> download file is too large + class Gvat(ProviderBase): def __init__( self, - base_url=SETTINGS['tables']['bathymetry']['sources']['gv_at']['url'], + base_url=SETTINGS['tables']['further_sources']['sources']['gv_at']['url'], output_raster='at_raster.tif', - bbox_extent=SETTINGS['tables']['bathymetry']['extent'], + bbox_extent=SETTINGS['tables']['further_sources']['extent'], auth_parameters=None, filename='dhm_at_lamb_10m_2018.tif' ): @@ -29,8 +31,6 @@ def __init__( def download_data(self): """Download tiles and save to disk.""" - file_counter = 0 - if not path.exists(path.join(TILES_DIR, self.filename)): with zipfile.ZipFile(BytesIO(requests.get(self.base_url).content)) as zip_obj: # Loop through the files in the zip @@ -40,16 +40,14 @@ def download_data(self): # Write byte contents to file with open(path.join(TILES_DIR, file), 'wb') as f: f.write(data) - file_counter += 1 log.debug("Downloaded file {} to {}".format(file, TILES_DIR)) else: log.debug("{} already exists in {}".format(self.filename, TILES_DIR)) - # if only one file exists, clip file by extent - if file_counter == 1: + # if file exists, clip file by extent + if data: log.info("Starting tile processing ...") raster_processing.clip_raster(self.filename, self.output_raster) def merge_tiles(self): - """Resamples and merges downloaded files to one raster tile.""" pass From bf8c89f07ad07816f93487819c08d5ece90f10c9 Mon Sep 17 00:00:00 2001 From: Isabell Klipper Date: Tue, 17 Dec 2019 16:58:18 +0100 Subject: [PATCH 10/11] Modified table naming; Fixed extent parameter for clip_raster() --- manage.py | 14 +- .../server/db_import/filestreams.py | 150 +++++++++--------- .../server/db_import/models.py | 29 ++-- .../server/db_import/raster_processing.py | 6 +- openelevationservice/server/sources/etopo1.py | 9 +- openelevationservice/server/sources/gmted.py | 6 +- openelevationservice/server/sources/gv_at.py | 7 +- openelevationservice/server/sources/srtm.py | 4 +- 8 files changed, 108 insertions(+), 117 deletions(-) diff --git a/manage.py b/manage.py index 0f72e83..f34e8bd 100644 --- a/manage.py +++ b/manage.py @@ -30,19 +30,21 @@ def merge(): @app.cli.command() def create(): """Creates all tables defined in models.py""" - + db.create_all() - for table in SETTINGS['provider_parameters']['tables']: - log.info("Table {} was created.".format(table)) - + for table in SETTINGS['provider_parameters']['tables'].items(): + if table[1]: + log.info("Table {} was created.".format(table[1])) + @app.cli.command() def drop(): """Drops all tables defined in models.py""" db.drop_all() - for table in SETTINGS['provider_parameters']['tables']: - log.info("Table {} was dropped.".format(table)) + for table in SETTINGS['provider_parameters']['tables'].items(): + if table[1]: + log.info("Table {} was dropped.".format(table[1])) @app.cli.command() diff --git a/openelevationservice/server/db_import/filestreams.py b/openelevationservice/server/db_import/filestreams.py index f182614..4e2c719 100644 --- a/openelevationservice/server/db_import/filestreams.py +++ b/openelevationservice/server/db_import/filestreams.py @@ -18,41 +18,38 @@ def download(): extent_settings = SETTINGS['tables']['terrestrial']['extent'] download_list = [] - if not SETTINGS['provider_parameters']['tables']: - log.error("Please define at least one table in ops_settings.yml") - - else: - # for table in SETTINGS['provider_parameters']['tables']: - if 'terrestrial' in SETTINGS['provider_parameters']['tables']: - - # only SRTM data download - if 60 >= extent_settings['max_y'] >= -60: - provider = PROVIDER_MAPPING['terrestrial']['srtm']() + if SETTINGS['provider_parameters']['tables']['terrestrial']: + + # only SRTM data download + if 60 >= extent_settings['max_y'] >= -60: + provider = PROVIDER_MAPPING['terrestrial']['srtm']() + download_list.append(provider) + + # SRTM and GMTED data download + elif extent_settings['max_y'] > 60 >= extent_settings['min_y'] or \ + extent_settings['max_y'] > -60 >= extent_settings['min_y']: + for source in SETTINGS['tables']['terrestrial']['sources']: + provider = PROVIDER_MAPPING['terrestrial'][source]() download_list.append(provider) - # SRTM and GMTED data download - elif extent_settings['max_y'] > 60 >= extent_settings['min_y'] or \ - extent_settings['max_y'] > -60 >= extent_settings['min_y']: - for source in SETTINGS['tables']['terrestrial']['sources']: - provider = PROVIDER_MAPPING['terrestrial'][source]() - download_list.append(provider) + # only GMTED data download + else: + provider = PROVIDER_MAPPING['terrestrial']['gmted']() + download_list.append(provider) - # only GMTED data download - else: - provider = PROVIDER_MAPPING['terrestrial']['gmted']() - download_list.append(provider) + if SETTINGS['provider_parameters']['tables']['bathymetry']: + provider = PROVIDER_MAPPING['etopo1']() + download_list.append(provider) - else: - for p in PROVIDER_MAPPING: - if p != 'terrestrial': - provider = PROVIDER_MAPPING[p]() - download_list.append(provider) + if SETTINGS['provider_parameters']['tables']['at']: + provider = PROVIDER_MAPPING['gv_at']() + download_list.append(provider) - for download_provider in download_list: - try: - download_provider.download_data() - except Exception as err: - log.error(err) + for download_provider in download_list: + try: + download_provider.download_data() + except Exception as err: + log.error(err) def merge_data(): @@ -61,34 +58,30 @@ def merge_data(): extent_settings = SETTINGS['tables']['terrestrial']['extent'] merge_list = [] - if not SETTINGS['provider_parameters']['tables']: - log.error("Please define at least one table in ops_settings.yml") - - else: - for table in SETTINGS['provider_parameters']['tables']: - if 'terrestrial' in table: + if SETTINGS['provider_parameters']['tables']['terrestrial']: + # only SRTM data + if 60 >= extent_settings['max_y'] >= -60: + provider = PROVIDER_MAPPING['terrestrial']['srtm']() + merge_list.append(provider) - # only SRTM data - if 60 >= extent_settings['max_y'] >= -60: - provider = PROVIDER_MAPPING['terrestrial']['srtm']() - merge_list.append(provider) + # GMTED data or GMTED and SRTM data + elif extent_settings['max_y'] > 60 or extent_settings['min_y'] < -60: + provider = PROVIDER_MAPPING['terrestrial']['gmted']() + merge_list.append(provider) - # GMTED data or GMTED and SRTM data - elif extent_settings['max_y'] > 60 or extent_settings['min_y'] < -60: - provider = PROVIDER_MAPPING['terrestrial']['gmted']() - merge_list.append(provider) + if SETTINGS['provider_parameters']['tables']['bathymetry']: + provider = PROVIDER_MAPPING['etopo1']() + merge_list.append(provider) - else: - for p in PROVIDER_MAPPING: - if p != 'terrestrial': - provider = PROVIDER_MAPPING[p]() - merge_list.append(provider) + if SETTINGS['provider_parameters']['tables']['at']: + provider = PROVIDER_MAPPING['gv_at']() + merge_list.append(provider) - for merge_provider in merge_list: - try: - merge_provider.merge_tiles() - except Exception as err: - log.error(err) + for merge_provider in merge_list: + try: + merge_provider.merge_tiles() + except Exception as err: + log.error(err) def raster2pgsql(): @@ -107,27 +100,28 @@ def raster2pgsql(): env_current = environ.copy() env_current['PGPASSWORD'] = pg_settings['password'] - for table in SETTINGS['provider_parameters']['tables']: - filename = path.join(TILES_DIR + '/' + table + '_raster.tif') - - # Tried to import every raster individually by user-specified xyrange - # similar to download(), but raster2pgsql fuck it up somehow.. The PostGIS - # raster will not be what one would expect. Instead doing a bulk import of all files. - cmd_raster2pgsql = r"raster2pgsql -s 4326 -a -C -M -t 50x50 {filename} {table_name} | psql -q -h {host} -p {port} -U {user_name} -d {db_name}" - # -s: raster SRID - # -a: append to table (assumes it's been create with 'create()') - # -C: apply all raster Constraints - # -M: vacuum analyze after import - # -t: specifies the pixel size of each row. Important to keep low for performance! - - cmd_raster2pgsql = cmd_raster2pgsql.format(**{'filename': filename, 'table_name': table, **pg_settings}) - - proc = subprocess.Popen(cmd_raster2pgsql, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - shell=True, - env=env_current) - - return_code = proc.wait() - if return_code: - raise subprocess.CalledProcessError(return_code, cmd_raster2pgsql) + for table in SETTINGS['provider_parameters']['tables'].items(): + if table[1] is not None: + filename = path.join(TILES_DIR + '/' + table[0] + '_raster.tif') + + # Tried to import every raster individually by user-specified xyrange + # similar to download(), but raster2pgsql fuck it up somehow.. The PostGIS + # raster will not be what one would expect. Instead doing a bulk import of all files. + cmd_raster2pgsql = r"raster2pgsql -s 4326 -a -C -M -t 50x50 {filename} {table_name} | psql -q -h {host} -p {port} -U {user_name} -d {db_name}" + # -s: raster SRID + # -a: append to table (assumes it's been create with 'create()') + # -C: apply all raster Constraints + # -M: vacuum analyze after import + # -t: specifies the pixel size of each row. Important to keep low for performance! + + cmd_raster2pgsql = cmd_raster2pgsql.format(**{'filename': filename, 'table_name': table[1], **pg_settings}) + + proc = subprocess.Popen(cmd_raster2pgsql, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + shell=True, + env=env_current) + + return_code = proc.wait() + if return_code: + raise subprocess.CalledProcessError(return_code, cmd_raster2pgsql) diff --git a/openelevationservice/server/db_import/models.py b/openelevationservice/server/db_import/models.py index 5138742..f6d4046 100644 --- a/openelevationservice/server/db_import/models.py +++ b/openelevationservice/server/db_import/models.py @@ -11,45 +11,40 @@ log = logger.get_logger(__name__) -class Terrestrial(db.Model): - """Database model for SRTM v4.1 aka CGIAR and GMTED dataset.""" +if SETTINGS['provider_parameters']['tables']['terrestrial']: + class Terrestrial(db.Model): + """Database model for SRTM v4.1 aka CGIAR and GMTED dataset.""" - __tablename__ = list(SETTINGS['provider_parameters']['tables'].keys())[0] + __tablename__ = SETTINGS['provider_parameters']['tables']['terrestrial'] - rid = db.Column(db.Integer, primary_key=True) - rast = db.Column(Raster) + rid = db.Column(db.Integer, primary_key=True) + rast = db.Column(Raster) - def __repr__(self): - return ''.format(self.rid, self.rast) + def __repr__(self): + return ''.format(self.rid, self.rast) -try: - # creates more tables if defined in ops_settings.yml +if SETTINGS['provider_parameters']['tables']['bathymetry']: class Bathymetry(db.Model): """Database model for Etopo1 dataset.""" - __tablename__ = list(SETTINGS['provider_parameters']['tables'].keys())[1] + __tablename__ = SETTINGS['provider_parameters']['tables']['bathymetry'] rid = db.Column(db.Integer, primary_key=True) rast = db.Column(Raster) def __repr__(self): return ''.format(self.rid, self.rast) -except: - pass -try: - # creates more tables if defined in ops_settings.yml +if SETTINGS['provider_parameters']['tables']['at']: class At(db.Model): """Database model for austrian government dataset.""" - __tablename__ = list(SETTINGS['provider_parameters']['tables'].keys())[2] + __tablename__ = SETTINGS['provider_parameters']['tables']['at'] rid = db.Column(db.Integer, primary_key=True) rast = db.Column(Raster) def __repr__(self): return ''.format(self.rid, self.rast) -except: - pass diff --git a/openelevationservice/server/db_import/raster_processing.py b/openelevationservice/server/db_import/raster_processing.py index aaff685..aed610b 100644 --- a/openelevationservice/server/db_import/raster_processing.py +++ b/openelevationservice/server/db_import/raster_processing.py @@ -54,7 +54,7 @@ def merge_raster(input_filename, output_filename, reference=None): return output_filename -def clip_raster(merged_filename_extent, output_filename): +def clip_raster(merged_filename_extent, output_filename, extent): """ Clip merged raster by defined extent. """ if not path.exists(path.join(TILES_DIR, output_filename)): @@ -64,7 +64,9 @@ def clip_raster(merged_filename_extent, output_filename): merged_data = gdal.Open(merged_file, gdalconst.GA_ReadOnly) output_clip = path.join(TILES_DIR + '/' + output_filename) - extent = str(list(SETTINGS['tables']['terrestrial']['extent'].values()))[1:-1] + extent = str(list(extent.values()))[1:-1] + + # extent = str(list(SETTINGS['tables']['terrestrial']['extent'].values()))[1:-1] cmd = {'extent': extent, 'outfile_format': 'GTiff', diff --git a/openelevationservice/server/sources/etopo1.py b/openelevationservice/server/sources/etopo1.py index 8061bd1..9332bd5 100644 --- a/openelevationservice/server/sources/etopo1.py +++ b/openelevationservice/server/sources/etopo1.py @@ -20,7 +20,7 @@ def __init__( self, base_url=SETTINGS['tables']['further_sources']['sources']['etopo1']['url'], output_raster='bathymetry_raster.tif', - bbox_extent=None, + bbox_extent=SETTINGS['tables']['further_sources']['extent'], auth_parameters=None, filename='ETOPO1_Bed_g_geotiff.tif' ): @@ -41,10 +41,9 @@ def download_data(self): else: log.debug("{} already exists in {}".format(self.filename, TILES_DIR)) - # if file exists, clip file by extent - if data: - log.info("Starting tile processing ...") - raster_processing.clip_raster(self.filename, self.output_raster) + # clip raster tile by defined extent + log.info("Starting tile processing ...") + raster_processing.clip_raster(self.filename, self.output_raster, self.bbox_extent) def merge_tiles(self): pass diff --git a/openelevationservice/server/sources/gmted.py b/openelevationservice/server/sources/gmted.py index 6168f07..4d79921 100644 --- a/openelevationservice/server/sources/gmted.py +++ b/openelevationservice/server/sources/gmted.py @@ -42,7 +42,7 @@ def download_data(self): # if only one file exists, clip file by extent if len(tiles_list) == 1: log.info("Starting tile processing ...") - raster_processing.clip_raster(tiles_list[0][0], self.output_raster) + raster_processing.clip_raster(tiles_list[0][0], self.output_raster, self.bbox_extent) @staticmethod def intersects(buffered_bbox, bbox): @@ -104,7 +104,7 @@ def merge_tiles(self): # resample and merge tiles log.info("Starting tile preprocessing ...") srtm_merged_filename = raster_processing.merge_raster('srtm_*', 'srtm_merged.tif') - raster_processing.clip_raster(srtm_merged_filename, srtm_reference_file) + raster_processing.clip_raster(srtm_merged_filename, srtm_reference_file, self.bbox_extent) gmted_merged_filename = raster_processing.merge_raster('*_gmted_*.tif', gmted_merged) log.info("Starting tile resampling ...") @@ -122,4 +122,4 @@ def merge_tiles(self): merged_filename = raster_processing.merge_raster('*_gmted_*.tif', gmted_merged) log.info("Starting tile merging ...") - raster_processing.clip_raster(merged_filename, self.output_raster) + raster_processing.clip_raster(merged_filename, self.output_raster, self.bbox_extent) diff --git a/openelevationservice/server/sources/gv_at.py b/openelevationservice/server/sources/gv_at.py index fcf3017..3afbca7 100644 --- a/openelevationservice/server/sources/gv_at.py +++ b/openelevationservice/server/sources/gv_at.py @@ -44,10 +44,9 @@ def download_data(self): else: log.debug("{} already exists in {}".format(self.filename, TILES_DIR)) - # if file exists, clip file by extent - if data: - log.info("Starting tile processing ...") - raster_processing.clip_raster(self.filename, self.output_raster) + # clip raster tile by defined extent + log.info("Starting tile processing ...") + raster_processing.clip_raster(self.filename, self.output_raster, self.bbox_extent) def merge_tiles(self): pass diff --git a/openelevationservice/server/sources/srtm.py b/openelevationservice/server/sources/srtm.py index 9ed0f4b..5bcec1a 100644 --- a/openelevationservice/server/sources/srtm.py +++ b/openelevationservice/server/sources/srtm.py @@ -79,7 +79,7 @@ def download_data(self): # if only one file exists, clip file by extent if file_counter == 1: log.info("Starting tile processing ...") - raster_processing.clip_raster(data, self.output_raster) + raster_processing.clip_raster(data, self.output_raster, self.bbox_extent) @staticmethod def srtm_x_value(x_min, x_max): @@ -139,4 +139,4 @@ def merge_tiles(self): merged_filename = raster_processing.merge_raster('srtm_*', 'srtm_merged.tif') log.info("Starting tile merging ...") - raster_processing.clip_raster(merged_filename, self.output_raster) + raster_processing.clip_raster(merged_filename, self.output_raster, self.bbox_extent) From 92937e1e6bad57cc2ccdb0f293fa73aff5f62ed1 Mon Sep 17 00:00:00 2001 From: Isabell Klipper Date: Mon, 13 Jan 2020 10:12:52 +0100 Subject: [PATCH 11/11] updated file structure --- manage.py | 2 +- .../server/ops_settings.sample.yml | 46 ++++++++++--------- ops_settings_docker.sample.yml | 46 ++++++++++--------- 3 files changed, 49 insertions(+), 45 deletions(-) diff --git a/manage.py b/manage.py index f34e8bd..af7258d 100644 --- a/manage.py +++ b/manage.py @@ -49,7 +49,7 @@ def drop(): @app.cli.command() def importdata(): - """Imports all data found in ./tiles""" + """Imports all '_raster.tif' files found in ./tiles""" log.info("Starting to import data...") filestreams.raster2pgsql() diff --git a/openelevationservice/server/ops_settings.sample.yml b/openelevationservice/server/ops_settings.sample.yml index 8fcb011..1927c17 100644 --- a/openelevationservice/server/ops_settings.sample.yml +++ b/openelevationservice/server/ops_settings.sample.yml @@ -3,45 +3,47 @@ attribution: "service by https://openrouteservice.org | data by http://srtm.csi. coord_precision: 1e-6 maximum_nodes: 2000 provider_parameters: - table_name_terrestrial: terrestrial - table_name_bathymetry: bathymetry + tables: + terrestrial: t + bathymetry: b + at: db_name: oes_db - user_name: postgres - password: postgres + user_name: gis + password: gis host: localhost port: 5432 tables: terrestrial: - table_name: terrestrial sources: srtm: url: http://data.cgiar-csi.org/srtm/tiles/GeoTIFF/ srtm_parameters: - user: data_public - password: GDdci + user: user + password: password gmted: - url: http://edcintl.cr.usgs.gov/downloads/sciweb1/shared/topo/downloads/GMTED/Global_tiles_GMTED - ys: [-90, -70, -50, -30, -10, 10, 30, 50, 70] - xs: [-180, -150, -120, -90, -60, -30, 0, 30, 60, 90, 120, 150] + url: https://edcintl.cr.usgs.gov/downloads/sciweb1/shared/topo/downloads/GMTED/Global_tiles_GMTED + xys: + ys: [-90, -70, -50, -30, -10, 10, 30, 50, 70] + xs: [-180, -150, -120, -90, -60, -30, 0, 30, 60, 90, 120, 150] extent: -# min_x: 8.305664 -# min_y: 61.653379 -# max_x: 8.931885 -# max_y: 61.944118 - min_x: 8.257599 - min_y: 49.185294 - max_x: 8.997803 - max_y: 49.624056 + min_x: 8.305664 + min_y: 61.653379 + max_x: 8.931885 + max_y: 61.944118 +# min_x: 8.257599 +# min_y: 49.185294t +# max_x: 8.997803 +# max_y: 49.624056 # min_x: -1.406250 # min_y: 57.088515 # max_x: 15.820313 # max_y: 64.091408 - - bathymetry: - table_name: bathymetry + further_sources: sources: etopo1: - url: https://www.ngdc.noaa.gov/mgg/global/relief/ETOPO1/data/bedrock/grid_registered/georeferenced_tiff/ETOPO1_Bed_g_geotiff.zip + url: https://www.ngdc.noaa.gov/mgg/global/relief/ETOPO1/data/bedrock/grid_registered/georeferenced_tiff/ETOPO1_Bed_g_geotiff.zip + gv_at: + url: https://gis.ktn.gv.at/OGD/Geographie_Planung/ogd-10m-at.zip extent: min_x: 8.662891 min_y: 49.395223 diff --git a/ops_settings_docker.sample.yml b/ops_settings_docker.sample.yml index 8fcb011..1927c17 100644 --- a/ops_settings_docker.sample.yml +++ b/ops_settings_docker.sample.yml @@ -3,45 +3,47 @@ attribution: "service by https://openrouteservice.org | data by http://srtm.csi. coord_precision: 1e-6 maximum_nodes: 2000 provider_parameters: - table_name_terrestrial: terrestrial - table_name_bathymetry: bathymetry + tables: + terrestrial: t + bathymetry: b + at: db_name: oes_db - user_name: postgres - password: postgres + user_name: gis + password: gis host: localhost port: 5432 tables: terrestrial: - table_name: terrestrial sources: srtm: url: http://data.cgiar-csi.org/srtm/tiles/GeoTIFF/ srtm_parameters: - user: data_public - password: GDdci + user: user + password: password gmted: - url: http://edcintl.cr.usgs.gov/downloads/sciweb1/shared/topo/downloads/GMTED/Global_tiles_GMTED - ys: [-90, -70, -50, -30, -10, 10, 30, 50, 70] - xs: [-180, -150, -120, -90, -60, -30, 0, 30, 60, 90, 120, 150] + url: https://edcintl.cr.usgs.gov/downloads/sciweb1/shared/topo/downloads/GMTED/Global_tiles_GMTED + xys: + ys: [-90, -70, -50, -30, -10, 10, 30, 50, 70] + xs: [-180, -150, -120, -90, -60, -30, 0, 30, 60, 90, 120, 150] extent: -# min_x: 8.305664 -# min_y: 61.653379 -# max_x: 8.931885 -# max_y: 61.944118 - min_x: 8.257599 - min_y: 49.185294 - max_x: 8.997803 - max_y: 49.624056 + min_x: 8.305664 + min_y: 61.653379 + max_x: 8.931885 + max_y: 61.944118 +# min_x: 8.257599 +# min_y: 49.185294t +# max_x: 8.997803 +# max_y: 49.624056 # min_x: -1.406250 # min_y: 57.088515 # max_x: 15.820313 # max_y: 64.091408 - - bathymetry: - table_name: bathymetry + further_sources: sources: etopo1: - url: https://www.ngdc.noaa.gov/mgg/global/relief/ETOPO1/data/bedrock/grid_registered/georeferenced_tiff/ETOPO1_Bed_g_geotiff.zip + url: https://www.ngdc.noaa.gov/mgg/global/relief/ETOPO1/data/bedrock/grid_registered/georeferenced_tiff/ETOPO1_Bed_g_geotiff.zip + gv_at: + url: https://gis.ktn.gv.at/OGD/Geographie_Planung/ogd-10m-at.zip extent: min_x: 8.662891 min_y: 49.395223