tasks.py

#########################################################################
#
# Copyright (C) 2016 OSGeo
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
#########################################################################
import os
import re
import ast
import json
import time
import docker
import socket
import logging
import datetime

from urllib.parse import urlparse
from invoke import task

BOOTSTRAP_IMAGE_CHEIP = 'codenvy/che-ip:nightly'

logger = logging.getLogger(__name__)


@task
def waitfordbs(ctx):
    print("**************************databases*******************************")
    db_host = os.getenv('DATABASE_HOST', 'db')
    ctx.run(f"/usr/bin/wait-for-databases {db_host}", pty=True)


@task
def waitforgeoserver(ctx):
    print("****************************geoserver********************************")
    while not _rest_api_availability(f"{os.environ['GEOSERVER_LOCATION']}rest"):
        print("Wait for GeoServer API availability...")
    print("GeoServer is available for HTTP calls!")


@task
def update(ctx):
    print("***************************setting env*********************************")
    ctx.run("env", pty=True)
    pub_ip = _geonode_public_host_ip()
    print(f"Public Hostname or IP is {pub_ip}")
    pub_port = _geonode_public_port()
    print(f"Public PORT is {pub_port}")
    pub_protocol = 'https' if pub_port == '443' else 'http'
    if pub_protocol == 'https' or pub_port == '80':
        pub_port = None
    db_url = _update_db_connstring()
    geodb_url = _update_geodb_connstring()
    service_ready = False
    while not service_ready:
        try:
            socket.gethostbyname('geonode')
            service_ready = True
        except Exception:
            time.sleep(10)

    override_env = "$HOME/.override_env"
    if os.path.exists(override_env):
        os.remove(override_env)
    else:
        print(f"Can not delete the {override_env} file as it doesn't exists")

    if pub_port:
        siteurl = f'{pub_protocol}://{pub_ip}:{pub_port}/'
        gs_pub_loc = f'http://{pub_ip}:{pub_port}/geoserver/'
    else:
        siteurl = f'{pub_protocol}://{pub_ip}/'
        gs_pub_loc = f'http://{pub_ip}/geoserver/'
    envs = {
        "local_settings": str(_localsettings()),
        "siteurl": os.environ.get('SITEURL', siteurl),
        "geonode_docker_host": str(socket.gethostbyname('geonode')),
        "public_protocol": pub_protocol,
        "public_fqdn": str(pub_ip) + str(f':{pub_port}' if pub_port else ''),
        "public_host": str(pub_ip),
        "dburl": os.environ.get('DATABASE_URL', db_url),
        "geodburl": os.environ.get('GEODATABASE_URL', geodb_url),
        "static_root": os.environ.get('STATIC_ROOT', '/mnt/volumes/statics/static/'),
        "media_root": os.environ.get('MEDIA_ROOT', '/mnt/volumes/statics/uploaded/'),
        "geoip_path": os.environ.get('GEOIP_PATH', '/mnt/volumes/statics/geoip.db'),
        "monitoring": os.environ.get('MONITORING_ENABLED', True),
        "monitoring_host_name": os.environ.get('MONITORING_HOST_NAME', 'geonode'),
        "monitoring_service_name": os.environ.get('MONITORING_SERVICE_NAME', 'local-geonode'),
        "monitoring_data_ttl": os.environ.get('MONITORING_DATA_TTL', 7),
        "geonode_geodb_passwd": os.environ.get('GEONODE_GEODATABASE_PASSWORD', 'geonode_data'),
        "default_backend_datastore": os.environ.get('DEFAULT_BACKEND_DATASTORE', 'datastore'),
        "geonode_db_passwd": os.environ.get('GEONODE_DATABASE_PASSWORD', 'geonode'),
        "geonode_geodb": os.environ.get('GEONODE_GEODATABASE', 'geonode_data'),
        "db_url": os.environ.get('DATABASE_URL', 'postgis://geonode:geonode@db:5432/geonode'),
        "geodb_url": os.environ.get('GEODATABASE_URL', 'postgis://geonode:geonode@db:5432/geonode_data'),
        "geonode_db": os.environ.get('GEONODE_DATABASE', 'geonode'),
        "gs_loc": os.environ.get('GEOSERVER_LOCATION', 'http://geoserver:8080/geoserver/'),
        "gs_web_ui_loc": os.environ.get('GEOSERVER_WEB_UI_LOCATION', gs_pub_loc),
        "gs_pub_loc": os.environ.get('GEOSERVER_PUBLIC_LOCATION', gs_pub_loc),
        "gs_admin_pwd": os.environ.get('GEOSERVER_ADMIN_PASSWORD', 'geoserver'),
        "override_fn": override_env
    }
    try:
        current_allowed = ast.literal_eval(
            os.getenv('ALLOWED_HOSTS') or
            "['{public_fqdn}', '{public_host}', 'localhost', 'django', 'geonode',]".format(**envs))
    except ValueError:
        current_allowed = []
    current_allowed.extend([str(pub_ip), f'{pub_ip}:{pub_port}'])
    allowed_hosts = [str(c) for c in current_allowed] + ['"geonode"', '"django"']

    ctx.run("echo export DJANGO_SETTINGS_MODULE=\
{local_settings} >> {override_fn}".format(**envs), pty=True)
    ctx.run("echo export MONITORING_ENABLED=\
{monitoring} >> {override_fn}".format(**envs), pty=True)
    ctx.run("echo export MONITORING_HOST_NAME=\
{monitoring_host_name} >> {override_fn}".format(**envs), pty=True)
    ctx.run("echo export MONITORING_SERVICE_NAME=\
{monitoring_service_name} >> {override_fn}".format(**envs), pty=True)
    ctx.run("echo export MONITORING_DATA_TTL=\
{monitoring_data_ttl} >> {override_fn}".format(**envs), pty=True)
    ctx.run("echo export GEOIP_PATH=\
{geoip_path} >> {override_fn}".format(**envs), pty=True)
    ctx.run("echo export GEONODE_GEODATABASE_PASSWORD=\
{geonode_geodb_passwd} >> {override_fn}".format(**envs), pty=True)
    ctx.run("echo export DEFAULT_BACKEND_DATASTORE=\
{default_backend_datastore} >> {override_fn}".format(**envs), pty=True)
    ctx.run("echo export GEONODE_DATABASE_PASSWORD=\
{geonode_db_passwd} >> {override_fn}".format(**envs), pty=True)
    ctx.run("echo export GEONODE_GEODATABASE=\
{geonode_geodb} >> {override_fn}".format(**envs), pty=True)
    ctx.run("echo export DATABASE_URL=\
{db_url} >> {override_fn}".format(**envs), pty=True)
    ctx.run("echo export GEODATABASE_URL=\
{geodb_url} >> {override_fn}".format(**envs), pty=True)
    ctx.run("echo export GEONODE_DATABASE=\
{geonode_db} >> {override_fn}".format(**envs), pty=True)
    ctx.run("echo export GEOSERVER_LOCATION=\
{gs_loc} >> {override_fn}".format(**envs), pty=True)
    ctx.run("echo export GEOSERVER_WEB_UI_LOCATION=\
{gs_web_ui_loc} >> {override_fn}".format(**envs), pty=True)
    ctx.run("echo export GEOSERVER_PUBLIC_LOCATION=\
{gs_pub_loc} >> {override_fn}".format(**envs), pty=True)
    ctx.run("echo export GEOSERVER_ADMIN_PASSWORD=\
{gs_admin_pwd} >> {override_fn}".format(**envs), pty=True)
    ctx.run("echo export SITEURL=\
{siteurl} >> {override_fn}".format(**envs), pty=True)
    ctx.run('echo export ALLOWED_HOSTS=\
"\\"{}\\"" >> {override_fn}'.format(allowed_hosts, **envs), pty=True)
    ctx.run("echo export DATABASE_URL=\
{dburl} >> {override_fn}".format(**envs), pty=True)
    ctx.run("echo export GEODATABASE_URL=\
{geodburl} >> {override_fn}".format(**envs), pty=True)
    ctx.run("echo export STATIC_ROOT=\
{static_root} >> {override_fn}".format(**envs), pty=True)
    ctx.run("echo export MEDIA_ROOT=\
{media_root} >> {override_fn}".format(**envs), pty=True)
    ctx.run("echo export GEOIP_PATH=\
{geoip_path} >> {override_fn}".format(**envs), pty=True)
    ctx.run("echo export LOGIN_URL=\
{siteurl}account/login/ >> {override_fn}".format(**envs), pty=True)
    ctx.run("echo export LOGOUT_URL=\
{siteurl}account/logout/ >> {override_fn}".format(**envs), pty=True)
    ctx.run("echo export LOGIN_REDIRECT_URL=\
{siteurl} >> {override_fn}".format(**envs), pty=True)
    ctx.run("echo export LOGOUT_REDIRECT_URL=\
{siteurl} >> {override_fn}".format(**envs), pty=True)
    ctx.run(f"source {override_env}", pty=True)
    print("****************************finalize env**********************************")
    ctx.run("env", pty=True)


@task
def migrations(ctx):
    print("**************************migrations*******************************")
    ctx.run(f"python manage.py migrate --noinput --settings={_localsettings()}", pty=True)
    try:
        ctx.run(f"python manage.py rebuild_index --noinput --settings={_localsettings()}", pty=True)
    except Exception:
        pass


@task
def statics(ctx):
    print("**************************statics*******************************")
    try:
        ctx.run('mkdir -p /mnt/volumes/statics/{static,uploads}')
        ctx.run(f"python manage.py collectstatic --noinput --settings={_localsettings()}", pty=True)
    except Exception:
        import traceback
        traceback.print_exc()


@task
def prepare(ctx):
    print("**********************prepare fixture***************************")
    ctx.run("rm -rf /tmp/default_oauth_apps_docker.json", pty=True)
    _prepare_oauth_fixture()
    ctx.run("rm -rf /tmp/default_site.json", pty=True)
    _prepare_site_fixture()
    # Updating OAuth2 Service Config
    new_ext_ip = os.environ['SITEURL']
    client_id = os.environ['OAUTH2_CLIENT_ID']
    client_secret = os.environ['OAUTH2_CLIENT_SECRET']
    oauth_config = "/geoserver_data/data/security/filter/geonode-oauth2/config.xml"
    ctx.run(
        f'sed -i "s|<cliendId>.*</cliendId>|<cliendId>{client_id}</cliendId>|g" {oauth_config}',
        pty=True)
    ctx.run(
        f'sed -i "s|<clientSecret>.*</clientSecret>|<clientSecret>{client_secret}</clientSecret>|g" {oauth_config}',
        pty=True)
    ctx.run(
        f'sed -i "s|<userAuthorizationUri>.*</userAuthorizationUri>|<userAuthorizationUri>{new_ext_ip}o/authorize/</userAuthorizationUri>|g" {oauth_config}',  # noqa
        pty=True)
    ctx.run(
        f'sed -i "s|<redirectUri>.*</redirectUri>|<redirectUri>{new_ext_ip}geoserver/index.html</redirectUri>|g" {oauth_config}',  # noqa
        pty=True)
    ctx.run(
        f'sed -i "s|<logoutUri>.*</logoutUri>|<logoutUri>{new_ext_ip}account/logout/</logoutUri>|g" {oauth_config}',
        pty=True)


@task
def fixtures(ctx):
    print("**************************fixtures********************************")
    ctx.run(f"python manage.py loaddata sample_admin \
--settings={_localsettings()}", pty=True)
    ctx.run(f"python manage.py loaddata /tmp/default_oauth_apps_docker.json \
--settings={_localsettings()}", pty=True)
    ctx.run(f"python manage.py loaddata geonode/base/fixtures/initial_data.json \
--settings={_localsettings()}", pty=True)


@task
def collectstatic(ctx):
    print("************************static artifacts******************************")
    ctx.run(f"django-admin.py collectstatic --noinput \
--settings={_localsettings()}", pty=True)


@task
def geoserverfixture(ctx):
    print("********************geoserver fixture********************************")
    _geoserver_info_provision(f"{os.environ['GEOSERVER_LOCATION']}rest/")


@task
def monitoringfixture(ctx):
    print("*******************monitoring fixture********************************")
    ctx.run("rm -rf /tmp/default_monitoring_apps_docker.json", pty=True)
    _prepare_monitoring_fixture()
    try:
        ctx.run(f"django-admin.py loaddata /tmp/default_monitoring_apps_docker.json \
--settings={_localsettings()}", pty=True)
    except Exception as e:
        logger.error(f"ERROR installing monitoring fixture: {str(e)}")


@task
def updategeoip(ctx):
    print("**************************update geoip*******************************")
    if ast.literal_eval(os.environ.get('MONITORING_ENABLED', 'False')):
        ctx.run(f"django-admin.py updategeoip --settings={_localsettings()}", pty=True)


@task
def updateadmin(ctx):
    print("***********************update admin details**************************")
    ctx.run("rm -rf /tmp/django_admin_docker.json", pty=True)
    _prepare_admin_fixture(
        os.environ.get('ADMIN_PASSWORD', 'admin'),
        os.environ.get('ADMIN_EMAIL', 'admin@example.org'))
    ctx.run(f"django-admin.py loaddata /tmp/django_admin_docker.json \
--settings={_localsettings()}", pty=True)


@task
def collectmetrics(ctx):
    print("************************collect metrics******************************")
    ctx.run(f"python -W ignore manage.py collect_metrics  \
--settings={_localsettings()} -n -t xml", pty=True)


@task
def initialized(ctx):
    print("**************************init file********************************")
    ctx.run('date > /mnt/volumes/statics/geonode_init.lock')


def _docker_host_ip():
    try:
        client = docker.from_env(version='1.24')
        ip_list = client.containers.run(
            BOOTSTRAP_IMAGE_CHEIP,
            network_mode='host').split("\n")
    except Exception:
        import traceback
        traceback.print_exc()
        ip_list = ['127.0.0.1', ]
    if len(ip_list) > 1:
        print(f"Docker daemon is running on more than one \
address {ip_list}")
        print(f"Only the first address:{ip_list[0]} will be returned!")
    else:
        print(f"Docker daemon is running at the following \
address {ip_list[0]}")
    return ip_list[0]


def _container_exposed_port(component, instname):
    port = '80'
    try:
        client = docker.from_env(version='1.24')
        ports_dict = json.dumps(
            [c.attrs['Config']['ExposedPorts'] for c in client.containers.list(
                filters={
                    'label': f'org.geonode.component={component}',
                    'status': 'running'
                }
            ) if str(instname) in c.name][0]
        )
        for key in json.loads(ports_dict):
            port = re.split('/tcp', key)[0]
    except Exception:
        import traceback
        traceback.print_exc()
    return port


def _update_db_connstring():
    user = os.getenv('GEONODE_DATABASE', 'geonode')
    pwd = os.getenv('GEONODE_DATABASE_PASSWORD', 'geonode')
    dbname = os.getenv('GEONODE_DATABASE', 'geonode')
    dbhost = os.getenv('DATABASE_HOST', 'db')
    dbport = os.getenv('DATABASE_PORT', 5432)
    connstr = f'postgis://{user}:{pwd}@{dbhost}:{dbport}/{dbname}'
    return connstr


def _update_geodb_connstring():
    geouser = os.getenv('GEONODE_GEODATABASE', 'geonode_data')
    geopwd = os.getenv('GEONODE_GEODATABASE_PASSWORD', 'geonode_data')
    geodbname = os.getenv('GEONODE_GEODATABASE', 'geonode_data')
    dbhost = os.getenv('DATABASE_HOST', 'db')
    dbport = os.getenv('DATABASE_PORT', 5432)
    geoconnstr = f'postgis://{geouser}:{geopwd}@{dbhost}:{dbport}/{geodbname}'
    return geoconnstr


def _localsettings():
    settings = os.getenv('DJANGO_SETTINGS_MODULE', 'geonode.settings')
    return settings


def _rest_api_availability(url):
    import requests
    try:
        r = requests.request('get', url, verify=False)
        r.raise_for_status()  # Raises a HTTPError if the status is 4xx, 5xxx
    except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e:
        logger.error(f"GeoServer connection error is {e}")
        return False
    except requests.exceptions.HTTPError as er:
        logger.error(f"GeoServer HTTP error is {er}")
        return False
    else:
        logger.info("GeoServer API are available!")
        return True


def _geonode_public_host_ip():
    gn_pub_hostip = os.getenv('GEONODE_LB_HOST_IP', None)
    if not gn_pub_hostip:
        gn_pub_hostip = _docker_host_ip()
    return gn_pub_hostip


def _geonode_public_port():
    gn_pub_port = os.getenv('GEONODE_LB_PORT', '')
    if not gn_pub_port:
        gn_pub_port = _container_exposed_port(
            'nginx',
            os.getenv('GEONODE_INSTANCE_NAME', 'geonode')
        )
    elif gn_pub_port in ('80', '443'):
        gn_pub_port = None
    return gn_pub_port


def _geoserver_info_provision(url):
    from django.conf import settings
    from geoserver.catalog import Catalog
    print("Setting GeoServer Admin Password...")
    cat = Catalog(
        url,
        username=settings.OGC_SERVER_DEFAULT_USER,
        password=settings.OGC_SERVER_DEFAULT_PASSWORD
    )
    headers = {
        "Content-type": "application/xml",
        "Accept": "application/xml"
    }
    data = f"""<?xml version="1.0" encoding="UTF-8"?>
<userPassword>
    <newPassword>{(os.getenv('GEOSERVER_ADMIN_PASSWORD', 'geoserver'))}</newPassword>
</userPassword>"""

    response = cat.http_request(f"{cat.service_url}/security/self/password", method="PUT", data=data, headers=headers)
    print(f"Response Code: {response.status_code}")
    if response.status_code == 200:
        print("GeoServer admin password updated SUCCESSFULLY!")
    else:
        logger.warning(f"WARNING: GeoServer admin password *NOT* updated: code [{response.status_code}]")


def _prepare_oauth_fixture():
    upurl = urlparse(os.environ['SITEURL'])
    net_scheme = upurl.scheme
    pub_ip = _geonode_public_host_ip()
    print(f"Public Hostname or IP is {pub_ip}")
    pub_port = _geonode_public_port()
    print(f"Public PORT is {pub_port}")
    default_fixture = [
        {
            "model": "oauth2_provider.application",
            "pk": 1001,
            "fields": {
                "skip_authorization": True,
                "created": "2018-05-31T10:00:31.661Z",
                "updated": "2018-05-31T11:30:31.245Z",
                "algorithm": "RS256",
                "redirect_uris": f"{net_scheme}://{pub_ip}:{pub_port}/geoserver/index.html"
                                 if pub_port else f"{net_scheme}://{pub_ip}/geoserver/index.html",
                "name": "GeoServer",
                "authorization_grant_type": "authorization-code",
                "client_type": "confidential",
                "client_id": str(os.environ['OAUTH2_CLIENT_ID']),
                "client_secret": str(os.environ['OAUTH2_CLIENT_SECRET']),
                "user": [
                    "admin"
                ]
            }
        }
    ]
    with open('/tmp/default_oauth_apps_docker.json', 'w') as fixturefile:
        json.dump(default_fixture, fixturefile)


def _prepare_site_fixture():
    upurl = urlparse(os.environ['SITEURL'])
    default_fixture = [
        {
            "model": "sites.site",
            "pk": 1,
            "fields": {
                "domain": str(upurl.hostname),
                "name": str(upurl.hostname)
            }
        }
    ]
    with open('/tmp/default_site.json', 'w') as fixturefile:
        json.dump(default_fixture, fixturefile)


def _prepare_monitoring_fixture():
    # upurl = urlparse(os.environ['SITEURL'])
    # net_scheme = upurl.scheme
    # net_loc = upurl.netloc
    pub_ip = _geonode_public_host_ip()
    print(f"Public Hostname or IP is {pub_ip}")
    pub_port = _geonode_public_port()
    print(f"Public PORT is {pub_port}")
    try:
        geonode_ip = socket.gethostbyname('geonode')
    except Exception:
        geonode_ip = pub_ip
    try:
        geoserver_ip = socket.gethostbyname('geoserver')
    except Exception:
        geoserver_ip = pub_ip
    d = '1970-01-01 00:00:00'
    default_fixture = [
        {
            "fields": {
                "active": True,
                "ip": str(geonode_ip),
                "name": str(os.environ['MONITORING_HOST_NAME'])
            },
            "model": "monitoring.host",
            "pk": 1
        },
        {
            "fields": {
                "active": True,
                "ip": str(geoserver_ip),
                "name": "geoserver"
            },
            "model": "monitoring.host",
            "pk": 2
        },
        {
            "fields": {
                "name": str(os.environ['MONITORING_SERVICE_NAME']),
                "url": str(os.environ['SITEURL']),
                "notes": "",
                "last_check": d,
                "active": True,
                "host": 1,
                "check_interval": "00:01:00",
                "service_type": 1
            },
            "model": "monitoring.service",
            "pk": 1
        },
        {
            "fields": {
                "name": "geoserver-hostgeonode",
                "url": str(os.environ['SITEURL']),
                "notes": "",
                "last_check": d,
                "active": True,
                "host": 1,
                "check_interval": "00:01:00",
                "service_type": 3
            },
            "model": "monitoring.service",
            "pk": 2
        },
        {
            "fields": {
                "name": "geoserver-hostgeoserver",
                "url": str(os.environ['GEOSERVER_PUBLIC_LOCATION']),
                "notes": "",
                "last_check": d,
                "active": True,
                "host": 2,
                "check_interval": "00:01:00",
                "service_type": 4
            },
            "model": "monitoring.service",
            "pk": 3
        },
        {
            "fields": {
                "name": "default-geoserver",
                "url": "http://geoserver:8080/geoserver/",
                "notes": "",
                "last_check": d,
                "active": True,
                "host": 2,
                "check_interval": "00:01:00",
                "service_type": 2
            },
            "model": "monitoring.service",
            "pk": 4
        }
    ]
    with open('/tmp/default_monitoring_apps_docker.json', 'w') as fixturefile:
        json.dump(default_fixture, fixturefile)


def _prepare_admin_fixture(admin_password, admin_email):
    from django.contrib.auth.hashers import make_password
    d = datetime.datetime.now()
    mdext_date = f"{d.isoformat()[:23]}Z"
    default_fixture = [
        {
            "fields": {
                "date_joined": mdext_date,
                "email": admin_email,
                "first_name": "",
                "groups": [],
                "is_active": True,
                "is_staff": True,
                "is_superuser": True,
                "last_login": mdext_date,
                "last_name": "",
                "password": make_password(admin_password),
                "user_permissions": [],
                "username": "admin"
            },
            "model": "people.Profile",
            "pk": 1000
        }
    ]
    with open('/tmp/django_admin_docker.json', 'w') as fixturefile:
        json.dump(default_fixture, fixturefile)