Upgrade to python3

.circleci/config.yml

@@ -2,7 +2,7 @@ version: 2
 jobs:
   build:
     docker:
-      - image: circleci/python:2.7
+      - image: circleci/python:3.6
       - image: circleci/postgres:9.6.2
         environment:
           POSTGRES_USER: pred_user
@@ -13,8 +13,8 @@ jobs:
           key: deps9-{{ .Branch }}-{{ checksum "requirements.txt" }}-{{ checksum "devRequirements.txt" }}-{{ checksum "portal/package.json" }}
       - run:
           command: |
-            virtualenv env
-            source env/bin/activate
+            python -m venv venv
+            source venv/bin/activate
             pip install -r requirements.txt
             pip install -r devRequirements.txt
             curl -sL https://deb.nodesource.com/setup_8.x | sudo  sudo -E bash -
@@ -29,7 +29,7 @@ jobs:
             - "portal/node_modules"
       - run:
           command: |
-            source env/bin/activate
+            source venv/bin/activate
             TF_TEST_WITH_POSTGRES=true nosetests
       - run:
           command: |

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:2.7.11
+FROM python:3.6
 EXPOSE 80
 ENV MYDIR /tfdnapredictions
 RUN curl -sL https://deb.nodesource.com/setup_8.x | bash -

portal/package.json

@@ -6,7 +6,7 @@
   "scripts": {
     "webpack": "webpack --mode development --progress --colors --watch",
     "build": "webpack -p",
-    "test": "mocha --compilers js:babel-core/register src/tests"
+    "test": "mocha --require babel-core/register src/tests"
   },
   "author": "",
   "license": "MIT",

pred/config.py

@@ -1,4 +1,4 @@
-from yaml import load
+import yaml
 import os
 
 DEFAULT_DB_HOST = "localhost"
@@ -39,7 +39,7 @@ def get_data_source_type(data_type):
 def parse_config(filename):
     config = None
     with open(filename) as data_file:
-        data = load(data_file)
+        data = yaml.safe_load(data_file)
         return parse_config_from_dict(data)
 
 

pred/load/download.py

@@ -5,7 +5,7 @@
 import re
 from ftplib import FTP
 import gzip
-from urlparse import urlparse
+from urllib.parse import urlparse
 import requests
 import subprocess
 import yaml

pred/webserver/sequencelist.py

@@ -6,7 +6,7 @@
 from pred.webserver.errors import ClientException, ErrorType, raise_on_too_big_uploaded_data
 from pred.queries.dbutil import update_database, read_database
 from Bio import SeqIO
-from StringIO import StringIO
+from io import StringIO
 
 class SequenceList(object):
     """

production/Dockerfile

@@ -1,10 +1,18 @@
 ARG BASE_IMAGE=dukegcb/imads
 FROM ${BASE_IMAGE}
 LABEL maintainer="john.bradley@duke.edu"
-# apache2
-RUN apt-get update \
-  && apt-get install -y apache2 apache2-utils libapache2-mod-wsgi\
-  && apt-get clean
+
+RUN apt-get update && apt-get install -y \
+  apache2 \
+  apache2-dev
+
+RUN pip install mod_wsgi==4.6.4
+
+# Since mod_wsgi is installed in python libs directory, this command
+# links it into apache for easier usage.
+RUN mod_wsgi-express install-module
+
+RUN ["pip", "install", "mod_wsgi"]
 
 # Configure Apache
 ADD imads.conf /etc/apache2/sites-available/imads.conf

production/imads.conf

@@ -17,6 +17,8 @@ Alias /static/ /tfdnapredictions/static/
 </Directory>
 
 # WSGI
+LoadModule wsgi_module "/usr/lib/apache2/modules/mod_wsgi-py36.cpython-36m-x86_64-linux-gnu.so"
+WSGIPythonHome "/usr/local"
 WSGIScriptAlias / /tfdnapredictions/wsgi.py
 WSGIPythonPath /tfdnapredictions
 # Pass authorization headers to python app, such as api tokens

requirements.txt

@@ -1,8 +1,8 @@
-SQLAlchemy==1.0.11
-Flask==0.10.1
+SQLAlchemy==1.3.5
+Flask==0.12.3
 psycopg2==2.7.3.2
-PyYAML==3.11
+PyYAML==5.1
 requests==2.20.0
-Jinja2==2.8
+Jinja2==2.10.1
 twobitreader==3.1.4
 biopython==1.66

tests/test_integration.py

@@ -1,6 +1,6 @@
 import unittest
 import os
-from StringIO import StringIO
+from io import StringIO
 from pred.config import parse_config_from_dict, DataType
 from load import run_sql_command
 from pred.load import loaddatabase

tests/test_too_big_upload.py

@@ -65,7 +65,7 @@ def test_gene_list_too_big(self):
         self.assertEqual(400, cm.exception.status_code)
 
     def test_gene_list_good_size(self):
-        save_custom_file(db=FakeDB(), user_info=None, type=GENE_LIST_TYPE, content="WASH7P\ELK1\NETS2")
+        save_custom_file(db=FakeDB(), user_info=None, type=GENE_LIST_TYPE, content="WASH7P\nELK1\nNETS2")
 
 
 class TestTooBigCustomSequence(TestCase):

tests/test_webserver.py

@@ -0,0 +1,12 @@
+from unittest import TestCase
+from webserver import base64_string_encode, base64_string_decode
+
+
+class TestBase64Funcs(TestCase):
+    def test_base64_string_encode(self):
+        result = base64_string_encode("abcdefg")
+        self.assertEqual(result, "YWJjZGVmZw==")
+
+    def test_base64_string_decode(self):
+        result = base64_string_decode("VGVzdGluZzEyMw==")
+        self.assertEqual(result, "Testing123")

util/jobsimulator.py

@@ -5,12 +5,12 @@
 from __future__ import print_function
 import sys
 import requests
-import base64
 from Bio import SeqIO
 from StringIO import StringIO
 import random
 from multiprocessing import Pool
 import time
+from pred.webserver import base64_string_decode, base64_string_encode
 
 BASE_URL = "http://localhost:5000/api/v1"
 
@@ -50,15 +50,15 @@ def get_sequence(job):
     r = requests.get(url)
     r.raise_for_status()
     data = r.json()['data']
-    return base64.b64decode(data)
+    return base64_string_decode(data)
 
 
 def save_custom_predictions(job, bed_file_data):
     url = make_url("custom_predictions")
     r = requests.post(url, json={
         'job_id': job['id'],
         'model_name': job['model_name'],
-        'bed_data': base64.b64encode(bed_file_data),
+        'bed_data': base64_string_encode(bed_file_data),
     })
     r.raise_for_status()
 
@@ -125,10 +125,11 @@ def error_next_job():
         job = jobs[0]
     mark_job_error(job, "Processing failed with error DAN-8.")
 
+
 def create_job():
     # upload sequence
     url = make_url("sequences")
-    data = {'data': base64.b64encode('>myseq\nAACCGGTT'), 'title':'MySeq'}
+    data = {'data': base64_string_encode('>myseq\nAACCGGTT'), 'title':'MySeq'}
     r = requests.post(url, json=data)
     r.raise_for_status()
     sequence_id = r.json()['id']

webserver.py

@@ -30,6 +30,24 @@ def log_info(message):
     print(message)
 
 
+def base64_string_decode(data):
+    """
+    Decodes a base64 encoded string into a string
+    :param data: str: string to decode
+    :return: str
+    """
+    return base64.b64decode(data).decode('utf-8')
+
+
+def base64_string_encode(data):
+    """
+    Encodes a string into it's base64 string representation
+    :param data: str: string to encode
+    :return: str
+    """
+    return base64.b64encode(data.encode('utf-8')).decode('utf-8')
+
+
 def get_db():
     db = getattr(g, '_database', None)
     if db is None:
@@ -160,15 +178,15 @@ def get_custom_sequences_data(sequence_id):
     seq.load(get_db())
     return make_json_response({
             "id": seq.seq_uuid,
-            "data": base64.b64encode(seq.content),
+            "data": base64_string_encode(seq.content),
             "created": seq.created
         })
 
 
 @app.route('/api/v1/sequences', methods=['POST'])
 def post_custom_sequences():
     (data, title) = get_required_json_props(request, ["data", "title"])
-    decoded_data = base64.b64decode(data)
+    decoded_data = base64_string_decode(data)
     seq_uuid = SequenceList.create_with_content_and_title(get_db(), decoded_data, title)
     return make_ok_json_response({'id': seq_uuid})
 
@@ -262,7 +280,7 @@ def post_custom_result():
     required_prop_names = ["job_id", "model_name"]
     (job_id, model_name) = get_required_json_props(request, required_prop_names)
     bed_data = request.get_json().get('bed_data')
-    decoded_bed_data = base64.b64decode(bed_data)
+    decoded_bed_data = base64_string_decode(bed_data)
     result_uuid = CustomResultData.new_uuid()
     result_data = CustomResultData(get_db(), result_uuid, job_id, model_name, decoded_bed_data)
     result_data.save()