Python 3

README.md

@@ -1,5 +1,4 @@
-What is CoG?
-============
+# What is CoG?
 
 CoG is a multi-institutional project that seeks to examine, within the Earth sciences, 
 the organizational characteristics of community software projects and to create software infrastructure 
@@ -12,24 +11,25 @@ and that exposing those characteristics and linking them is a way to make a comp
 The CoG project created the CoG environment - this website - as a way of promoting good governance across numerous projects 
 and organizations in the Earth system sciences.  Its structure encourages information sharing among projects, transparency and trust.
 
-For more information, visit: http://earthsystemcog.org/
+For more information, visit: http://esgf.github.io/COG
 
-Manual CoG update
------------------
+## Recent Updates
 
-```
-cd /usr/local/cog/cog_install
-git fetch
-git checkout v3.15.4 # or master, devel, etc.
-/etc/cog-wsgi-8889/apachectl restart  # for ESGF > v4.0.0 
-# previous versions restart node: esg-node restart 
-```
+### v4.0.3
 
-Latest release: v3.15.4
------------------------
+* Drop processing of use of next=url in add user query string.
 
- - Add link to clear data cart to appear on pages that import the "middle navbar"
+### v4.0.2
+* Bug fix for Show Citation (missing data from API causes all information not to display)
 
-Previous updates
+## Python3 Update
+
+This is v4.0.3 of CoG that supports Python3 and Django 2.2.24.  Given that this Django 2.2 version has run in LLNL production for over eight months now, it is time that we release this version of COG as the master branch.  We strongly encourage all sites to upgrade their deployments to this version.
+
+The last v3.x version of CoG running with Python2 is preserved for legacy.  This version is still needed to create the cogdb database (see below) for a standard legacy ESGF deployment using the Ansible playbooks (although a Postgres >9.5 supported version may be forthcoming....).
+
+## Updating CoG
+
+Use the Ansible playbook from the following branch: `python3_cog` - See https://esgf.github.io/esgf-ansible for instructions on how to run the playbooks. 
+The installation must be performed on a server that already has the cogdb deployed to PostgreSQL.  In other words, in order to install CoG v4 from scratch, you must first run the latest Ansible playbook for ESGF, which will deploy CoG v3.15 (Python2) and create the cogdb database.  Then, run the  `python3_cog` branch playbook to setup CoG.  Note that this installation will create a distinct Python3 conda environment for which to run the CoG service in mod_wsgi_express `py3_cog`.  The previous `cog` environment will still be present, but is no longer used (and could be safely deleted).
 
- - Updates to templates for help information in preparation for the retirement of the CU CoG site.

apache/wsgi.py

@@ -28,18 +28,18 @@
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings")
 
 # print debugging information
-print 'Using Python version: %s' % sys.version
-print 'Using Python path: %s' % sys.path
-print 'PYTHONPATH=%s' % os.environ.get('PYTHONPATH', None)
-print 'LD_LIBRARY_PATH=%s' % os.environ.get('LD_LIBRARY_PATH', None)
-print 'SSL_CERT_DIR=%s' % os.environ.get('SSL_CERT_DIR', None)
-print 'SSL_CERT_FILE=%s' % os.environ.get('SSL_CERT_FILE', None)
+print('Using Python version: %s' % sys.version)
+print('Using Python path: %s' % sys.path)
+print('PYTHONPATH=%s' % os.environ.get('PYTHONPATH', None))
+print('LD_LIBRARY_PATH=%s' % os.environ.get('LD_LIBRARY_PATH', None))
+print('SSL_CERT_DIR=%s' % os.environ.get('SSL_CERT_DIR', None))
+print('SSL_CERT_FILE=%s' % os.environ.get('SSL_CERT_FILE', None))
 
 try:
     application = get_wsgi_application()
-    print 'WSGI without exception'
+    print('WSGI without exception')
 except Exception:
-    print 'handling WSGI exception'
+    print('handling WSGI exception')
     # Error loading applications
     if 'mod_wsgi' in sys.modules:
         traceback.print_exc()

cog/config/search/__init__.py

@@ -1 +1 @@
-from config_project_search import SearchConfigParser
+from .config_project_search import SearchConfigParser

cog/config/search/config_project_search.py

@@ -1,4 +1,4 @@
-import sys, os, ConfigParser
+import sys, os, configparser
 from django.conf import settings
 from cog.models import Project, SearchGroup, SearchFacet
 from cog.utils import str2bool
@@ -15,7 +15,7 @@ def __init__(self, project):
     def _getConfigParser(self):
 
         # read project configuration
-        configParser = ConfigParser.RawConfigParser()
+        configParser = configparser.RawConfigParser()
         # must set following line explicitly to preserve the case of configuration keys
         configParser.optionxform = str 
 
@@ -25,7 +25,7 @@ def _getConfigParser(self):
     def write(self):
         '''Writes the project search configuration to the file $COG_CONFIG_DIR/projects/<project_short_name>/search.cfg'''
 
-        print 'Writing search configuration for project=%s' % self.project.short_name
+        print('Writing search configuration for project=%s' % self.project.short_name)
 
         # load project search profile
         project = Project.objects.get(short_name=self.project.short_name)
@@ -66,23 +66,23 @@ def write(self):
     def read(self):
         '''Reads the project search configuration from the file $COG_CONFIG_DIR/projects/<project_short_name>/search.cfg'''
 
-        print 'Reading search configuration for project=%s' % self.project.short_name
+        print('Reading search configuration for project=%s' % self.project.short_name)
 
         # load project search profile
         project = Project.objects.get(short_name=self.project.short_name)
         search_profile = project.searchprofile
 
         # remove existing groups of facets
         for group in search_profile.groups.all():
-            print 'Deleting search group=%s' % group
+            print('Deleting search group=%s' % group)
             group.delete()
 
         # read project configuration
         projConfig = self._getConfigParser()
         try:
             projConfig.read( self.config_file_path )
         except Exception as e:
-            print "Configuration file %s not found" % self.config_file_path
+            print("Configuration file %s not found" % self.config_file_path)
             raise e
 
         # loop over configuration sections
@@ -110,7 +110,7 @@ def read(self):
 
                 for option in projConfig.options(section):
                     value = projConfig.get(section, option)
-                    print section, option, value
+                    print(section, option, value)
                     parts = value.split("|")
                     facet_order = int(option)
                     facet_key = parts[0]

cog/db_migrations/django_openid_auth/0001_initial.py

@@ -39,7 +39,7 @@ class Migration(migrations.Migration):
                 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                 ('claimed_id', models.TextField(unique=True, max_length=2047)),
                 ('display_id', models.TextField(max_length=2047)),
-                ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL)),
+                ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)),
             ],
         ),
     ]

cog/forms/__init__.py

@@ -1,10 +1,10 @@
-from forms_image import *
-from forms_account import *
-from forms_bookmarks import *
-from forms_governance import *
-from forms_others import *
-from forms_post import *
-from forms_project import *
-from forms_aboutus import *
-from forms_search import *
-from forms_access_control import *
+from .forms_image import *
+from .forms_account import *
+from .forms_bookmarks import *
+from .forms_governance import *
+from .forms_others import *
+from .forms_post import *
+from .forms_project import *
+from .forms_aboutus import *
+from .forms_search import *
+from .forms_access_control import *

cog/forms/forms_account.py

@@ -286,7 +286,7 @@ def validate_username(form, user_id):
 
                         # once the openid is validated, choose the closest possible username
                         _username = createUsername(username)
-                        print 'Created username=%s from=%s' % (_username, username)
+                        print('Created username=%s from=%s' % (_username, username))
                         cleaned_data['username'] = _username  # override form data
             else:
                 # django will automatically check that the username is unique in the CoG database
@@ -304,6 +304,6 @@ def validate_field(form, field_name, field_value):
 def validate_ascii(form, field_name, field_value):
     if field_value:
         try:
-            field_value.decode('ascii')
+            field_value.encode('ascii')
         except (UnicodeDecodeError, UnicodeEncodeError):
             form._errors[field_name] = form.error_class(["'%s' contains invalid characters." % field_name])

cog/forms/forms_image.py

@@ -23,6 +23,6 @@ def clean(self):
                 self._errors["image"] = self.error_class(["Image size exceeds the maximum allowed."])
         except OSError as e:
             # image not existing on disk
-            print e
+            print(e)
 
         return self.cleaned_data

cog/forms/forms_others.py

@@ -126,10 +126,10 @@ def clean(self):
 
             file_ext = str(os.path.splitext(thefile.name)[1])
             mime_type = magic.from_buffer(thefile.read(1024), mime=True)
-            print "Validating file extension=%s, mime type=%s" % (file_ext, mime_type)
+            print("Validating file extension=%s, mime type=%s" % (file_ext, mime_type))
             if not file_ext:
                 self._errors["file"] = self.error_class(["File name must have an extension."])
-            elif file_ext.lower() not in VALID_MIME_TYPES.keys():
+            elif file_ext.lower() not in list(VALID_MIME_TYPES.keys()):
                 self._errors["file"] = self.error_class(["File extension %s is not supported." % file_ext])
             elif mime_type not in VALID_MIME_TYPES[file_ext.lower()]:
                 self._errors["file"] = self.error_class(["File extension %s does not match its valid mime type." %

cog/forms/forms_post.py

@@ -137,7 +137,7 @@ def clean(self):
         # validate "topic"
         # cannot set both 'topic' and 'newtopic'
         if topic and newtopic:
-            errmsg = u"Please either choose an existing topic OR create a new one"
+            errmsg = "Please either choose an existing topic OR create a new one"
             self._errors["topic"] = self.error_class([errmsg])
             del cleaned_data["topic"]
             del cleaned_data["newtopic"]
@@ -152,6 +152,13 @@ def clean(self):
 
             cleaned_data["topic"] = topic
 
+        # check if the 'label' field is set for hyperlink and page posts
+        # if 'label' is blank, then fill the field with the first 28 characters of 'title'        
+        if type == Post.TYPE_PAGE or type == Post.TYPE_HYPERLINK:
+            label = cleaned_data["label"]
+            if label is None:
+                cleaned_data["label"] = cleaned_data["title"][:28]
+
         # prevent XSS on fields 'title', 'label', 'newtopic'
         for key in ["title", "label", "newtopic"]:
             if key in cleaned_data:

cog/forms/forms_project.py

@@ -19,10 +19,10 @@ class ProjectForm(ModelForm):
     # define the widget for parent/peer selection so we can set the styling. The class is set to .selectfilter and its
     # styles are controlled in cogstyle.css
 
-    parents = forms.ModelMultipleChoiceField("parents", required=False,
+    parents = forms.ModelMultipleChoiceField(queryset=Project.objects.all(), required=False,
                                              widget=forms.SelectMultiple(attrs={'size': '20',
                                                                                 'class': 'selectprojects'}))
-    peers = forms.ModelMultipleChoiceField("peers", required=False,
+    peers = forms.ModelMultipleChoiceField(queryset=Project.objects.all(), required=False,
                                            widget=forms.SelectMultiple(attrs={'size': '20',
                                                                               'class': 'selectprojects'}))
     # filtering of what is see in the form is done down below. 
@@ -99,7 +99,7 @@ def clean_long_name(self):
 
         # check for non-ascii characters
         try:
-            long_name.decode('ascii')
+            long_name.encode('ascii')
         except (UnicodeDecodeError, UnicodeEncodeError):
             raise forms.ValidationError("Project long name contains invalid non-ASCII characters")
         return long_name
@@ -156,7 +156,7 @@ def clean(self):
         features = self.cleaned_data.get('software_features')
         if not hasText(features):
             self._errors["software_features"] = self.error_class(["'SoftwareFeatures' must not be empty."])
-            print 'error'
+            print('error')
         return self.cleaned_data
 
 
@@ -174,7 +174,7 @@ class ProjectTagForm(ModelForm):
     # form constructor in views_project.py
 
     # field['tags'] is the list of preexisting tags
-    tags = forms.ModelMultipleChoiceField("tags", required=False,
+    tags = forms.ModelMultipleChoiceField(queryset=Project.objects.all(), required=False,
                                           widget=forms.SelectMultiple(attrs={'size': '7'}))
 
     # override __init__ method to change the queryset for 'tags'

cog/forms/forms_search.py

@@ -15,7 +15,12 @@ def clean_url(self):
         return clean_url_field(self, 'url')
 
     def clean_constraints(self):
-        return clean_url_field(self, 'constraints')
+        # if no constraints are found, then return an empty string
+        data = self.cleaned_data['constraints']
+        if data is None:
+            return ''
+        else:
+            return clean_url_field(self, 'constraints')
 
 
 class SearchFacetForm(ModelForm):

cog/forms/forms_utils.py

@@ -26,7 +26,7 @@ def validate_image(form, field_name):
         # validate image header
         try:
             image_type = imghdr.what(image)
-            print 'Validating image header: detected image type=%s' % image_type
+            print('Validating image header: detected image type=%s' % image_type)
             if image_type is None:
                 form._errors[field_name] = form.error_class(["Invalid image type: %s" % image_type])
         except Exception as e:

cog/installation/config.py

@@ -23,15 +23,15 @@
 
 '''
 
-import ConfigParser
-import StringIO
+import configparser
+import io
 import collections
 import logging
 import os
 import time
 from django.utils.crypto import get_random_string
 
-from constants import (SECTION_DEFAULT, COG_SECTION_DEFAULT, SECTION_ESGF, SECTION_EMAIL,
+from .constants import (SECTION_DEFAULT, COG_SECTION_DEFAULT, SECTION_ESGF, SECTION_EMAIL,
                        ESGF_PROPERTIES_FILE, ESGF_PASSWORD_FILE,
                        IDP_WHITELIST, KNOWN_PROVIDERS, PEER_NODES,
                        DEFAULT_PROJECT_SHORT_NAME)
@@ -59,7 +59,7 @@ def _readCogConfig(self):
         '''Method that reads an existing COG configuration file, or create a new one if not existing.'''
 
         # initialize COG configuration file
-        self.cogConfig = ConfigParser.ConfigParser(allow_no_value=True,
+        self.cogConfig = configparser.ConfigParser(allow_no_value=True, 
                                                    dict_type=collections.OrderedDict)
         # must set following line explicitly to preserve the case of configuration keys
         self.cogConfig.optionxform = str
@@ -79,7 +79,7 @@ def _readCogConfig(self):
                 logging.info("Configuration file: %s not found, will create new one" % CONFIGFILEPATH )
 
         except Exception as e:
-            print e
+            print(e)
             logging.error("Error reading configuration file: %s" % CONFIGFILEPATH)
             logging.error(e)
 
@@ -88,7 +88,7 @@ def _readEsgfConfig(self):
         '''Method that reads local parameters from ESGF configuration file esgf.properties.'''
 
         # read ESGF configuration file ($esg_config_dir/esgf.properties), if available
-        self.esgfConfig = ConfigParser.ConfigParser()
+        self.esgfConfig = configparser.ConfigParser()
         try:
             self.esgfConfig.read(ESGF_PROPERTIES_FILE)
         except IOError:
@@ -103,7 +103,7 @@ def _readEsgfConfig(self):
                 with open(ESGF_PROPERTIES_FILE, 'r') as f:
                     # transform Java properties file into python configuration file: must prepend a section
                     config_string = '[%s]\n' % SECTION_DEFAULT + f.read()
-                    config_file = StringIO.StringIO(config_string)
+                    config_file = io.StringIO(config_string)
                     self.esgfConfig.readfp(config_file)
                 logging.info("Read ESGF configuration parameters from file: %s" % ESGF_PROPERTIES_FILE)
 
@@ -168,7 +168,7 @@ def _setupConfig(self):
         # default search service URL, before any project customization
         self._safeSet('DEFAULT_SEARCH_URL','http://%s/esg-search/search/' % hostName)
         # interval between updates of user's projects, during user session
-        self._safeSet('MY_PROJECTS_REFRESH_SECONDS', 3600)
+        self._safeSet('MY_PROJECTS_REFRESH_SECONDS', '3600')
         # optional number of days after which password expire
         self._safeSet('PWD_EXPIRATION_DAYS','0')
         # optional top-level URL to redirect user registration (no trailing '/')
@@ -188,13 +188,13 @@ def _setupConfig(self):
         # PEER_NODES = /esg/config/esgf_cogs.xml
         self._safeSet('PEER_NODES', PEER_NODES)
         # option to send SESSION and CSRF cookies via SSL only - requires full SSL-encrypted site
-        self._safeSet('PRODUCTION_SERVER', True)
+        self._safeSet('PRODUCTION_SERVER', 'True')
         # ESGF software stack version
         esgfVersion = self._safeGet("version", default=None)
         if esgfVersion:
             self._safeSet('ESGF_VERSION', esgfVersion, override=True)
         # option to disable CAPTCHA for creating account in automatic testing
-        self._safeSet('USE_CAPTCHA', True)
+        self._safeSet('USE_CAPTCHA', 'True')
 
 
         #[ESGF]

cog/installation/install.py

@@ -168,7 +168,7 @@ def _createObjects(self):
             except ObjectDoesNotExist:
                 site = Site.objects.create(name=idpHostname, domain=idpHostname)
                 idpPeerSite = PeerSite.objects.create(site=site, enabled=True)
-            print '\tCreated IdP Peer site: %s with enabled=%s' % (idpPeerSite, idpPeerSite.enabled)
+            print('\tCreated IdP Peer site: %s with enabled=%s' % (idpPeerSite, idpPeerSite.enabled))
 
 
     def _getRootAdminPassword(self):