Merge ESGF/COG devel with Python 3 support

apache/wsgi.py

@@ -28,18 +28,18 @@
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings")
 
 # print debugging information
-print 'Using Python version: %s' % sys.version
-print 'Using Python path: %s' % sys.path
-print 'PYTHONPATH=%s' % os.environ.get('PYTHONPATH', None)
-print 'LD_LIBRARY_PATH=%s' % os.environ.get('LD_LIBRARY_PATH', None)
-print 'SSL_CERT_DIR=%s' % os.environ.get('SSL_CERT_DIR', None)
-print 'SSL_CERT_FILE=%s' % os.environ.get('SSL_CERT_FILE', None)
+print('Using Python version: %s' % sys.version)
+print('Using Python path: %s' % sys.path)
+print('PYTHONPATH=%s' % os.environ.get('PYTHONPATH', None))
+print('LD_LIBRARY_PATH=%s' % os.environ.get('LD_LIBRARY_PATH', None))
+print('SSL_CERT_DIR=%s' % os.environ.get('SSL_CERT_DIR', None))
+print('SSL_CERT_FILE=%s' % os.environ.get('SSL_CERT_FILE', None))
 
 try:
     application = get_wsgi_application()
-    print 'WSGI without exception'
+    print('WSGI without exception')
 except Exception:
-    print 'handling WSGI exception'
+    print('handling WSGI exception')
     # Error loading applications
     if 'mod_wsgi' in sys.modules:
         traceback.print_exc()

cog/backends/esgf.py

@@ -0,0 +1,204 @@
+from social_core.backends.oauth import BaseOAuth2
+from social_core.backends.open_id import OpenIdAuth, OPENID_ID_FIELD
+from social_core.exceptions import AuthMissingParameter
+from six.moves.urllib_parse import urlencode, unquote
+
+import os
+import logging
+from base64 import b64encode
+from OpenSSL import crypto
+from urllib.parse import urlparse
+
+from openid.yadis.services import getServiceEndpoints
+from openid.yadis.discover import DiscoveryFailure
+
+class ESGFOAuth2(BaseOAuth2):
+    name = 'esgf'
+    REDIRECT_STATE = True
+    ACCESS_TOKEN_METHOD = 'POST'
+    EXTRA_DATA = [
+        ('access_token', 'access_token', True),
+        ('expires_in', 'expires_in', True),
+        ('refresh_token', 'refresh_token', True),
+        ('openid', 'openid', True)
+    ]
+
+    @property
+    def AUTHORIZATION_URL(self):
+        return self._authorization_url
+
+    @property
+    def ACCESS_TOKEN_URL(self):
+        return self._access_token_url
+
+    @property
+    def CERTIFICATE_URL(self):
+        return self._certificate_url
+
+    @property
+    def DEFAULT_SCOPE(self):
+        return self._default_scope
+
+
+    def __init__(self, strategy=None, redirect_uri=None):
+
+        self._authorization_url = None
+        self._access_token_url = None
+        self._certificate_url = None
+        self._default_scope = None
+
+        super(ESGFOAuth2, self).__init__(strategy, redirect_uri)
+
+        # Get openid_identifier added to the session by PSA. Requires
+        # SOCIAL_AUTH_FIELDS_STORED_IN_SESSION = ['openid_identifier',]
+        # set in settings.py
+        if strategy:
+            openid = self.strategy.session_get(OPENID_ID_FIELD, None)
+            self.set_urls(openid)
+
+
+    # get XRDS and extract authorize, access token, and certificate service URLs
+    def set_urls(self, openid):
+        try:
+            uri, endpoints = getServiceEndpoints(openid)
+        except DiscoveryFailure:
+            return
+        for e in endpoints:
+            if e.matchTypes(['urn:esg:security:oauth:endpoint:authorize']):
+                self._authorization_url = e.uri
+            elif e.matchTypes(['urn:esg:security:oauth:endpoint:access']):
+                self._access_token_url = e.uri
+            elif e.matchTypes(['urn:esg:security:oauth:endpoint:resource']):
+                self._certificate_url = e.uri
+                self._default_scope = [e.uri]
+
+
+
+    def get_certificate(self, access_token):
+        """ Generate a new key pair """
+        key_pair = crypto.PKey()
+        key_pair.generate_key(crypto.TYPE_RSA, 2048)
+        self.private_key = crypto.dump_privatekey(crypto.FILETYPE_PEM, key_pair).decode('utf-8')
+
+        """ Generate a certificate request """
+        cert_request = crypto.X509Req()
+        cert_request.set_pubkey(key_pair)
+        cert_request.sign(key_pair, 'md5')
+        cert_request = crypto.dump_certificate_request(crypto.FILETYPE_ASN1, cert_request)
+
+        headers = {'Authorization': 'Bearer {}'.format(access_token)}
+        url = self.CERTIFICATE_URL
+
+        request_args = {'headers': headers,
+                        'data': {'certificate_request': b64encode(cert_request)}
+        }
+        try:
+            response = self.request(url, method='POST', **request_args)
+            response.raise_for_status()
+        except Exception as e:
+            logging.error(e)
+        self.cert = response.text
+        cert = crypto.load_certificate(crypto.FILETYPE_PEM, response.text)
+        self.subject = cert.get_subject()
+        return (self.private_key, self.cert, self.subject.commonName)
+
+
+    # TO DO when OIDC is deployed on ESGF IdP nodes
+    # extract user info from id_token (OpenID Connect)
+#    def user_data(self, access_token, *args, **kwargs):
+#        print 'user_data'
+#        response = kwargs.get('response')
+#        id_token = response.get('id_token')
+#        try:
+#            decoded_id_token = jwt_decode(id_token, verify=False)
+#        except (DecodeError, ExpiredSignature) as de:
+#            raise AuthTokenError(self, de)
+#        for key in decoded_id_token:
+#            print '%s:%s' % (key, decoded_id_token[key])
+#
+#        return {'uid': decoded_id_token.get('sub'),
+#                'username': decoded_id_token.get('preferred_username'),
+#                'name': decoded_id_token.get('name'),
+#                'email': decoded_id_token.get('email')
+#                }
+
+
+    # return values that will be stored in the database as:
+    # social_auth_usersocialauth.extra_data['openid'],
+    # auth_user.username
+    def get_user_details(self, response):
+        access_token = response.get('access_token')
+        pkey, cert, openid = self.get_certificate(access_token)
+        username = os.path.basename(urlparse(openid).path)
+        return {'openid': openid,
+                'username': username,
+                }
+
+
+    # PSA compares returned value with social_auth_usersocialauth.uid (provider=='esgf')
+    # if it's new, PSA creates a new user
+    def get_user_id(self, details, response):
+        return details['openid']
+
+
+class ESGFOpenId(OpenIdAuth):
+    name = 'esgf-openid'
+
+    # Extend original openid.openid_url() to a case where openid_identifier is set in the session only
+    def openid_url(self):
+        """Return service provider URL.
+        This base class is generic accepting a POST parameter that specifies
+        provider URL."""
+        if self.URL:
+            return self.URL
+        elif OPENID_ID_FIELD in self.data:
+            return self.data[OPENID_ID_FIELD]
+        elif self.strategy.session_get(OPENID_ID_FIELD, None):
+            return self.strategy.session_get(OPENID_ID_FIELD, None)
+        else:
+            raise AuthMissingParameter(self, OPENID_ID_FIELD)
+
+
+    def get_user_details(self, response):
+        username = os.path.basename(urlparse(response.identity_url).path)
+        return {'openid': response.identity_url,
+                'username': username,
+                }
+
+
+def associate_by_openid(backend, details, user=None, *args, **kwargs):
+    """
+    Associate current auth with a user with the same social.uid in the DB.
+    """
+
+    if user:
+        return None
+
+    openid = details.get('openid')
+    if openid:
+        # Try to associate accounts registered with the same OpenId.
+        # Probably it is possible to get backend.strategy.storage from kwargs['storage'].
+        if backend.name == 'esgf':
+            social = backend.strategy.storage.user.get_social_auth(provider='esgf-openid', uid=kwargs['uid'])
+        elif backend.name == 'esgf-openid':
+            social = backend.strategy.storage.user.get_social_auth(provider='esgf', uid=kwargs['uid'])
+        if social:
+            return {'user': social.user, 'is_new': False}
+    return None
+
+
+def discover(openid):
+    try:
+        uri, endpoints = getServiceEndpoints(openid)
+    except DiscoveryFailure:
+        return None
+    authorize = None
+    access = None
+    for e in endpoints:
+        if e.matchTypes(['urn:esg:security:oauth:endpoint:authorize']):
+            authorize = True
+        elif e.matchTypes(['urn:esg:security:oauth:endpoint:access']):
+            access = True
+    if authorize and access:
+        return 'OAuth2'
+    return 'OpenID'

cog/config/search/__init__.py

@@ -1 +1 @@
-from config_project_search import SearchConfigParser
+from .config_project_search import SearchConfigParser

cog/config/search/config_project_search.py

@@ -1,4 +1,4 @@
-import sys, os, ConfigParser
+import sys, os, configparser
 from django.conf import settings
 from cog.models import Project, SearchGroup, SearchFacet
 from cog.utils import str2bool
@@ -15,7 +15,7 @@ def __init__(self, project):
     def _getConfigParser(self):
 
         # read project configuration
-        configParser = ConfigParser.RawConfigParser()
+        configParser = configparser.RawConfigParser()
         # must set following line explicitly to preserve the case of configuration keys
         configParser.optionxform = str 
 
@@ -25,7 +25,7 @@ def _getConfigParser(self):
     def write(self):
         '''Writes the project search configuration to the file $COG_CONFIG_DIR/projects/<project_short_name>/search.cfg'''
 
-        print 'Writing search configuration for project=%s' % self.project.short_name
+        print('Writing search configuration for project=%s' % self.project.short_name)
 
         # load project search profile
         project = Project.objects.get(short_name=self.project.short_name)
@@ -66,23 +66,23 @@ def write(self):
     def read(self):
         '''Reads the project search configuration from the file $COG_CONFIG_DIR/projects/<project_short_name>/search.cfg'''
 
-        print 'Reading search configuration for project=%s' % self.project.short_name
+        print('Reading search configuration for project=%s' % self.project.short_name)
 
         # load project search profile
         project = Project.objects.get(short_name=self.project.short_name)
         search_profile = project.searchprofile
 
         # remove existing groups of facets
         for group in search_profile.groups.all():
-            print 'Deleting search group=%s' % group
+            print('Deleting search group=%s' % group)
             group.delete()
 
         # read project configuration
         projConfig = self._getConfigParser()
         try:
             projConfig.read( self.config_file_path )
         except Exception as e:
-            print "Configuration file %s not found" % self.config_file_path
+            print("Configuration file %s not found" % self.config_file_path)
             raise e
 
         # loop over configuration sections
@@ -110,7 +110,7 @@ def read(self):
 
                 for option in projConfig.options(section):
                     value = projConfig.get(section, option)
-                    print section, option, value
+                    print(section, option, value)
                     parts = value.split("|")
                     facet_order = int(option)
                     facet_key = parts[0]

cog/db_migrations/django_openid_auth/0001_initial.py

@@ -39,7 +39,7 @@ class Migration(migrations.Migration):
                 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                 ('claimed_id', models.TextField(unique=True, max_length=2047)),
                 ('display_id', models.TextField(max_length=2047)),
-                ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL)),
+                ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)),
             ],
         ),
     ]

cog/db_migrations/django_openid_auth/0003_auto_20200324_1100.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.11 on 2020-03-24 11:00
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('django_openid_auth', '0002_auto_20160106_0812'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='useropenid',
+            name='claimed_id',
+            field=models.TextField(max_length=2047),
+        ),
+    ]

cog/forms/__init__.py

@@ -1,10 +1,10 @@
-from forms_image import *
-from forms_account import *
-from forms_bookmarks import *
-from forms_governance import *
-from forms_others import *
-from forms_post import *
-from forms_project import *
-from forms_aboutus import *
-from forms_search import *
-from forms_access_control import *
+from .forms_image import *
+from .forms_account import *
+from .forms_bookmarks import *
+from .forms_governance import *
+from .forms_others import *
+from .forms_post import *
+from .forms_project import *
+from .forms_aboutus import *
+from .forms_search import *
+from .forms_access_control import *

cog/forms/forms_account.py

@@ -52,6 +52,12 @@ class Meta:
         fields = ('claimed_id',)
 
 
+class OAuth2LoginForm(Form):
+
+    openid_identifier = CharField()
+    next = CharField(required=False)
+
+
 class PasswordResetForm(Form):
 
     openid = CharField(required=True, widget=TextInput(attrs={'size': '50'}))
@@ -286,7 +292,7 @@ def validate_username(form, user_id):
 
                         # once the openid is validated, choose the closest possible username
                         _username = createUsername(username)
-                        print 'Created username=%s from=%s' % (_username, username)
+                        print('Created username=%s from=%s' % (_username, username))
                         cleaned_data['username'] = _username  # override form data
             else:
                 # django will automatically check that the username is unique in the CoG database
@@ -304,6 +310,6 @@ def validate_field(form, field_name, field_value):
 def validate_ascii(form, field_name, field_value):
     if field_value:
         try:
-            field_value.decode('ascii')
+            field_value.encode('ascii')
         except (UnicodeDecodeError, UnicodeEncodeError):
             form._errors[field_name] = form.error_class(["'%s' contains invalid characters." % field_name])

cog/forms/forms_image.py

@@ -23,6 +23,6 @@ def clean(self):
                 self._errors["image"] = self.error_class(["Image size exceeds the maximum allowed."])
         except OSError as e:
             # image not existing on disk
-            print e
+            print(e)
 
         return self.cleaned_data