From ff769436cb082e1cffac87138ab4f78c67306022 Mon Sep 17 00:00:00 2001 From: Jeremiah Orem Date: Fri, 14 Dec 2018 16:50:36 -0800 Subject: [PATCH] rip out product_details (no longer works with this django version) --- apps/api/views.py | 88 +++++++++++++++++++-------------------- apps/lib/__init__.py | 7 ++++ apps/lib/tests.py | 4 +- apps/mirror/models.py | 97 +++++++++++++++++++++++-------------------- apps/php/tests.py | 42 ++++++++++--------- bin/start | 12 ------ requirements.txt | 2 +- settings-dist.py | 17 +++----- 8 files changed, 131 insertions(+), 138 deletions(-) diff --git a/apps/api/views.py b/apps/api/views.py index 00d1a1d..5821b4b 100644 --- a/apps/api/views.py +++ b/apps/api/views.py @@ -7,7 +7,7 @@ from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_GET, require_POST -from product_details import product_details +from lib import product_details from api.decorators import has_perm_or_basicauth, logged_in_or_basicauth from mirror.models import Location, Mirror, OS, Product, ProductAlias @@ -17,8 +17,8 @@ def _get_command_list(): - templates = os.listdir(os.path.join(os.path.dirname(__file__), 'templates', - 'api', 'docs')) + templates = os.listdir( + os.path.join(os.path.dirname(__file__), 'templates', 'api', 'docs')) # cut out the command names only commands = [t[:-5] for t in templates if t.endswith('.html')] commands.sort() @@ -28,8 +28,8 @@ def _get_command_list(): def docindex(request): """API Doc Index""" data = {'commands': _get_command_list()} - return render_to_response('api/index.html', data, context_instance= - RequestContext(request)) + return render_to_response( + 'api/index.html', data, context_instance=RequestContext(request)) @require_GET @@ -42,14 +42,15 @@ def docs(request, command): # XXX special cases are ugly if command in ('product_add', 'product_language_add', 'product_language_delete'): - langs = product_details.languages.keys() + langs = product_details('languages').keys() oses = OS.objects.order_by('name') os_list = [os.name for os in oses] - data.update({'languages': langs, - 'oses': os_list}) + data.update({'languages': langs, 'oses': os_list}) - return render_to_response('api/docs/%s.html' % command, data, - context_instance=RequestContext(request)) + return render_to_response( + 'api/docs/%s.html' % command, + data, + context_instance=RequestContext(request)) @has_perm_or_basicauth('mirror.view_uptake', HTTP_AUTH_REALM) @@ -61,8 +62,8 @@ def uptake(request): fuzzy = request.GET.get('fuzzy', False) xml = XMLRenderer() if not product and not os: - return xml.error('product and/or os are required GET parameters.', - errno=101) + return xml.error( + 'product and/or os are required GET parameters.', errno=101) product_names = None if product: @@ -131,7 +132,7 @@ def product_add(request): # check languages langs = request.POST.getlist('languages') - locales = product_details.languages.keys() + locales = product_details('languages').keys() ssl_only = bool(request.POST.get('ssl_only', False) == "true") if [l for l in langs if l not in locales]: return xml.error('invalid language code(s)', errno=103) @@ -166,9 +167,7 @@ def product_delete(request): prodname = request.POST.get('product', None) if not (prod_id or prodname): return xml.error( - 'Either product_id or product is required.', - errno=101 - ) + 'Either product_id or product is required.', errno=101) try: if prod_id: prod = Product.objects.get(pk=prod_id) @@ -209,7 +208,7 @@ def product_language_add(request): # check languages langs = request.POST.getlist('languages') - locales = product_details.languages.keys() + locales = product_details('languages').keys() if [l for l in langs if l not in locales]: return xml.error('Invalid language code(s)', errno=103) if prod.languages.filter(lang__in=langs): @@ -255,7 +254,7 @@ def product_language_delete(request): prod.languages.all().delete() return xml.success('Deleted all languages from product %s' % prod.name) - locales = product_details.languages.keys() + locales = product_details('languages').keys() if [l for l in langs if l not in locales]: return xml.error('Invalid language code(s)', errno=103) @@ -265,8 +264,8 @@ def product_language_delete(request): return xml.error(e) products = Product.objects.filter(pk=prod.pk) - return xml.success('Deleted languages %s from product %s' % ( - ', '.join(langs), prod.name)) + return xml.success( + 'Deleted languages %s from product %s' % (', '.join(langs), prod.name)) @require_GET @@ -306,8 +305,8 @@ def location_add(request): osname = request.POST.get('os', None) path = request.POST.get('path', None) if not (prodname and osname and path): - return xml.error('product, os, and path are required POST parameters.', - errno=101) + return xml.error( + 'product, os, and path are required POST parameters.', errno=101) try: product = Product.objects.get(name=prodname) @@ -399,45 +398,36 @@ def create_update_alias(request): if 'alias' in form.errors: if 'required' in form.errors['alias'][0]: return xml.error( - 'alias name is required.', - errno=form.E_ALIAS_REQUIRED - ) + 'alias name is required.', errno=form.E_ALIAS_REQUIRED) if 'same name' in form.errors['alias'][0]: return xml.error( ('You cannot create an alias with the same name as a ' 'product'), - errno=form.E_ALIAS_PRODUCT_MATCH - ) + errno=form.E_ALIAS_PRODUCT_MATCH) if 'related_product' in form.errors: if 'required' in form.errors['related_product'][0]: return xml.error( 'related_product name is required.', - errno=form.E_RELATED_NAME_REQUIRED - ) + errno=form.E_RELATED_NAME_REQUIRED) if 'same name as an existing' in form.errors['related_product'][0]: return xml.error( 'You cannot create alias with the same name as a product', - errno=form.E_ALIAS_PRODUCT_MATCH - ) + errno=form.E_ALIAS_PRODUCT_MATCH) if 'invalid' in form.errors['related_product'][0]: return xml.error( 'You must specify a valid product to match with an alias', - errno=form.E_PRODUCT_DOESNT_EXIST - ) + errno=form.E_PRODUCT_DOESNT_EXIST) return xml.error( 'There was a problem validating the data provided', - errno=form.E_ALIAS_GENERAL_VALIDATION_ERROR - ) + errno=form.E_ALIAS_GENERAL_VALIDATION_ERROR) alias = form.cleaned_data['alias'] redirect = form.cleaned_data['related_product'] alias_obj, created = ProductAlias.objects.get_or_create( - alias=alias, - defaults={'related_product': redirect} - ) + alias=alias, defaults={'related_product': redirect}) if not created: alias_obj.related_product = redirect @@ -528,10 +518,12 @@ def prepare_uptake_fake(self, products, oses): def prepare_uptake(self, uptake): """Product uptake""" - content_map = {'product': 'location__product__name', - 'os': 'location__os__name', - 'available': 'available', - 'total': 'total'} + content_map = { + 'product': 'location__product__name', + 'os': 'location__os__name', + 'available': 'available', + 'total': 'total' + } root = self.doc.createElement('mirror_uptake') self.doc.appendChild(root) @@ -549,10 +541,14 @@ def success(self, message, render=True): def error(self, message, errno=0, render=True): """Prepare an error message""" - return self.message(message, type='error', number=errno, - render=render, status=400) - - def message(self, message, type='info', number=None, render=True, + return self.message( + message, type='error', number=errno, render=render, status=400) + + def message(self, + message, + type='info', + number=None, + render=True, status=200): """Prepare a single message""" root = self.doc.createElement(type) diff --git a/apps/lib/__init__.py b/apps/lib/__init__.py index 0e58865..8c7afc5 100644 --- a/apps/lib/__init__.py +++ b/apps/lib/__init__.py @@ -1,6 +1,13 @@ +import requests import sys +def product_details(item): + res = requests.get( + 'https://product-details.mozilla.org/1.0/%s.json' % item) + return res.json() + + def docstring_trim(docstring): """Dedent a docstring. Code copied from PEP 257.""" if not docstring: diff --git a/apps/lib/tests.py b/apps/lib/tests.py index 86efde0..83464a6 100644 --- a/apps/lib/tests.py +++ b/apps/lib/tests.py @@ -1,6 +1,6 @@ from django import test -from product_details import product_details +from lib import product_details class LanguageTestCase(test.TestCase): @@ -8,4 +8,4 @@ class LanguageTestCase(test.TestCase): def test_ja_jp_mac(self): """ja-JP-mac is not a default language, but we need it.""" - assert 'ja-JP-mac' in product_details.languages.keys() + assert 'ja-JP-mac' in product_details('languages').keys() diff --git a/apps/mirror/models.py b/apps/mirror/models.py index 761edf1..6537369 100644 --- a/apps/mirror/models.py +++ b/apps/mirror/models.py @@ -4,38 +4,37 @@ from django.utils.html import escape from django.forms import TextInput -from product_details import product_details - +from lib import product_details # get the possible languages from product details LANG_CHOICES = [(key, "%s: %s" % (key, value['English'])) - for key, value in product_details.languages.items()] + for key, value in product_details('languages').items()] # This is to get a longer input box when entering locations class LongDisplayCharField(models.CharField): def formfield(self, **kwargs): - kwargs.update( - {"widget": TextInput(attrs={'style': 'width: 60em;'})} - ) + kwargs.update({"widget": TextInput(attrs={'style': 'width: 60em;'})}) return super(LongDisplayCharField, self).formfield(**kwargs) class Mirror(models.Model): """A single mirror.""" id = models.AutoField(primary_key=True) - name = models.CharField(max_length=64, unique=True, - verbose_name='Host Name') - baseurl = models.CharField(max_length=255, verbose_name='Base URL', - help_text='No trailing slash.') + name = models.CharField( + max_length=64, unique=True, verbose_name='Host Name') + baseurl = models.CharField( + max_length=255, + verbose_name='Base URL', + help_text='No trailing slash.') rating = models.IntegerField() active = models.BooleanField() - count = models.DecimalField(max_digits=20, decimal_places=0, default=0, - db_index=True) - regions = models.ManyToManyField('geoip.Region', - db_table='geoip_mirror_region_map') - contacts = models.ManyToManyField(User, verbose_name="Admin Contact", - blank=True, null=True) + count = models.DecimalField( + max_digits=20, decimal_places=0, default=0, db_index=True) + regions = models.ManyToManyField( + 'geoip.Region', db_table='geoip_mirror_region_map') + contacts = models.ManyToManyField( + User, verbose_name="Admin Contact", blank=True, null=True) class Meta: db_table = 'mirror_mirrors' @@ -46,12 +45,14 @@ def __unicode__(self): def admin_contacts(self): """get the administrative contacts for this mirror as HTML""" contacts = self.contacts.order_by('last_name', 'first_name') - contacts = ['%s' % ( - reverse('admin:auth_user_change', args=(c.pk,)), - escape('%s: %s' % (c.get_full_name() or c.username, - c.email)) - ) for c in contacts] + contacts = [ + '%s' % + (reverse('admin:auth_user_change', args=(c.pk, )), + escape('%s: %s' % (c.get_full_name() or c.username, c.email))) + for c in contacts + ] return '
'.join(contacts) or '' + admin_contacts.allow_tags = True @@ -72,17 +73,16 @@ class Meta: class Product(models.Model): """A single product, e.g., Firefox-3.5.6.""" id = models.AutoField(primary_key=True) - name = models.CharField(max_length=255, unique=True, - verbose_name='Product Name') + name = models.CharField( + max_length=255, unique=True, verbose_name='Product Name') priority = models.IntegerField(default=1) - count = models.DecimalField(max_digits=20, decimal_places=0, default=0, - verbose_name='Downloads') + count = models.DecimalField( + max_digits=20, decimal_places=0, default=0, verbose_name='Downloads') active = models.BooleanField(db_index=True, default=True) - checknow = models.BooleanField(db_index=True, verbose_name='Check Now?', - default=True) - ssl_only = models.BooleanField(db_index=False, - verbose_name='Serve Over SSL Only?', - default=False) + checknow = models.BooleanField( + db_index=True, verbose_name='Check Now?', default=True) + ssl_only = models.BooleanField( + db_index=False, verbose_name='Serve Over SSL Only?', default=False) def __unicode__(self): return self.name @@ -107,8 +107,11 @@ class ProductLanguage(models.Model): "all languages" or "not applicable". """ product = models.ForeignKey('Product', related_name='languages') - lang = models.CharField(max_length=30, choices=LANG_CHOICES, - db_column='language', verbose_name='Language') + lang = models.CharField( + max_length=30, + choices=LANG_CHOICES, + db_column='language', + verbose_name='Language') class Meta: db_table = 'mirror_product_langs' @@ -124,23 +127,23 @@ class Location(models.Model): product = models.ForeignKey('Product') os = models.ForeignKey('OS', verbose_name='OS') path = LongDisplayCharField( - max_length=255, help_text=( - 'Always use a leading slash.
' - 'The placeholder :lang will be replaced with the requested ' - 'language at download time.')) + max_length=255, + help_text=('Always use a leading slash.
' + 'The placeholder :lang will be replaced with the requested ' + 'language at download time.')) class Meta: db_table = 'mirror_locations' unique_together = ('product', 'os') - permissions = ( - ('view_uptake', 'Can view mirror uptake'), - ) + + permissions = (('view_uptake', 'Can view mirror uptake'), ) def __unicode__(self): return self.path @staticmethod - def get_mirror_uptake(products=None, oses=None, + def get_mirror_uptake(products=None, + oses=None, order_by='location__product__name'): """ Given a list of product IDs and/or OS IDs, return a list of these @@ -193,11 +196,15 @@ class LocationMirrorLanguageException(models.Model): serve the respective location in all available languages. Entries in this table mark the exceptions from that rule. """ - lmm = models.ForeignKey('LocationMirrorMap', - related_name='lang_exceptions', - db_column='location_mirror_map_id') - lang = models.CharField(max_length=30, choices=LANG_CHOICES, - db_column='language', verbose_name='Language') + lmm = models.ForeignKey( + 'LocationMirrorMap', + related_name='lang_exceptions', + db_column='location_mirror_map_id') + lang = models.CharField( + max_length=30, + choices=LANG_CHOICES, + db_column='language', + verbose_name='Language') class Meta: db_table = 'mirror_lmm_lang_exceptions' diff --git a/apps/php/tests.py b/apps/php/tests.py index 702b6c8..8861fcb 100644 --- a/apps/php/tests.py +++ b/apps/php/tests.py @@ -13,7 +13,7 @@ from django.test import TestCase from mirror.models import Location -from product_details import product_details +from lib import product_details class BounceTestCase(TestCase): @@ -34,8 +34,8 @@ def test_noparams(self): self.conn.request('HEAD', self.parsed_url.path) req = self.conn.getresponse() self.assertEqual(req.status, 302) - self.assertTrue(req.getheader('Location', '').startswith( - 'http://www.mozilla.com')) + self.assertTrue( + req.getheader('Location', '').startswith('http://www.mozilla.com')) def test_defaults(self): """ @@ -48,32 +48,34 @@ def test_defaults(self): data = {'product': myprod.name} - self.conn.request('HEAD', '%s?%s' % (self.parsed_url.path, - urllib.urlencode(data))) + self.conn.request( + 'HEAD', '%s?%s' % (self.parsed_url.path, urllib.urlencode(data))) req = self.conn.getresponse() self.assertEqual(req.status, 302) - self.assertTrue(req.getheader('Location', '').endswith( - self._lang_placeholder(myloc.path, 'en-US'))) + self.assertTrue( + req.getheader('Location', '').endswith( + self._lang_placeholder(myloc.path, 'en-US'))) def test_languages_and_oses(self): """Test all known locations""" locs = Location.objects.all() - langs = product_details.languages.keys() + langs = product_details('languages').keys() for loc in locs: for lang in langs: - data = {'product': loc.product.name, - 'os': loc.os.name, - 'lang': lang} + data = { + 'product': loc.product.name, + 'os': loc.os.name, + 'lang': lang + } - self.conn.request('HEAD', '%s?%s' % (self.parsed_url.path, - urllib.urlencode(data))) + self.conn.request( + 'HEAD', + '%s?%s' % (self.parsed_url.path, urllib.urlencode(data))) req = self.conn.getresponse() - self.assertEqual( - req.status, - 302, - 'Require redirect for %s' % str(data) - ) - self.assertTrue(req.getheader('Location', '').endswith( - self._lang_placeholder(loc.path, lang))) + self.assertEqual(req.status, 302, + 'Require redirect for %s' % str(data)) + self.assertTrue( + req.getheader('Location', '').endswith( + self._lang_placeholder(loc.path, lang))) diff --git a/bin/start b/bin/start index efa3ea0..cf0ef1f 100755 --- a/bin/start +++ b/bin/start @@ -3,18 +3,6 @@ set -e set -u -PID=$$ -python manage.py update_product_details - -( - sleep 1800 - while true; do - python manage.py update_product_details && \ - kill -s HUP $PID - sleep 1800 - done -) & - rm -rf /static/media /static/admin-media mkdir -p /static/media diff --git a/requirements.txt b/requirements.txt index d39f6b9..8b5a0f1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,4 @@ ipaddr==2.1.11 Markdown==2.0.3 South>=0.6.2 python-memcached==1.48 -django-mozilla-product-details==0.13 +requests==2.21.0 \ No newline at end of file diff --git a/settings-dist.py b/settings-dist.py index f8d4ac8..1ba1923 100644 --- a/settings-dist.py +++ b/settings-dist.py @@ -27,8 +27,8 @@ 'PORT': '', 'OPTIONS': { 'init_command': 'SET storage_engine=InnoDB', - 'charset' : 'utf8', - 'use_unicode' : True, + 'charset': 'utf8', + 'use_unicode': True, }, 'TEST_CHARSET': 'utf8', 'TEST_COLLATION': 'utf8_general_ci', @@ -85,10 +85,8 @@ ) # auth backends: uses django first, and converts old Bouncer users as needed -AUTHENTICATION_BACKENDS = ( - 'django.contrib.auth.backends.ModelBackend', - 'users.auth.backend.ConversionBackend' -) +AUTHENTICATION_BACKENDS = ('django.contrib.auth.backends.ModelBackend', + 'users.auth.backend.ConversionBackend') # user profile stuff AUTH_PROFILE_MODULE = 'users.UserProfile' @@ -100,8 +98,7 @@ # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates". # Always use forward slashes, even on Windows. # Don't forget to use absolute paths, not relative paths. - path('templates') -) + path('templates')) # path to mozilla product details PROD_DETAILS_DIR = path('inc', 'product-details', 'json') @@ -113,10 +110,7 @@ 'lib', #'php', # enable this if you want to run tests on the bounce script 'users', - - 'product_details', 'south', - 'django.contrib.admin', 'django.contrib.auth', 'django.contrib.contenttypes', @@ -139,4 +133,3 @@ from local_settings import * except ImportError, exp: pass -