Merge pull request #85 from ride90/v0.5

V0.5

client/belga/360archive/BelgaSearchPanelController.ts

@@ -0,0 +1,43 @@
+interface Subject {
+  id: string;
+  name: string;
+}
+
+export default class BelgaSearchPanelController {
+  languages: Array<Subject>;
+  periods: Array<Subject>;
+  types: Array<Subject>;
+
+  $onInit() {
+    this.languages = [
+      { name: "DE", id: "de" },
+      { name: "FR", id: "fr" },
+      { name: "EN", id: "en" },
+      { name: "ES", id: "es" },
+      { name: "NL", id: "nl" }
+    ];
+
+    this.types = [
+      { name: "alert", id: "alert" },
+      { name: "text", id: "text" },
+      { name: "brief", id: "brief" },
+      { name: "short", id: "short" }
+    ];
+
+    this.periods = [
+      { name: "Whenever", id: "" },
+      { name: "Last 4 hours", id: "last4h" },
+      { name: "Last 8 hours", id: "last8h" },
+      { name: "Today", id: "today" },
+      { name: "Yesterday", id: "yesterday" },
+      { name: "Last 24 hours", id: "day" },
+      { name: "Last 48 hours", id: "last48h" },
+      { name: "Last 72 hours", id: "last74h" },
+      { name: "Last week", id: "week1" },
+      { name: "Last 2 weeks", id: "week2" },
+      { name: "Last month", id: "month" },
+      { name: "Last year", id: "year" },
+      { name: "Last 2 years", id: "year2" }
+    ];
+  }
+}

client/belga/360archive/index.js

@@ -0,0 +1,14 @@
+import angular from 'angular';
+
+import BelgaSearchPanelController from './BelgaSearchPanelController';
+
+export default angular.module('belga.360archive', [
+])
+    .controller('BelgaSearchPanelController', BelgaSearchPanelController)
+    .run(['$templateCache',($templateCache) => {
+        $templateCache.put(
+            'search-panel-belga_360archive.html',
+            require('./views/search-panel.html')
+        );
+    }])
+;

client/belga/360archive/views/search-panel.html

@@ -0,0 +1,52 @@
+<fieldset ng-controller="BelgaSearchPanelController as searchPanel">
+    <div class="field">
+        <label for="search-headline">{{:: 'Headline' | translate}}</label>
+        <div class="control">
+            <input type="text" id="search-headline" ng-model="params.headline" tabindex="1">
+        </div>
+    </div>
+
+    <div class="field">
+        <label class="search-label">Language</label>
+        <select ng-model="params.languages">
+            <option ng-repeat="language in searchPanel.languages" value="{{ language.id }}">{{ language.name }}</option>
+        </select>
+    </div>
+
+    <div class="field">
+        <label class="search-label">Content type</label>
+        <select ng-model="params.types">
+            <option ng-repeat="type in searchPanel.types" value="{{ type.id }}">{{ type.name }}</option>
+        </select>
+    </div>
+
+    <div class="field">
+        <label for="search-headline">{{:: 'Credits' | translate}}</label>
+        <div class="control">
+            <input type="text" id="search-credits" ng-model="params.credits" tabindex="1">
+        </div>
+    </div>
+
+    <div class="field">
+        <label class="search-label" translate>Period</label>
+        <select ng-model="params.period">
+            <option ng-repeat="period in searchPanel.periods" value="{{ period.id }}">{{ period.name }}</option>
+        </select>
+    </div>
+    <div class="form__row form__row--flex form__row--small-padding">
+        <div class="form__row-item">
+            <label class="form-label" translate>From</label>
+            <div sd-datepicker
+                ng-model="params.dates.start"
+                data-on-change="onDatesChange()"
+            ></div>
+        </div>
+        <div class="form__row-item">
+            <label class="form-label" translate>To</label>
+            <div sd-datepicker
+                ng-model="params.dates.end"
+                data-on-change="onDatesChange()"
+            ></div>
+        </div>
+    </div>
+</fieldset>

client/belga/index.tsx

@@ -0,0 +1,27 @@
+import React from 'react';
+import angular from 'angular';
+import {IUser} from 'superdesk-api'
+import {startApp} from 'superdesk-core/scripts/index';
+import belgaImage from './image';
+import belga360Archive from './360archive';
+
+class UserAvatar extends React.Component<{user: IUser}> {
+    render() {
+        return (
+            <div className="user-avatar" data-test-id="user-avatar">
+                <figure className="avatar avatar--no-margin initials">
+                    {this.props.user.sign_off}
+                </figure>
+            </div>
+        );
+    }
+}
+
+setTimeout(() => {
+    startApp({UserAvatar});
+});
+
+export default angular.module('belga', [
+    belgaImage.name,
+    belga360Archive.name,
+]);

client/belga/typings/refs.d.ts

@@ -0,0 +1 @@
+/// <reference path='../../../client/node_modules/superdesk-core/scripts/core/superdesk-api.d.ts' />

client/extensions/belgaCoverage/src/belga-coverage-images.tsx

@@ -24,7 +24,7 @@ function getImageUrl(image: IBelgaImage, rendition: IProps['rendition']): string
     }
 }
 
-export default class BelgaCoverage extends React.Component<IProps, IState> {
+export default class BelgaCoverage extends React.PureComponent<IProps, IState> {
 
     constructor(props: IProps) {
         super(props);

client/superdesk.config.js

@@ -47,6 +47,11 @@ module.exports = function(grunt) {
             planning: true,
             searchShortcut: true,
             editFeaturedImage: false,
+            customAuthoringTopbar: {
+                publish: true,
+                publishAndContinue: true,
+                closeAndContinue: true,
+            },
         },
         workspace: {
             analytics: true,
@@ -91,7 +96,16 @@ module.exports = function(grunt) {
                 'fetchedDesk',
                 'nestedlink',
                 'associatedItems',
+                'translations',
             ]
         },
+        monitoring: {
+            scheduled: {
+                sort: {
+                    default: { field: 'publish_schedule', order: 'asc' },
+                    allowed_fields_to_sort: [ 'publish_schedule' ]
+                }
+            },
+        },
     };
 };

server/Procfile

@@ -3,4 +3,5 @@ wamp: python3 -u ws.py
 work: celery -A worker worker
 beat: celery -A worker beat --pid=
 capi: gunicorn -c gunicorn_config.py content_api.wsgi
+papi: gunicorn -c gunicorn_config.py prod_api.wsgi
 #highcharts: python3 -u -m analytics.reports.highcharts_server

server/belga/io/feed_parsers/__init__.py

@@ -5,7 +5,8 @@
 from . import belga_tass_newsml_1_2  # noqa
 from . import belga_iptc7901  # noqa
 from . import belga_anpa  # noqa
-from . import belga_spreadsheet  # noqa
 from . import belga_dpa_newsml_2_0  # noqa
 from . import belga_ats_newsml_1_2  # noqa
 from . import belga_kyodo_newsml_1_2  # noqa
+from . import belga_spreadsheet  # noqa
+from . import belga_tip_newsml_1_2 # noqa

server/belga/io/feed_parsers/base_belga_newsml_1_2.py

@@ -70,6 +70,13 @@ def parse(self, xml, provider=None):
                 try:
                     item = item_envelop.copy()
                     self.parser_newsitem(item, newsitem_el)
+                    # add product is GENERAL, if product is empty
+                    if not [it for it in item.get('subject', []) if it.get('scheme') == 'news_products']:
+                        product = {"name": 'GENERAL', "qcode": 'GENERAL', "scheme": "news_products"}
+                        item.setdefault('subject', []).append(product)
+                    # add service is NEW
+                    service = {"name": 'NEWS', "qcode": 'NEWS', "scheme": "news_services"}
+                    item.setdefault('subject', []).append(service)
                     item = self.populate_fields(item)
                 except SkipItemException:
                     continue
@@ -104,23 +111,6 @@ def parser_newsenvelop(self, envelop_el):
         if element is not None:
             item['ingest_provider_sequence'] = element.text
 
-        # news_services CV
-        for element in envelop_el.findall('NewsService'):
-            if element is not None and element.get('FormalName'):
-                item.setdefault('subject', []).append({
-                    "name": element.get('FormalName'),
-                    "qcode": element.get('FormalName'),
-                    "scheme": "news_services"
-                })
-
-        for element in envelop_el.findall('NewsProduct'):
-            if element is not None and element.get('FormalName'):
-                item.setdefault('subject', []).append({
-                    "name": element.get('FormalName'),
-                    "qcode": element.get('FormalName'),
-                    "scheme": "news_products"
-                })
-
         element = envelop_el.find('Priority')
         if element is not None:
             item['priority'] = int(element.get('FormalName', 0))

server/belga/io/feed_parsers/belga_afp_newsml_1_2.py

@@ -13,6 +13,7 @@
 from superdesk.io.registry import register_feed_parser
 
 from .base_belga_newsml_1_2 import BaseBelgaNewsMLOneFeedParser
+import unicodedata
 
 
 class BelgaAFPNewsMLOneFeedParser(BaseBelgaNewsMLOneFeedParser):
@@ -29,44 +30,30 @@ class BelgaAFPNewsMLOneFeedParser(BaseBelgaNewsMLOneFeedParser):
         'POL': 'POLITICS',
     }
 
-    def parse(self, xml, provider=None):
-        def add_unique_item_to_list(data, _list):
-            if data not in _list:
-                _list.append(data)
-
-        items = super().parse(xml, provider)
-        # mapping data follow xsl file
-        for item in items:
-            # mapping from keyword
-            for keyword in item.get('keywords', []):
-                keyword = keyword.strip('/').upper()
-                product_code = [k if keyword in v else 'GENERAL' for k, v in self.MAPPING_KEYWORDS.items()][0]
+    def parser_newsitem(self, item, newsitem_el):
+        super().parser_newsitem(item, newsitem_el)
+        # mapping product from keyword, and have only one product
+        product = {}
+        for keyword in item.get('keywords', []):
+            keyword = unicodedata.normalize('NFKD', keyword.strip(
+                '/').upper()).encode('ascii', 'ignore').decode('utf-8')
+            product_codes = [k for k, v in self.MAPPING_KEYWORDS.items() for it in v if keyword in it]
+            if product_codes:
                 product = {
-                    "name": product_code,
-                    "qcode": product_code,
+                    "name": product_codes[0],
+                    "qcode": product_codes[0],
                     "scheme": "news_products"
                 }
-                add_unique_item_to_list(product, item.get('subject', []))
-            # mapping from anpa_category
-            for category in item.get('anpa_category', []):
-                qcode = category.get('qcode')
-                product = {
-                    "name": self.MAPPING_CATEGORY.get(qcode, 'GENERAL'),
-                    "qcode": self.MAPPING_CATEGORY.get(qcode, 'GENERAL'),
-                    "scheme": "news_products"
-                }
-                add_unique_item_to_list(product, item.get('subject', []))
-        return items
-
-    def parse(self, xml, provider=None):
-        items = super().parse(xml, provider)
-        for item in items:
-            if item.get('urgency') in ('1', '2') and not item.get('headline'):
-                first_line = item.get('body_html', '').strip().split('\n')[0]
-                first_line = etree.fromstring(first_line).text
-                headline = 'URGENT: ' + first_line.strip()
-                item['headline'] = headline
-        return items
+                item.setdefault('subject', []).append(product)
+                break
+
+        # add content for headline when it is empty
+        if item.get('urgency') in ('1', '2') and not item.get('headline'):
+            first_line = item.get('body_html', '').strip().split('\n')[0]
+            first_line = etree.fromstring(first_line).text
+            headline = 'URGENT: ' + first_line.strip()
+            item['headline'] = headline
+        return item
 
 
 register_feed_parser(BelgaAFPNewsMLOneFeedParser.NAME, BelgaAFPNewsMLOneFeedParser())

server/belga/io/feed_parsers/belga_anp_newsml_1_2.py

@@ -32,22 +32,16 @@ def parser_newsmanagement(self, item, manage_el):
         item['firstcreated'] = item['firstcreated'].astimezone(pytz.utc)
         item['versioncreated'] = item['versioncreated'].astimezone(pytz.utc)
 
-    def parse(self, xml, provider=None):
-        items = super().parse(xml, provider)
-        for item in items:
-            news_products = []
-            for subject in item.get('subject', []):
-                if subject.get('scheme', '') == 'genre':
-                    qcode = subject.get('name')
-                    product = {
-                        'name': self.MAPPING_PRODUCTS.get(qcode, 'GENERAL'),
-                        'qcode': self.MAPPING_PRODUCTS.get(qcode, 'GENERAL'),
-                        'scheme': 'news_products',
-                    }
-                    if product not in item.get('subject', []):
-                        news_products.append(product)
-            item.setdefault('subject', []).extend(news_products)
-        return items
+    def parser_newsitem(self, item, newsitem_el):
+        super().parser_newsitem(item, newsitem_el)
+        product = {}
+        for subject in item.get('subject', []):
+            if subject.get('scheme', '') == 'genre':
+                qcode = self.MAPPING_PRODUCTS.get(subject.get('name'))
+                product = {'name': qcode, 'qcode': qcode, 'scheme': 'news_products'} if qcode else None
+                item.setdefault('subject', []).append(product)
+                break
+        return item
 
 
 register_feed_parser(BelgaANPNewsMLOneFeedParser.NAME, BelgaANPNewsMLOneFeedParser())

server/belga/io/feed_parsers/belga_anpa.py

@@ -64,11 +64,12 @@ def parse(self, file_path, provider=None):
                 qcode = m.group(2).decode().upper()
                 item['priority'] = self.map_priority(m.group(1).decode())
                 item['anpa_category'] = [{'qcode': qcode}]
-                item['subject'] = [{
-                    'qcode': self.MAPPING_PRODUCTS.get(qcode, 'GENERAL'),
-                    'name': self.MAPPING_PRODUCTS.get(qcode, 'GENERAL'),
-                    'scheme': 'news_products'
-                }]
+                # Mapping product
+                qcode = self.MAPPING_PRODUCTS.get(qcode, 'GENERAL')
+                item.setdefault('subject', []).append({'qcode': qcode, 'name': qcode, 'scheme': 'news_products'})
+                # service is always equal NEWS
+                service = {"name": 'NEWS', "qcode": 'NEWS', "scheme": "news_services"}
+                item.setdefault('subject', []).append(service)
                 item['slugline'] = m.group(6).decode('latin-1', 'replace')
                 item['anpa_take_key'] = m.group(7).decode('latin-1', 'replace').strip()
                 item['word_count'] = int(m.group(10).decode())

server/belga/io/feed_parsers/belga_efe_newsml_1_2.py

@@ -22,21 +22,20 @@ class BelgaEFENewsMLOneFeedParser(BaseBelgaNewsMLOneFeedParser):
 
     MAPPING_PRODUCTS = {
         'SPO': 'SPORTS',
-        'ECO': 'ECONOMY',
+        'POL': 'POLITICS',
     }
 
     # efe related logic goes here
     def parser_contentitem(self, item, content_el):
         super().parser_contentitem(item, content_el)
         categoria = content_el.find('DataContent/nitf/head/meta[@name="categoria"]')
-        qcode = categoria.attrib.get('content').upper() if categoria is not None else 'GENERAL'
-
-        item.setdefault('anpa_category', []).append({'qcode': qcode})
-        item.setdefault('subject', []).append({
-            'qcode': self.MAPPING_PRODUCTS.get(qcode, 'GENERAL'),
-            'name': self.MAPPING_PRODUCTS.get(qcode, 'GENERAL'),
-            'scheme': 'news_products',
-        })
+        if categoria is not None:
+            qcode = categoria.attrib.get('content').upper() if categoria is not None else None
+            if qcode:
+                item.setdefault('anpa_category', []).append({'qcode': qcode})
+                qcode = self.MAPPING_PRODUCTS.get(qcode)
+                item.setdefault('subject', []).append({'qcode': qcode, 'name': qcode, 'scheme': 'news_products'})
+        return item
 
 
 register_feed_parser(BelgaEFENewsMLOneFeedParser.NAME, BelgaEFENewsMLOneFeedParser())