diff --git a/datapackage_pipelines_budgetkey/pipelines/budgetkey/elasticsearch/add_support_helpers.py b/datapackage_pipelines_budgetkey/pipelines/budgetkey/elasticsearch/add_support_helpers.py new file mode 100644 index 00000000..5a814a36 --- /dev/null +++ b/datapackage_pipelines_budgetkey/pipelines/budgetkey/elasticsearch/add_support_helpers.py @@ -0,0 +1,32 @@ +from datapackage_pipelines.wrapper import process + + +def process_row(row, _1, spec, _2, params, _3): + if spec['name'] == 'supports': + payments = row.get('payments', []) + if payments: + row['support_title'] = payments[0].get('support_title') + row['supporting_ministry'] = payments[0].get('supporting_ministry', '') + + return row + + +def modify_datapackage(dp, params, *_): + for res in dp['resources']: + if res['name'] == 'supports': + res['schema']['fields'].append({ + 'name': 'support_title', + 'type': 'string', + 'es:title': True + }) + res['schema']['fields'].append({ + 'name': 'supporting_ministry', + 'type': 'string', + 'es:keyword': True + }) + return dp + + +if __name__ == '__main__': + process(modify_datapackage=modify_datapackage, + process_row=process_row) diff --git a/datapackage_pipelines_budgetkey/pipelines/budgetkey/elasticsearch/budgetkey.source-spec.yaml b/datapackage_pipelines_budgetkey/pipelines/budgetkey/elasticsearch/budgetkey.source-spec.yaml index 4fe1692d..391e02c3 100644 --- a/datapackage_pipelines_budgetkey/pipelines/budgetkey/elasticsearch/budgetkey.source-spec.yaml +++ b/datapackage_pipelines_budgetkey/pipelines/budgetkey/elasticsearch/budgetkey.source-spec.yaml @@ -105,6 +105,7 @@ supports: resource: supports source-field: budget_code target-field: 'nice-budget-code' + - run: add_support_helpers # keep-history: # - # key: payments diff --git a/datapackage_pipelines_budgetkey/pipelines/simpledb/process.py b/datapackage_pipelines_budgetkey/pipelines/simpledb/process.py index 09ce1076..eec76125 100644 --- a/datapackage_pipelines_budgetkey/pipelines/simpledb/process.py +++ b/datapackage_pipelines_budgetkey/pipelines/simpledb/process.py @@ -505,7 +505,7 @@ def debug_source(source, debug): search=dict( index='supports', field_map={ - 'budget_code': 'nice-code', + 'budget_code': 'nice-budget-code', 'purpose': 'support_title', 'supporting_ministry': 'supporting_ministry', 'request_type': 'request_type',