diff --git a/app/assets/_dev/css/pages/map-page.css b/app/assets/_dev/css/pages/map-page.css index 47fd961b..31d0832b 100644 --- a/app/assets/_dev/css/pages/map-page.css +++ b/app/assets/_dev/css/pages/map-page.css @@ -1,4 +1,5 @@ -.find-country, .browse-by-region { +.find-country, +.browse-by-region { @apply my-8; } @@ -16,7 +17,7 @@ .country-search__form-wrapper { @apply relative w-4/6 flex; - + .btn { @apply p-1; } @@ -37,7 +38,7 @@ } &.--no-filters { - background-color: transparent; + background-color: transparent; @include max-width(lg) { @apply flex-col; @@ -46,7 +47,6 @@ } } } - } .map__country-data-box { @@ -63,10 +63,10 @@ .ghost-link { @apply my-0; + margin-bottom: 10px; } } - .map__country-data-header { @apply flex items-center w-full; @apply m-0 p-2 font-bold font-body text-16; @@ -76,13 +76,12 @@ svg { @apply mr-4; * { - stroke: #3C31D5; + stroke: #3c31d5; } @include max-width(lg) { display: none; } - } > * { @@ -93,8 +92,7 @@ .map__country-data-box.--active { @apply bg-white; - - .map__country-data-header{ + .map__country-data-header { @apply bg-blue-light; @apply border-2 border-blue-light; @apply text-black; @@ -106,10 +104,9 @@ } } } - .country__header-card_row { + .country__header-card_row { @apply text-black; } - } .country__header-card_row { @@ -173,8 +170,8 @@ @apply p-2; } - -.map-page__intro, .region-page__intro { +.map-page__intro, +.region-page__intro { @apply border-b border-black/50; @apply pb-10; @apply my-10; @@ -197,4 +194,4 @@ @apply bg-blue-light text-blue-medium; } } -} \ No newline at end of file +} diff --git a/app/assets/dist/files/metadata.csv b/app/assets/dist/files/metadata.csv new file mode 100644 index 00000000..f5d58c91 --- /dev/null +++ b/app/assets/dist/files/metadata.csv @@ -0,0 +1,10 @@ +Beneficial ownership register information,Metadata +Country,Name of the country that holds the beneficial ownership register. +Name of register,"Name associated with the sectoral or full-economy beneficial ownership register, either as documented in legislation or as referenced by public authorities. Data does not include subnational registers." +Registrar,The government department that is mandated by the regulation to hold the BO registry. +Type of agency,"Type of government departments that is responsible for maintaining the BO registry. Options include Company registrars, Tax authorities, Financial Intelligence Units, Extractive Industries regulators, among others." +Status,"This field relates to the implementation status of the BO register. Can be one of the following:- Planned: The country has publicly committed to implementing a beneficial ownership register (either sectorial or full-economy), but there is no public evidence that the work has started yet.- Implementing: The country is in the process of setting up the legal and technical systems required for a beneficial ownership register, but the information is not yet being collected. e.g. a bill has been drafted.- Live register: There is a register where beneficial ownership information is being collected and stored. Means of accessing the information may vary among data users." +Access,"Describes which data users have specific and dedicated access provisions to the BOI contained in the register, based on the legislation:- Registrar (only the authority holding the register can access the information)- General public (this implies anyone can access the information in the register. Being accessible to the general public doesn’t prevent the register to impose a fee or to justify a legitimate interest for accessing the information)- Competent authorities (such as Financial Intelligence Units, investigations and prosecutions departments or drug and organised crime agencies)- Obliged entities (i.e. AML-regulated entities)- Non-obliged entities (i.e. all other businesses, regular businesses)- Civil society (non-government organisations according to the definition of each jurisdiction, media outlets, academia, other).- Non-competent authorities (any government agency that doesn’t have an anti-money laundering mandate, such as procurement agencies)- Foreign competent authorities (FIUs or others from a different jurisdiction to the one holding the register)" +Scope,"This field describes the scope of the register, meaning the type of corporate vehicles requested to submit information. Options include:- Full economy (covering corporate vehicles registered in the country, and operating in all the sectors of the economy)- Sectorial (covering corporate vehicles registered in the country, operating in a particular sector of the economy, such as procurement, extractives, land or other). For further information, we recommend to consult the regulation which will specify which type of corporate vehicles are required to disclose their information, and what’s the extent of the register." +Register link,URL with a link to the beneficial ownership register or to a location where further information about how to access beneficial ownership information in a country. +Data structured in BODS,Whether a country has decided to structure their BO data under the Beneficial Ownership Data Standard (BODS). \ No newline at end of file diff --git a/app/config/urls.py b/app/config/urls.py index 3c7f40e6..b32779ba 100644 --- a/app/config/urls.py +++ b/app/config/urls.py @@ -1,42 +1,48 @@ # 3rd party from django.conf import settings -from django.urls import path, re_path from django.conf.urls import include from django.conf.urls.i18n import i18n_patterns from django.conf.urls.static import static +from django.contrib import admin from django.contrib.staticfiles.urls import staticfiles_urlpatterns from django.http import HttpResponse -from django.contrib import admin +from django.urls import path, re_path +from wagtail import urls as wagtail_urls from wagtail.admin import urls as wagtailadmin_urls from wagtail.contrib.sitemaps.views import sitemap -from wagtail import urls as wagtail_urls from wagtail.documents import urls as wagtaildocs_urls +from modules.content.views import CountryView, RegionView, SearchView + # Module from modules.core.views import ( - robots, error_400_view, error_403_view, error_404_test, error_404_view, error_500_view + error_400_view, + error_403_view, + error_404_test, + error_404_view, + error_500_view, + robots, ) - -from modules.content.views import CountryView, RegionView, SearchView -from modules.notion.views import CountryExport, CountriesExport - +from modules.notion.views import CountriesExport, CountryExport, serve_csv_file urlpatterns = [ # Django and Wagtail views - re_path(r'^admin/', include(wagtailadmin_urls)), - path('django-admin/', admin.site.urls), - re_path(r'^documents/', include(wagtaildocs_urls)), - re_path(r'^404/$', error_404_test, ), - re_path(r'^500/$', error_500_view, ), + re_path(r"^admin/", include(wagtailadmin_urls)), + path("django-admin/", admin.site.urls), + re_path(r"^documents/", include(wagtaildocs_urls)), + re_path(r"^404/$", error_404_test), + re_path(r"^500/$", error_500_view), # Server / Robots / Verification etc - re_path(r'^robots\.txt$', robots), - re_path(r'^sitemap\.xml$', sitemap), + re_path(r"^robots\.txt$", robots), + re_path(r"^sitemap\.xml$", sitemap), re_path( - r'^googleverfication\.html$', + r"^googleverfication\.html$", lambda r: HttpResponse( - "google-site-verification: foo.html", content_type="text/plain" - ) + "google-site-verification: foo.html", + content_type="text/plain", + ), ), + path("metadata.csv", serve_csv_file, name="serve_metadata"), ] if settings.DEBUG: @@ -51,10 +57,10 @@ handler500 = error_500_view urlpatterns = urlpatterns + i18n_patterns( - path('map/country//', CountryView.as_view(), name="country-tag"), - path('map/country/.csv', CountryExport.as_view(), name="country-export"), - path('map/region//', RegionView.as_view(), name="region"), - path('map/oo_all_country_data.csv', CountriesExport.as_view(), name="countries-export"), + path("map/country//", CountryView.as_view(), name="country-tag"), + path("map/country/.csv", CountryExport.as_view(), name="country-export"), + path("map/region//", RegionView.as_view(), name="region"), + path("map/oo_all_country_data.csv", CountriesExport.as_view(), name="countries-export"), path("search/", SearchView.as_view(), name="search"), - re_path(r'', include(wagtail_urls)), + re_path(r"", include(wagtail_urls)), ) diff --git a/app/modules/notion/urls.py b/app/modules/notion/urls.py new file mode 100644 index 00000000..692d08ce --- /dev/null +++ b/app/modules/notion/urls.py @@ -0,0 +1,7 @@ +from django.urls import path + +from .views import serve_csv_file + +urlpatterns = [ + path("metadata.csv", serve_csv_file, name="serve_csv_file"), +] diff --git a/app/modules/notion/views.py b/app/modules/notion/views.py index 1f4ddb56..fec49c52 100644 --- a/app/modules/notion/views.py +++ b/app/modules/notion/views.py @@ -1,51 +1,49 @@ # stdlib import csv +from pathlib import Path # 3rd party import arrow from consoler import console # NOQA -from django.http import Http404, HttpResponse -from django.views import View +from django.conf import settings +from django.http import FileResponse, Http404, HttpResponse from django.utils.functional import cached_property +from django.views import View +from loguru import logger # NOQA # Project from modules.notion.models import Commitment, CountryTag, DisclosureRegime - BASE_HEADERS = [ - 'Name of register', - # Implementation / regime fields - 'Link', - 'Central register implemented', - 'Public access', - 'Scope', - 'Register launched', - 'Data structured in BODS', - 'Responsible agency', - 'Agency type', - 'Who can access', + "Name of register", # Register name from Implementation tracker + "Link", + "Scope", + "Register launched", + "Data structured in BODS", + "Responsible agency", + "Agency type", + "Who can access", ] -COUNTRY_HEADERS = ['', 'Stage'] + BASE_HEADERS +COUNTRY_HEADERS = ["", "Stage"] + BASE_HEADERS -ALL_HEADERS = ['Country', 'Stage'] + BASE_HEADERS +ALL_HEADERS = ["Country", "Stage"] + BASE_HEADERS class DataExportBase(View): - """Shared functionality between the exporters for both the CountryExport and CountriesExport classes. """ def _yes_no(self, val): - if val: + if val.lower() == "yes": return "Yes" - return "No" + return "" def _format_date(self, val): try: dt = arrow.get(val) - return dt.format('YYYY-MM-DD') + return dt.format("YYYY-MM-DD") except Exception: return "" @@ -75,7 +73,7 @@ def _get_commitment_row(self, commitment: Commitment, skip_one: bool = False) -> row.append("") return row - def _get_regime_row(self, regime: DisclosureRegime) -> list: + def _get_regime_row(self, regime: DisclosureRegime, is_single: bool = True) -> list: """Creates a data row (list) from a DisclosureRegime object Args: @@ -84,13 +82,12 @@ def _get_regime_row(self, regime: DisclosureRegime) -> list: Returns: list: Description """ + logger.info(regime.title) row = [] - row.append("Implementation") - row.append("") row.append("") - row.append("") - row.append(regime.implementation_title) - # Implementation / regime fields + if is_single: + row.append("") + row.append(regime.title) row.append(regime.public_access_register_url) row.append(regime.display_scope) row.append(regime.display_register_launched) @@ -103,16 +100,14 @@ def _get_regime_row(self, regime: DisclosureRegime) -> list: def _tag_names_to_string(self, field): if isinstance(field, str): return field - return ' | '.join([tag.name for tag in field.all()]) + return " | ".join([tag.name for tag in field.all()]) class CountryExport(DataExportBase): - - """A class for exporting a country's data as CSV - """ + """A class for exporting a country's data as CSV""" def setup(self, request, *args, **kwargs): - self.slug = kwargs.pop('slug') + self.slug = kwargs.pop("slug") try: self.country = CountryTag.objects.get(slug=self.slug) except CountryTag.DoesNotExist as err: @@ -122,8 +117,8 @@ def setup(self, request, *args, **kwargs): def get(self, *args, **kwargs): # noqa: ARG002 response = HttpResponse( - content_type='text/csv', - headers={'Content-Disposition': f'attachment; filename="{self.slug}.csv"'}, + content_type="text/csv", + headers={"Content-Disposition": f'attachment; filename="{self.slug}.csv"'}, ) self._generate_csv(response) return response @@ -131,27 +126,28 @@ def get(self, *args, **kwargs): # noqa: ARG002 def _generate_csv(self, response: HttpResponse): writer = csv.writer(response) writer.writerow(COUNTRY_HEADERS) - row = ["", "", "", "", "", "", "", "", "", "", "", ""] + row = ["", "", "", "", "", "", "", "", "", ""] row[0] = self.country.name row[1] = self.country.category_display writer.writerow(row) - for commitment in self.country.commitments.all(): - writer.writerow(self._get_commitment_row(commitment)) + # NO MORE COMMITMENT ROWS! + # for commitment in self.country.commitments.all(): + # writer.writerow(self._get_commitment_row(commitment)) for regime in self.country.regimes.all(): if regime.display: - writer.writerow(self._get_regime_row(regime)) + row = self._get_regime_row(regime) + logger.info(row) + writer.writerow(row) return writer class CountriesExport(DataExportBase): - - """A class for exporting all countries' data as CSV - """ + """A class for exporting all countries' data as CSV""" def get(self, *args, **kwargs): # noqa: ARG002 response = HttpResponse( - content_type='text/csv', - headers={'Content-Disposition': 'attachment; filename="oo_all_country_data.csv"'}, + content_type="text/csv", + headers={"Content-Disposition": 'attachment; filename="oo_all_country_data.csv"'}, ) self._generate_csv(response) return response @@ -160,22 +156,29 @@ def _generate_csv(self, response: HttpResponse): writer = csv.writer(response) writer.writerow(ALL_HEADERS) for country in self._all_countries: - for commitment in country.commitments.all(): - row = [country.name] + self._get_commitment_row(commitment, skip_one=True) - # Replace "Commitment" Type with stage/category: - row[1] = country.category_display - writer.writerow(row) + # NO MORE COMMITMENT ROWS! + # for commitment in country.commitments.all(): + # row = [country.name] + self._get_commitment_row(commitment, skip_one=True) + # # Replace "Commitment" Type with stage/category: + # row[1] = country.category_display + # writer.writerow(row) for regime in country.regimes.all(): if regime.display: - row = [country.name] + self._get_regime_row(regime) - # Replace "Implementation" Type with stage/category: + row = [country.name] + self._get_regime_row(regime, is_single=False) row[1] = country.category_display writer.writerow(row) return writer - @cached_property def _all_countries(self): - countries = CountryTag.objects.exclude( - deleted=True, archived=True).order_by('name').all() + countries = CountryTag.objects.exclude(deleted=True, archived=True).order_by("name").all() return countries + + +def serve_csv_file(request): # noqa: ARG001 + file_path = Path(settings.STATIC_ROOT) / "files" / "metadata.csv" + if file_path.exists(): + return FileResponse(open(file_path, "rb"), content_type="text/csv") # noqa: PTH123, SIM115 + + msg = "CSV file does not exist" + raise Http404(msg) diff --git a/app/poetry.lock b/app/poetry.lock index 5c676c71..b328f74d 100644 --- a/app/poetry.lock +++ b/app/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "anyascii" @@ -1998,6 +1998,17 @@ files = [ [package.dependencies] wagtail = ">=4.0" +[[package]] +name = "wat-inspector" +version = "0.4.0" +description = "Deep inspection of Python objects" +optional = false +python-versions = ">=3.8" +files = [ + {file = "wat_inspector-0.4.0-py3-none-any.whl", hash = "sha256:2664ccc10ce22b6bc064be7316ea5c34d51fb1ff2d5f543c9ecef22226ad49dc"}, + {file = "wat_inspector-0.4.0.tar.gz", hash = "sha256:4262ecdf1fc2ea8e5c35ae65ca1ae18b37d30061936e8bd2f4d6b4020cc05ff3"}, +] + [[package]] name = "wcwidth" version = "0.2.6" @@ -2072,4 +2083,4 @@ dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"] [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.10" -content-hash = "e51765451e70b077bd0175895d28e20922a940b4a38155d2c88630b704b1de3c" +content-hash = "ae9078eb1443a9b6457bb9940ac1af9750a4ccf0092360f62255a5112d3893dc" diff --git a/app/pyproject.toml b/app/pyproject.toml index b5534514..b6ade06f 100644 --- a/app/pyproject.toml +++ b/app/pyproject.toml @@ -52,6 +52,7 @@ pytest-pythonpath = "*" pytest-sugar = "^0.9.7" codacy-coverage = "^1.3.11" py = "^1.11.0" +wat-inspector = "^0.4.0" [build-system] requires = ["poetry.core"] diff --git a/app/templates/content/map_page.jinja b/app/templates/content/map_page.jinja index f55fd820..139eab00 100644 --- a/app/templates/content/map_page.jinja +++ b/app/templates/content/map_page.jinja @@ -61,10 +61,11 @@ {% include "svg/filled-circle-icon.jinja" %} Countries where Open Ownership has engaged ({{ country_counts.engaged }}) - +
{% trans %}Download all country data{% endtrans %} + {% trans %}Download metadata{% endtrans %}
@@ -94,9 +95,9 @@
{% for country in region.countries.all().order_by('name') %} - + {{ country.name }} - + {% endfor %}
diff --git a/app/templates/views/country.jinja b/app/templates/views/country.jinja index 6ff2a6dd..1dc6b926 100644 --- a/app/templates/views/country.jinja +++ b/app/templates/views/country.jinja @@ -27,7 +27,7 @@

Records last updated: {{ country.last_updated|nicedate }}

- + {% if country.consultant %}

@@ -55,7 +55,7 @@ {% trans %}Committed to one or more{% endtrans %} - +

  • {% if country.category %} @@ -96,7 +96,7 @@
{% endif %} - + {% else %} @@ -137,6 +137,7 @@
@@ -231,9 +232,9 @@ {% if commitment.link %}link to commitment{% endif %} {% endfor %} - + - +