diff --git a/.github/workflows/python-source.yml b/.github/workflows/python-source.yml index 13514109d..f7717315b 100644 --- a/.github/workflows/python-source.yml +++ b/.github/workflows/python-source.yml @@ -25,7 +25,7 @@ jobs: OAR_DOCKERHUB_CRED: ${{ secrets.OAR_DOCKERHUB_CRED }} run: | bash scripts/dhsetup.sh - cd docker && bash ./dockbuild.sh pytest + cd docker && bash ./dockbuild.sh pdrtest - name: Run Unit Tests via Docker run: cd docker && ./testall -D python diff --git a/docker/pdrtest/Dockerfile b/docker/pdrtest/Dockerfile index 5aa7c295b..91a7e7e75 100644 --- a/docker/pdrtest/Dockerfile +++ b/docker/pdrtest/Dockerfile @@ -11,6 +11,10 @@ FROM oar-metadata/ejsonschema-py2 +ENV LANG=C.UTF-8 +COPY setdefenc.py /tmp/ +RUN cat /tmp/setdefenc.py >> /usr/lib/python2.7/sitecustomize.py + RUN apt-get update && apt-get install -y python-yaml nginx curl wget less sudo \ uwsgi uwsgi-plugin-python zip \ p7zip-full ca-certificates git @@ -20,9 +24,9 @@ COPY verify-asc.sh /usr/local/bin # install multibag from source RUN curl -L -o multibag-py.zip \ - https://github.com/usnistgov/multibag-py/archive/master.zip && \ + https://github.com/usnistgov/multibag-py/archive/0.6.zip && \ unzip -oq multibag-py.zip && \ - cd multibag-py-master && \ + cd multibag-py-0.6 && \ python setup.py install ENV GOSU_VERSION 1.14 diff --git a/docker/pdrtest/setdefenc.py b/docker/pdrtest/setdefenc.py new file mode 100644 index 000000000..1a94f31d5 --- /dev/null +++ b/docker/pdrtest/setdefenc.py @@ -0,0 +1,4 @@ + +import sys +sys.setdefaultencoding('utf8') + diff --git a/oar-metadata b/oar-metadata index 3930bc805..78b8bba5b 160000 --- a/oar-metadata +++ b/oar-metadata @@ -1 +1 @@ -Subproject commit 3930bc805364a670739b29423ad22305f75f0c56 +Subproject commit 78b8bba5bb573981487cf1f3620fbbe693ac8589 diff --git a/python/nistoar/pdr/preserv/bagger/midas3.py b/python/nistoar/pdr/preserv/bagger/midas3.py index 023529efd..65d66188c 100644 --- a/python/nistoar/pdr/preserv/bagger/midas3.py +++ b/python/nistoar/pdr/preserv/bagger/midas3.py @@ -28,7 +28,7 @@ from ....nerdm import utils as nerdutils from ... import def_merge_etcdir, utils, ARK_NAAN, PDR_PUBLIC_SERVER from .. import (SIPDirectoryError, SIPDirectoryNotFound, AIPValidationError, - ConfigurationException, StateException, PODError, + ConfigurationException, StateException, PODError, NERDError, PreservationStateError) from .... import pdr from .prepupd import UpdatePrepService @@ -324,7 +324,7 @@ def _filepaths_in_pod(self): pod = self._pod_rec() - return [self._distsvcurl.sub('', urllib.unquote(d['downloadURL'])) + return [self._distsvcurl.sub('', urllib.unquote(str(d['downloadURL']))) for d in pod.get('distribution',[]) if 'downloadURL' in d] diff --git a/python/nistoar/pdr/preserv/bagit/builder.py b/python/nistoar/pdr/preserv/bagit/builder.py index 078b99366..ea57d60d4 100644 --- a/python/nistoar/pdr/preserv/bagit/builder.py +++ b/python/nistoar/pdr/preserv/bagit/builder.py @@ -445,7 +445,7 @@ def _download_url(self, ediid, destpath): if ediid.startswith(arkpfx): # our convention is to omit the "ark:/88434/" prefix ediid = ediid[len(arkpfx):] - return self._distbase + ediid + '/' + urlencode(path) + return self._distbase + ediid + '/' + urlencode(str(path)) def assign_id(self, id, keep_conv=False): """ @@ -2495,7 +2495,7 @@ def _create_def_datafile_md(self, destpath): out = OrderedDict([ ("_schema", NERD_DEF + "Component"), ("@context", NERDM_CONTEXT), - ("@id", "cmps/" + urlencode(destpath)), + ("@id", "cmps/" + urlencode(str(destpath))), ("@type", deepcopy(self._comp_types["DataFile"][0])) ]) out["_extensionSchemas"] = deepcopy(self._comp_types["DataFile"][1]) @@ -2514,7 +2514,7 @@ def _create_def_chksum_md(self, destpath): out = OrderedDict([ ("_schema", NERD_DEF + "Component"), ("@context", NERDM_CONTEXT), - ("@id", "cmps/" + urlencode(destpath)), + ("@id", "cmps/" + urlencode(str(destpath))), ("@type", deepcopy(self._comp_types["ChecksumFile"][0])), ("filepath", destpath) ]) @@ -2543,7 +2543,7 @@ def _create_def_subcoll_md(self, destpath): out = OrderedDict([ ("_schema", NERD_DEF + "Component"), ("@context", NERDM_CONTEXT), - ("@id", "cmps/" + urlencode(destpath)), + ("@id", "cmps/" + urlencode(str(destpath))), ("@type", deepcopy(self._comp_types["Subcollection"][0])), ("_extensionSchemas", deepcopy(self._comp_types["Subcollection"][1])), ("filepath", destpath) diff --git a/python/nistoar/pdr/preserv/bagit/validate/base.py b/python/nistoar/pdr/preserv/bagit/validate/base.py index c98f68f52..bfc312690 100644 --- a/python/nistoar/pdr/preserv/bagit/validate/base.py +++ b/python/nistoar/pdr/preserv/bagit/validate/base.py @@ -1,6 +1,7 @@ """ This module provides the base validator class """ +import sys, traceback from abc import ABCMeta, abstractmethod, abstractproperty from collections import Sequence, OrderedDict @@ -11,6 +12,10 @@ PROB = 3 issuetypes = [ ERROR, WARN, REC ] +def _fmt_exc(): + return "".join(traceback.format_exception(*sys.exc_info())) + + class Validator(object): """ a class for validating a bag encapsulated in a directory. @@ -433,7 +438,7 @@ def validate(self, bag, want=ALL, results=None, *kw): out._err( ValidationIssue(self.profile[0], self.profile[1], "validator failure", ERROR, "test method, {0}, raised an exception: {1}" - .format(test, str(ex)), False), + .format(test, _fmt_exc()), False), False ) return out diff --git a/python/nistoar/pdr/preserv/bagit/validate/multibag.py b/python/nistoar/pdr/preserv/bagit/validate/multibag.py index 09b5db122..1ebd563f8 100644 --- a/python/nistoar/pdr/preserv/bagit/validate/multibag.py +++ b/python/nistoar/pdr/preserv/bagit/validate/multibag.py @@ -1,7 +1,8 @@ """ This module implements a validator for the Multibag Profile """ -import os, re +from __future__ import absolute_import +import os, re, io from collections import OrderedDict from urlparse import urlparse @@ -9,6 +10,8 @@ ERROR, WARN, REC, ALL, PROB) from ..bag import NISTBag +from multibag.constants import DEF_ENC + class MultibagValidator(ValidatorBase): """ A validator that runs tests for compliance with the Multibag Bagit Profile. @@ -243,7 +246,7 @@ def test_member_bags(self, bag, want=ALL, results=None): found = set() foundme = False last = None - with open(mbemf) as fd: + with io.open(mbemf, encoding=DEF_ENC) as fd: i = 0 for line in fd: i += 1 @@ -333,7 +336,7 @@ def test_file_lookup(self, bag, want=ALL, results=None, ishead=False): replicated = [] missing = [] paths = set() - with open(flirf) as fd: + with io.open(flirf, encoding=DEF_ENC) as fd: i = 0 for line in fd: i += 1 diff --git a/python/nistoar/pdr/publish/midas3/mdwsgi.py b/python/nistoar/pdr/publish/midas3/mdwsgi.py index db4187229..6d2e77856 100644 --- a/python/nistoar/pdr/publish/midas3/mdwsgi.py +++ b/python/nistoar/pdr/publish/midas3/mdwsgi.py @@ -6,7 +6,7 @@ This web service provides the public access to the metadata and the data files provided by the author to MIDAS. """ -import os, sys, logging, json, re +import os, sys, logging, json, re, urllib from wsgiref.headers import Headers from cgi import parse_qs, escape as escape_qp from collections import OrderedDict @@ -96,6 +96,7 @@ def __call__(self, env, start_resp): class Handler(object): badidre = re.compile(r"[<>\s]") + enc = "ISO-8859-1" def __init__(self, app, wsgienv, start_resp): self.app = app @@ -129,7 +130,8 @@ def add_header(self, name, value): # thus, this will raise a UnicodeEncodeError if the input strings # include Unicode (char code > 255). e = "ISO-8859-1" - self._hdr.add_header(name.encode(e), value.encode(e)) + onerr = "backslashreplace" + self._hdr.add_header(name.encode(e, onerr), value.encode(e, onerr)) def set_response(self, code, message): self._code = code @@ -410,10 +412,22 @@ def send_datafile(self, id, filepath): self.set_response(200, "Data file found") self.add_header('Content-Type', mtype) - self.add_header('Content-Disposition', - 'inline; filename="%s"' % os.path.basename(filepath)) + outname = os.path.basename(filepath) + try: + outname.encode("ISO-8859-1") + self.add_header('Content-Disposition', + 'inline; filename="%s"' % outname) + except UnicodeError: + outname = urllib.quote(outname.encode()) + self.add_header('Content-Disposition', + 'inline; filename*=UTF-8''"%s"' % outname) if xsend: - self.add_header('X-Accel-Redirect', xsend) + try: + xsend.encode("ISO-8859-1") + self.add_header('X-Accel-Redirect', xsend) + except UnicodeEncodeError: + xsend = urllib.quote(xsend.encode()) + self.add_header('X-Accel-Redirect', xsend) self.end_headers() if xsend: diff --git a/python/tests/nistoar/pdr/distrib/data/1491.1_0.mbag0_4-0.zip b/python/tests/nistoar/pdr/distrib/data/1491.1_0.mbag0_4-0.zip index b2940ca04..57b1d6194 100644 Binary files a/python/tests/nistoar/pdr/distrib/data/1491.1_0.mbag0_4-0.zip and b/python/tests/nistoar/pdr/distrib/data/1491.1_0.mbag0_4-0.zip differ diff --git a/python/tests/nistoar/pdr/preserv/bagger/test_midas.py b/python/tests/nistoar/pdr/preserv/bagger/test_midas.py index 7820c2001..4eed24869 100644 --- a/python/tests/nistoar/pdr/preserv/bagger/test_midas.py +++ b/python/tests/nistoar/pdr/preserv/bagger/test_midas.py @@ -492,7 +492,7 @@ def test_available_files(self): datafiles = self.bagr.available_files() self.assertIsInstance(datafiles, dict) - self.assertEqual(len(datafiles), 5) + self.assertEqual(len(datafiles), 6) self.assertIn("trial1.json", datafiles) self.assertIn("trial1.json.sha256", datafiles) self.assertIn("trial2.json", datafiles) @@ -505,7 +505,7 @@ def test_available_files(self): # copy of trial3a.json in upload overrides self.assertEqual(datafiles["trial3/trial3a.json"], os.path.join(uplsip, "trial3/trial3a.json")) - self.assertEqual(len(datafiles), 5) + self.assertEqual(len(datafiles), 6) def test_baggermd_file_for(self): self.bagr.ensure_base_bag() @@ -596,7 +596,7 @@ def test_ensure_data_files(self): self.bagr.ensure_data_files() self.assertIsNotNone(self.bagr.datafiles) - self.assertEqual(len(self.bagr.datafiles), 5) + self.assertEqual(len(self.bagr.datafiles), 6) self.assertEqual(len([d for d in self.bagr.datafiles.keys() if d.endswith(".sha256")]), 2) @@ -643,7 +643,7 @@ def test_registered_files(self): datafiles = self.bagr.registered_files() self.assertIsInstance(datafiles, dict) - self.assertEqual(len(datafiles), 5) + self.assertEqual(len(datafiles), 6) self.assertIn("trial1.json", datafiles) self.assertIn("trial1.json.sha256", datafiles) self.assertIn("trial2.json", datafiles) @@ -655,14 +655,14 @@ def test_registered_files(self): os.path.join(revsip, "trial2.json")) self.assertEqual(datafiles["trial3/trial3a.json"], os.path.join(revsip, "trial3/trial3a.json")) - self.assertEqual(len(datafiles), 5) + self.assertEqual(len(datafiles), 6) def test_available_files(self): revsip = os.path.join(self.revdir, self.midasid[32:]) datafiles = self.bagr.available_files() self.assertIsInstance(datafiles, dict) - self.assertEqual(len(datafiles), 5) + self.assertEqual(len(datafiles), 6) self.assertIn("trial1.json", datafiles) self.assertIn("trial1.json.sha256", datafiles) self.assertIn("trial2.json", datafiles) @@ -674,7 +674,7 @@ def test_available_files(self): os.path.join(revsip, "trial2.json")) self.assertEqual(datafiles["trial3/trial3a.json"], os.path.join(revsip, "trial3/trial3a.json")) - self.assertEqual(len(datafiles), 5) + self.assertEqual(len(datafiles), 6) def test_fileExaminer(self): # turn on asyncexamine (but turn off autolaunch so that we can test diff --git a/python/tests/nistoar/pdr/preserv/bagger/test_midas3.py b/python/tests/nistoar/pdr/preserv/bagger/test_midas3.py index d3de323d7..20b302660 100644 --- a/python/tests/nistoar/pdr/preserv/bagger/test_midas3.py +++ b/python/tests/nistoar/pdr/preserv/bagger/test_midas3.py @@ -107,7 +107,7 @@ def test_pod_rec(self): def test_available_files(self): datafiles = self.sip.available_files() self.assertIsInstance(datafiles, dict) - self.assertEqual(len(datafiles), 5) + self.assertEqual(len(datafiles), 6) self.assertIn("trial1.json", datafiles) self.assertIn("trial1.json.sha256", datafiles) self.assertIn("trial2.json", datafiles) @@ -120,7 +120,7 @@ def test_available_files(self): # copy of trial3a.json in upload overrides self.assertEqual(datafiles["trial3/trial3a.json"], os.path.join(self.sip.upldatadir, "trial3/trial3a.json")) - self.assertEqual(len(datafiles), 5) + self.assertEqual(len(datafiles), 6) def test_registered_files(self): pod = utils.read_json(os.path.join(self.revdir, "1491", "_pod.json")) @@ -130,7 +130,7 @@ def test_registered_files(self): datafiles = self.sip.registered_files() self.assertIsInstance(datafiles, dict) - self.assertEqual(len(datafiles), 4) + self.assertEqual(len(datafiles), 5) self.assertIn("trial1.json", datafiles) self.assertNotIn("trial1.json.sha256", datafiles) self.assertIn("trial2.json", datafiles) @@ -143,7 +143,7 @@ def test_registered_files(self): os.path.join(self.sip.revdatadir, "trial2.json")) self.assertEqual(datafiles["trial3/trial3a.json"], os.path.join(self.sip.revdatadir, "trial3/trial3a.json")) - self.assertEqual(len(datafiles), 4) + self.assertEqual(len(datafiles), 5) def test_fromPOD(self): podf = os.path.join(self.revdir, "1491", "_pod.json") @@ -775,7 +775,7 @@ def test_ensure_data_files(self): self.bagr.ensure_data_files(examine="sync") self.assertIsNotNone(self.bagr.datafiles) - self.assertEqual(len(self.bagr.datafiles), 5) + self.assertEqual(len(self.bagr.datafiles), 6) self.assertEqual(len([d for d in self.bagr.datafiles.keys() if d.endswith(".sha256")]), 2) @@ -836,14 +836,14 @@ def test_registered_files(self): os.path.join(revsip, "trial2.json")) self.assertEqual(datafiles["trial3/trial3a.json"], os.path.join(revsip, "trial3/trial3a.json")) - self.assertEqual(len(datafiles), 5) + self.assertEqual(len(datafiles), 6) def test_available_files(self): revsip = os.path.join(self.revdir, self.midasid[32:]) datafiles = self.bagr.sip.available_files() self.assertIsInstance(datafiles, dict) - self.assertEqual(len(datafiles), 5) + self.assertEqual(len(datafiles), 6) self.assertIn("trial1.json", datafiles) self.assertIn("trial1.json.sha256", datafiles) self.assertIn("trial2.json", datafiles) @@ -855,7 +855,7 @@ def test_available_files(self): os.path.join(revsip, "trial2.json")) self.assertEqual(datafiles["trial3/trial3a.json"], os.path.join(revsip, "trial3/trial3a.json")) - self.assertEqual(len(datafiles), 5) + self.assertEqual(len(datafiles), 6) def test_fileExaminer_autolaunch(self): # show that the async thread does its work with autolaunch diff --git a/python/tests/nistoar/pdr/preserv/bagit/validate/test_base.py b/python/tests/nistoar/pdr/preserv/bagit/validate/test_base.py index b65e98cb5..f989f302e 100644 --- a/python/tests/nistoar/pdr/preserv/bagit/validate/test_base.py +++ b/python/tests/nistoar/pdr/preserv/bagit/validate/test_base.py @@ -1,7 +1,12 @@ -import os, sys, pdb, json +import os, sys, pdb, json, re import unittest as test import nistoar.pdr.preserv.bagit.validate.base as base +import nistoar.pdr.preserv.bagit.bag as bag + +datadir = os.path.join( os.path.dirname(os.path.dirname( + os.path.dirname(__file__))), "data" ) +bagdir = os.path.join(datadir, "samplembag") class TestValidationIssue(test.TestCase): @@ -82,7 +87,51 @@ def test_description(self): self.assertEqual(issue.description, "ERROR: Life 3.1 A1.1: Life must self-replicate\n Little\n green") +class TestValidatorBase(test.TestCase): + + def setUp(self): + self.bag = bag.NISTBag(bagdir) + + class _TestBase(base.ValidatorBase): + def test_raise_except(self, bag, want=base.ALL, results=None): + out = results + if not out: + out = base.ValidationResults(bag.name, want) + + t = self._issue("re-goob", "raise an exception") + re.search(r"goob[g", "gurn") # a deliberate fail via exception + out._rec(t, True) + + return out + + def test_catch_except(self): + tests = self._TestBase({}) + try: + res = tests.test_raise_except(self.bag) + self.fail("failed to raise exception as necessary for test") + except Exception as ex: + pass + + res = tests.validate(self.bag) + self.assertEqual(res.count_applied(), 1) + self.assertEqual(res.count_failed(), 1) + self.assertIn("Traceback", res.failed()[0].description) + self.assertIn(", line ", res.failed()[0].description) + + + def test_fmt_exc(self): + a = {} + try: + a['hello'] + self.fail("failed to detect test KeyError") + except Exception as ex: + prob = base._fmt_exc() + self.assertIn('Traceback', prob) + self.assertIn(', line ', prob) + self.assertIn('KeyError', prob) + + if __name__ == '__main__': diff --git a/python/tests/nistoar/pdr/preserv/bagit/validate/test_multibag.py b/python/tests/nistoar/pdr/preserv/bagit/validate/test_multibag.py index 703788506..469c1e8ae 100644 --- a/python/tests/nistoar/pdr/preserv/bagit/validate/test_multibag.py +++ b/python/tests/nistoar/pdr/preserv/bagit/validate/test_multibag.py @@ -1,5 +1,5 @@ from __future__ import print_function -import os, sys, pdb, json, shutil, copy, re +import os, sys, pdb, json, shutil, copy, re, io import unittest as test from collections import OrderedDict @@ -10,6 +10,8 @@ import nistoar.pdr.preserv.bagit.exceptions as bagex import nistoar.pdr.exceptions as exceptions +from multibag.constants import DEF_ENC + datadir = os.path.join( os.path.dirname(os.path.dirname( os.path.dirname(__file__))), "data" ) bagdir = os.path.join(datadir, "samplembag") @@ -276,13 +278,14 @@ def test_test_member_bags(self): "False Positives: "+ str([str(e) for e in errs.failed()])) mbf = os.path.join(self.bag.multibag_dir, "member-bags.tsv") - with open(mbf, 'a') as fd: - print(self.bag.name, file=fd) - print("goober_bag\tbag.zip", file=fd) - print("goober_ bag\tbag.zip", file=fd) - print("goober_bag bag.zip", file=fd) - print("goober_bag bag.zip", file=fd) - print("gurn_bag\tbag.zip\tfoobar", file=fd) + with io.open(mbf, 'a', encoding=DEF_ENC) as fd: + print(self.bag.name.decode(DEF_ENC), file=fd) + print(u"goober_bag\tbag.zip", file=fd) + print(u"goober_ bag\tbag.zip", file=fd) + print(u"goober_bag bag.zip", file=fd) + print(u"goober_bag bag.zip", file=fd) + print(u"gurn_bag\tbag.zip\tfoobar", file=fd) + print(u"gurn\u03b1\tbag.7z\n", file=fd) errs = self.valid8.test_member_bags(self.bag) self.assertEqual(len(errs.failed()), 3, "Unexpected # of errors: [\n " + @@ -311,12 +314,13 @@ def test_test_file_lookup(self): "False Positives: "+ str([str(e) for e in errs.failed()])) flf = os.path.join(self.bag.multibag_dir, "file-lookup.tsv") - with open(flf, 'a') as fd: - print("data/goober.dat\t{0}".format(self.bag.name), file=fd) - print("data/goober.dat\totherbag", file=fd) - print("gurn.json", file=fd) - print("gurn.json\tbooger\tbonnet", file=fd) - print("data/trial1.json\totherbag", file=fd) + with io.open(flf, 'a', encoding=DEF_ENC) as fd: + print(u"data/goober.dat\t{0}".format(self.bag.name), file=fd) + print(u"data/goober.dat\totherbag", file=fd) + print(u"gurn.json", file=fd) + print(u"gurn.json\tbooger\tbonnet", file=fd) + print(u"data/trial1.json\totherbag", file=fd) + print(u"data/trial\u03b1.json\tother", file=fd) errs = self.valid8.test_file_lookup(self.bag) self.assertEqual(len(errs.failed()), 3, "Unexpected # of errors: [\n " + diff --git a/python/tests/nistoar/pdr/preserv/data/3A1EE2F169DD3B8CE0531A570681DB5D1491.json b/python/tests/nistoar/pdr/preserv/data/3A1EE2F169DD3B8CE0531A570681DB5D1491.json index ceb9e5178..4ec9f4bbd 100644 --- a/python/tests/nistoar/pdr/preserv/data/3A1EE2F169DD3B8CE0531A570681DB5D1491.json +++ b/python/tests/nistoar/pdr/preserv/data/3A1EE2F169DD3B8CE0531A570681DB5D1491.json @@ -203,6 +203,28 @@ ], "valid": false }, + { + "filepath": "trial3/trial3\u03b1.json", + "checksum": { + "hash": "7b58010c841b7748a48a7ac6366258d5b5a8d23d756951b6059c0e80daad516b", + "algorithm": { + "tag": "sha256", + "@type": "Thing" + } + }, + "mediaType": "application/json", + "downloadURL": "https://data.nist.gov/od/ds/3A1EE2F169DD3B8CE0531A570681DB5D1491/trial3/trial3%CE%B1.json", + "size": 70, + "@id": "cmps/trial3/trial3\u03b1.json", + "@type": [ + "nrdp:DataFile", + "nrdp:DownloadableFile", + "dcat:Distribution" + ], + "_extensionSchemas": [ + "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.2#/definitions/DataFile" + ] + }, { "description": "Simulation of experiment", "filepath": "sim++.json", @@ -328,4 +350,4 @@ "filepath": "sim++.json" } ] -} \ No newline at end of file +} diff --git a/python/tests/nistoar/pdr/preserv/data/midassip/review/1491/_pod.json b/python/tests/nistoar/pdr/preserv/data/midassip/review/1491/_pod.json index 4a0e0b08f..6ee6e12fd 100644 --- a/python/tests/nistoar/pdr/preserv/data/midassip/review/1491/_pod.json +++ b/python/tests/nistoar/pdr/preserv/data/midassip/review/1491/_pod.json @@ -36,6 +36,12 @@ "downloadURL": "https://data.nist.gov/od/ds/3A1EE2F169DD3B8CE0531A570681DB5D1491/trial3/trial3a.json.sha256", "mediaType": "text/plain" }, + { + "description": "Third trial of experiment", + "downloadURL": "https://data.nist.gov/od/ds/3A1EE2F169DD3B8CE0531A570681DB5D1491/trial3/trial3%CE%B1.json", + "mediaType": "application/json", + "title": "Trial 3-alpha: JSON version of the Mathematica notebook" + }, { "description": "Simulation of experiment", "downloadURL": "https://s3.amazonaws.com/nist-midas/1491/sim%2B%2B.json", diff --git "a/python/tests/nistoar/pdr/preserv/data/midassip/review/1491/trial3/trial3\316\261.json" "b/python/tests/nistoar/pdr/preserv/data/midassip/review/1491/trial3/trial3\316\261.json" new file mode 100644 index 000000000..eb50c4db6 --- /dev/null +++ "b/python/tests/nistoar/pdr/preserv/data/midassip/review/1491/trial3/trial3\316\261.json" @@ -0,0 +1,5 @@ +{ + "name": "tx3a", + "date": "2017-02-02", + "result": false +} diff --git a/python/tests/nistoar/pdr/preserv/service/test_siphandler_restricted.py b/python/tests/nistoar/pdr/preserv/service/test_siphandler_restricted.py index d32faa4a8..4128cf439 100644 --- a/python/tests/nistoar/pdr/preserv/service/test_siphandler_restricted.py +++ b/python/tests/nistoar/pdr/preserv/service/test_siphandler_restricted.py @@ -318,7 +318,7 @@ def test_update(self): nerdm = bag.nerdm_record() self.assertEqual(nerdm.get('accessLevel'), "restricted public") self.assertEqual(nerdm.get('disclaimer'), "Be careful.") - self.assertEqual(len(nerdm['components']), 9) + self.assertEqual(len(nerdm['components']), 10) self.assertEquals(nerdm['version'], "1.0.1") # serialize @@ -347,7 +347,7 @@ def test_update(self): nerdm = bag.nerdm_record() self.assertEqual(nerdm.get('accessLevel'), "restricted public") self.assertEqual(nerdm.get('disclaimer'), "Be careful.") - self.assertEqual(len(nerdm['components']), 9) + self.assertEqual(len(nerdm['components']), 10) self.assertEquals(nerdm['version'], "1.0.1") diff --git a/python/tests/nistoar/pdr/publish/midas3/test_mdwsgi.py b/python/tests/nistoar/pdr/publish/midas3/test_mdwsgi.py index ddbfd1080..d12bdce8a 100644 --- a/python/tests/nistoar/pdr/publish/midas3/test_mdwsgi.py +++ b/python/tests/nistoar/pdr/publish/midas3/test_mdwsgi.py @@ -116,7 +116,7 @@ def test_good_id(self): self.assertGreater(len([l for l in self.resp if "Content-Type:" in l]),0) data = json.loads(body[0]) self.assertEqual(data['ediid'], '3A1EE2F169DD3B8CE0531A570681DB5D1491') - self.assertEqual(len(data['components']), 8) + self.assertEqual(len(data['components']), 9) for cmp in data['components']: if 'downloadURL' in cmp: self.assertNotIn("/od/ds/", cmp['downloadURL']) @@ -172,6 +172,26 @@ def test_get_datafile2(self): self.assertGreater(len(redirect), 0) self.assertEqual(redirect[0],"X-Accel-Redirect: /midasdata/upload_dir/1491/trial3/trial3a.json") + def test_get_datafile_unicode(self): + req = { + 'PATH_INFO': '/3A1EE2F169DD3B8CE0531A570681DB5D1491/trial\xce\xb1.json', + 'REQUEST_METHOD': 'GET' + } +# body = self.svc(req, self.start) + hdlr = wsgi.Handler(self.svc, req, self.start) + body = hdlr.send_datafile('3A1EE2F169DD3B8CE0531A570681DB5D1491', + u"trial3/trial3\u03b1.json") + + self.assertGreater(len(self.resp), 0) + self.assertIn("200", self.resp[0]) + redirect = [r for r in self.resp if "X-Accel-Redirect:" in r] + self.assertGreater(len(redirect), 0) + self.assertEqual(redirect[0], + "X-Accel-Redirect: /midasdata/review_dir/1491/trial3/trial3%CE%B1.json") + mtype = [r for r in self.resp if "Content-Type:" in r] + self.assertGreater(len(mtype), 0) + self.assertEqual(mtype[0],"Content-Type: application/json") + def test_test_permission_read(self): hdlr = wsgi.Handler(self.svc, {}, self.start) body = hdlr.test_permission('mds2-2000', "read", "me") diff --git a/python/tests/nistoar/pdr/test_utils.py b/python/tests/nistoar/pdr/test_utils.py index e0006a93c..336b46d1c 100644 --- a/python/tests/nistoar/pdr/test_utils.py +++ b/python/tests/nistoar/pdr/test_utils.py @@ -8,6 +8,7 @@ testdatadir = os.path.join(testdir, 'data') testdatadir3 = os.path.join(testdir, 'preserv', 'data') testdatadir2 = os.path.join(testdatadir3, 'simplesip') +testdatadir4 = os.path.join(testdatadir3, 'midassip', 'review', '1491') loghdlr = None rootlog = None @@ -96,6 +97,8 @@ def test_checksum_of(self): self.assertEqual(utils.checksum_of(dfile), self.syssum(dfile)) dfile = os.path.join(testdatadir2,"trial3/trial3a.json") self.assertEqual(utils.checksum_of(dfile), self.syssum(dfile)) + dfile = os.path.join(testdatadir4,u"trial3/trial3\u03b1.json") + self.assertEqual(utils.checksum_of(dfile), self.syssum(dfile)) def syssum(self, filepath): cmd = ["sha256sum", filepath] @@ -285,6 +288,14 @@ def write_test_data(self): with open(self.testdata) as fd: data = json.load(fd) + def test_write_unicode_name(self): + data = utils.read_json(self.testdata) + data['foo'] = 'bar' + outf = self.tf(u"d\u03b1ta.json") + utils.write_json(data, outf) + data2 = utils.read_json(outf) + self.assertEqual(data2, data) + def test_writes(self): # this is not a definitive test that the use of LockedFile is working data = utils.read_json(self.testdata)