From d0fa7eeedcaae8c631e165a035ffc6ec4f805c22 Mon Sep 17 00:00:00 2001 From: "M. Sonntag" Date: Tue, 14 Nov 2017 16:07:09 +0100 Subject: [PATCH 01/23] [section] Add missing attributes to init --- odml/section.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/odml/section.py b/odml/section.py index fdefc315..3ae09ac1 100644 --- a/odml/section.py +++ b/odml/section.py @@ -25,12 +25,17 @@ class BaseSection(base.sectionable, mapping.mapableSection, Section): _format = format.Section - def __init__(self, name, type="undefined", parent=None, definition=None, mapping=None): + def __init__(self, name, type="undefined", parent=None, definition=None, + mapping=None, reference=None, repository=None, link=None, include=None): self._parent = parent self._name = name self._props = base.SmartList() self._definition = definition self._mapping = mapping + self._reference = reference + self._repository = repository + self._link = link + self._include = include super(BaseSection, self).__init__() # this may fire a change event, so have the section setup then self.type = type From 3e832056ee50ab7f5d1ab45b1071059b3f00a7eb Mon Sep 17 00:00:00 2001 From: "M. Sonntag" Date: Mon, 13 Nov 2017 16:52:46 +0100 Subject: [PATCH 02/23] [tools/odmlparser] Fix open JSON file bug --- odml/tools/odmlparser.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/odml/tools/odmlparser.py b/odml/tools/odmlparser.py index bdddbe31..966feac5 100644 --- a/odml/tools/odmlparser.py +++ b/odml/tools/odmlparser.py @@ -276,13 +276,13 @@ def from_file(self, file): return self.to_odml() elif self.parser == 'JSON': - try: - self.parsed_doc = json.load(file) - except json.decoder.JSONDecodeError as e: - print(e) - return - finally: - file.close() + with open(file) as json_data: + try: + self.parsed_doc = json.load(json_data) + except ValueError as e: # Python 2 does not support JSONDecodeError + print("JSON Decoder Error: %s" % e) + return + return self.to_odml() From cf60c4766a281bd650304d2ec6da8fda257d74f3 Mon Sep 17 00:00:00 2001 From: "M. Sonntag" Date: Mon, 13 Nov 2017 17:17:20 +0100 Subject: [PATCH 03/23] [tools/odmlparser] Fix open YAML file bug Closes #191. --- odml/tools/odmlparser.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/odml/tools/odmlparser.py b/odml/tools/odmlparser.py index 966feac5..667c7e03 100644 --- a/odml/tools/odmlparser.py +++ b/odml/tools/odmlparser.py @@ -150,8 +150,8 @@ class ODMLReader: based on the given data exchange format, like XML, YAML or JSON. Usage: - yaml_odml_doc = ODMLReader(parser='YAML').fromFile(open("odml_doc.yaml")) - json_odml_doc = ODMLReader(parser='JSON').fromFile(open("odml_doc.json")) + yaml_odml_doc = ODMLReader(parser='YAML').from_file("odml_doc.yaml") + json_odml_doc = ODMLReader(parser='JSON').from_file("odml_doc.json") """ def __init__(self, parser='XML'): @@ -266,13 +266,13 @@ def from_file(self, file): return odml_doc elif self.parser == 'YAML': - try: - self.parsed_doc = yaml.load(file) - except yaml.parser.ParserError as e: - print(e) - return - finally: - file.close() + with open(file) as yaml_data: + try: + self.parsed_doc = yaml.load(yaml_data) + except yaml.parser.ParserError as e: + print(e) + return + return self.to_odml() elif self.parser == 'JSON': From 52b094ff5b69b88b9f872f0ce298d155b371978b Mon Sep 17 00:00:00 2001 From: "M. Sonntag" Date: Mon, 13 Nov 2017 17:56:46 +0100 Subject: [PATCH 04/23] [tools/odmlparser] Fix YAML/JSON Reader.from_str --- odml/tools/odmlparser.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/odml/tools/odmlparser.py b/odml/tools/odmlparser.py index 667c7e03..26f8ea64 100644 --- a/odml/tools/odmlparser.py +++ b/odml/tools/odmlparser.py @@ -292,17 +292,19 @@ def from_string(self, string): odml_doc = xmlparser.XMLReader().fromString(string) self.doc = odml_doc return self.doc + elif self.parser == 'YAML': try: - odml_doc = yaml.load(string) + self.parsed_doc = yaml.load(string) except yaml.parser.ParserError as e: print(e) return return self.to_odml() + elif self.parser == 'JSON': try: - odml_doc = json.loads(string) - except json.decoder.JSONDecodeError as e: - print(e) + self.parsed_doc = json.loads(string) + except ValueError as e: # Python 2 does not support JSONDecodeError + print("JSON Decoder Error: %s" % e) return return self.to_odml() From ea3f263a40c3a316e9afb294cddca9dc477b65f2 Mon Sep 17 00:00:00 2001 From: "M. Sonntag" Date: Tue, 14 Nov 2017 14:10:52 +0100 Subject: [PATCH 05/23] [test/parser] Fix open YAML/JSON file bug --- test/test_parser.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/test_parser.py b/test/test_parser.py index 8f9156af..e593dc6f 100644 --- a/test/test_parser.py +++ b/test/test_parser.py @@ -38,14 +38,14 @@ def tearDown(self): def test_xml(self): self.xml_writer.write_file(self.odml_doc, self.xml_file) - xml_doc = self.xml_reader.from_file(open(self.xml_file)) + xml_doc = self.xml_reader.from_file(self.xml_file) self.assertEqual(xml_doc, self.odml_doc) def test_yaml(self): self.yaml_writer.write_file(self.odml_doc, self.yaml_file) - yaml_doc = self.yaml_reader.from_file(open(self.yaml_file)) + yaml_doc = self.yaml_reader.from_file(self.yaml_file) self.assertEqual(yaml_doc, self.odml_doc) @@ -53,7 +53,7 @@ def test_yaml(self): def test_json(self): self.json_writer.write_file(self.odml_doc, self.json_file) - json_doc = self.json_reader.from_file(open(self.json_file)) + json_doc = self.json_reader.from_file(self.json_file) self.assertEqual(json_doc, self.odml_doc) @@ -61,13 +61,13 @@ def test_json(self): def test_json_yaml_xml(self): self.json_writer.write_file(self.odml_doc, self.json_file) - json_doc = self.json_reader.from_file(open(self.json_file)) + json_doc = self.json_reader.from_file(self.json_file) self.yaml_writer.write_file(json_doc, self.yaml_file) - yaml_doc = self.yaml_reader.from_file(open(self.yaml_file)) + yaml_doc = self.yaml_reader.from_file(self.yaml_file) self.xml_writer.write_file(yaml_doc, self.xml_file) - xml_doc = self.xml_reader.from_file(open(self.xml_file)) + xml_doc = self.xml_reader.from_file(self.xml_file) self.assertEqual(json_doc, self.odml_doc) self.assertEqual(json_doc, yaml_doc) From e2be96f43e69c20482bcd51be3797447fa271719 Mon Sep 17 00:00:00 2001 From: "M. Sonntag" Date: Mon, 13 Nov 2017 16:47:38 +0100 Subject: [PATCH 06/23] [tools/odmlparser] Pep8 cleanup --- odml/tools/odmlparser.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/odml/tools/odmlparser.py b/odml/tools/odmlparser.py index 26f8ea64..52c66fe6 100644 --- a/odml/tools/odmlparser.py +++ b/odml/tools/odmlparser.py @@ -7,8 +7,8 @@ """ -import yaml import json +import yaml from .. import format from . import xmlparser @@ -20,13 +20,13 @@ class ODMLWriter: - ''' + """ A generic odML document writer, for XML, YAML and JSON. Usage: xml_writer = ODMLWriter(parser='XML') xml_writer.write_file(odml_document, filepath) - ''' + """ def __init__(self, parser='XML'): self.doc = None # odML document @@ -55,7 +55,6 @@ def to_dict(self, odml_document): self.parsed_doc = parsed_doc def get_sections(self, section_list): - section_seq = [] for section in section_list: @@ -81,7 +80,6 @@ def get_sections(self, section_list): return section_seq def get_properties(self, props_list): - props_seq = [] for prop in props_list: @@ -100,7 +98,7 @@ def get_properties(self, props_list): prop_dict[attr] = t props_seq.append(prop_dict) - + return props_seq def get_values(self, value_list): @@ -172,7 +170,6 @@ def is_valid_attribute(self, attr, fmt): return None def to_odml(self): - self.odml_version = self.parsed_doc.get('odml-version', odml_version) self.parsed_doc = self.parsed_doc['Document'] @@ -193,7 +190,6 @@ def to_odml(self): return self.doc def parse_sections(self, section_list): - odml_sections = [] for section in section_list: @@ -218,7 +214,6 @@ def parse_sections(self, section_list): return odml_sections - def parse_properties(self, props_list): odml_props = [] @@ -257,7 +252,6 @@ def parse_values(self, value_list): return odml_values - def from_file(self, file): if self.parser == 'XML' or self.parser == 'ODML': @@ -285,7 +279,6 @@ def from_file(self, file): return self.to_odml() - def from_string(self, string): if self.parser == 'XML' or self.parser == 'ODML': From d2550f6d9376c9efe5c9a149d0ea662f4f29c307 Mon Sep 17 00:00:00 2001 From: "M. Sonntag" Date: Wed, 6 Dec 2017 17:15:48 +0100 Subject: [PATCH 07/23] [init] Update version number --- odml/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/odml/__init__.py b/odml/__init__.py index ebc41066..e5fb599e 100644 --- a/odml/__init__.py +++ b/odml/__init__.py @@ -8,7 +8,7 @@ from .tools.odmlparser import allowed_parsers as parsers -__version__ = '1.3.2' +__version__ = '1.3.3' # the original property-function is overwritten # so get it back! From 00c995be7e85f8e58cf0018afd6c0b8f8c4cbb01 Mon Sep 17 00:00:00 2001 From: "M. Sonntag" Date: Wed, 6 Dec 2017 17:24:49 +0100 Subject: [PATCH 08/23] [info] Add info file to provide version and format --- odml/info.py | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 odml/info.py diff --git a/odml/info.py b/odml/info.py new file mode 100644 index 00000000..e6a9f827 --- /dev/null +++ b/odml/info.py @@ -0,0 +1,2 @@ +VERSION = '1.3.3' +FORMAT_VERSION = '1' From 88fc8c9b346aff3a60b005d526c3e6868ef396d9 Mon Sep 17 00:00:00 2001 From: "M. Sonntag" Date: Wed, 6 Dec 2017 17:25:46 +0100 Subject: [PATCH 09/23] [init] User version number from info --- odml/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/odml/__init__.py b/odml/__init__.py index e5fb599e..84e5b2c5 100644 --- a/odml/__init__.py +++ b/odml/__init__.py @@ -5,10 +5,10 @@ from . import section from .dtypes import DType from .fileio import load, save, display +from .info import VERSION from .tools.odmlparser import allowed_parsers as parsers - -__version__ = '1.3.3' +__version__ = VERSION # the original property-function is overwritten # so get it back! From 1820178fc33195d84def4de619dd6698719aab8e Mon Sep 17 00:00:00 2001 From: "M. Sonntag" Date: Wed, 6 Dec 2017 17:26:03 +0100 Subject: [PATCH 10/23] [odmlparser] Use format version number from info --- odml/tools/odmlparser.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/odml/tools/odmlparser.py b/odml/tools/odmlparser.py index 52c66fe6..125c22db 100644 --- a/odml/tools/odmlparser.py +++ b/odml/tools/odmlparser.py @@ -9,13 +9,11 @@ import json import yaml + from .. import format +from ..info import FORMAT_VERSION from . import xmlparser -# FIX ME: Version should not be hardcoded here. Import from odML module after -# fixing the circular imports issue. -odml_version = '1' - allowed_parsers = ['ODML', 'XML', 'YAML', 'JSON'] @@ -133,7 +131,7 @@ def to_string(self, odml_document): self.to_dict(odml_document) odml_output = {} odml_output['Document'] = self.parsed_doc - odml_output['odml-version'] = odml_version + odml_output['odml-version'] = FORMAT_VERSION if self.parser == 'YAML': string_doc = yaml.dump(odml_output, default_flow_style=False) @@ -170,7 +168,7 @@ def is_valid_attribute(self, attr, fmt): return None def to_odml(self): - self.odml_version = self.parsed_doc.get('odml-version', odml_version) + self.odml_version = self.parsed_doc.get('odml-version', FORMAT_VERSION) self.parsed_doc = self.parsed_doc['Document'] doc_attrs = {} From 7d61eac5b08d74fa52f96c0b9949e99d2a7d7268 Mon Sep 17 00:00:00 2001 From: "M. Sonntag" Date: Wed, 6 Dec 2017 18:47:45 +0100 Subject: [PATCH 11/23] [xmlparser] Use format version from info --- odml/tools/xmlparser.py | 5 ++--- test/test_samplefile.py | 5 +++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/odml/tools/xmlparser.py b/odml/tools/xmlparser.py index 8a34026a..3f26dc5a 100644 --- a/odml/tools/xmlparser.py +++ b/odml/tools/xmlparser.py @@ -13,6 +13,7 @@ from lxml.builder import E # this is needed for py2exe to include lxml completely from lxml import _elementpath as _dummy +from ..info import FORMAT_VERSION try: unicode = unicode @@ -35,8 +36,6 @@ format.Value._xml_attributes = {} format.Value._xml_content = 'value' -XML_VERSION = "1" - class XMLWriter: """ @@ -66,7 +65,7 @@ def save_element(e): # generate attributes if isinstance(fmt, format.Document.__class__): - cur.attrib['version'] = XML_VERSION + cur.attrib['version'] = FORMAT_VERSION for k, v in fmt._xml_attributes.items(): if not v or not hasattr(e, fmt.map(v)): diff --git a/test/test_samplefile.py b/test/test_samplefile.py index 9b8f547e..79405e3a 100644 --- a/test/test_samplefile.py +++ b/test/test_samplefile.py @@ -5,6 +5,7 @@ import sys import re +from odml.info import FORMAT_VERSION from odml.tools import xmlparser from odml.tools import jsonparser from odml.tools import dumper @@ -141,10 +142,10 @@ def test_xml_writer_version(self): val = unicode(xmlparser.XMLWriter(doc)) else: val = str(xmlparser.XMLWriter(doc)) - self.assertIn('version="%s"' % xmlparser.XML_VERSION, val) + self.assertIn('version="%s"' % FORMAT_VERSION, val) doc = xmlparser.XMLReader().fromString(val) # this test is switched off until the XML versioning support is implemented - # self.assertEqual(doc._xml_version, xmlparser.XML_VERSION) + # self.assertEqual(doc._xml_version, FORMAT_VERSION) def test_save(self): for module in [xmlparser.XMLWriter, jsonparser.JSONWriter]: From 9e0c50a4fb7cd894b5327a459b9167afc344f105 Mon Sep 17 00:00:00 2001 From: "M. Sonntag" Date: Wed, 6 Dec 2017 19:16:30 +0100 Subject: [PATCH 12/23] [xmlparser] Add odml format version check --- odml/tools/xmlparser.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/odml/tools/xmlparser.py b/odml/tools/xmlparser.py index 3f26dc5a..42a82b76 100644 --- a/odml/tools/xmlparser.py +++ b/odml/tools/xmlparser.py @@ -149,6 +149,26 @@ def __init__(self, ignore_errors=False, filename=None): self.ignore_errors = ignore_errors self.filename = filename + @staticmethod + def _handle_version(root): + """ + Check if the odML version of a handed in parsed lxml.etree is supported + by the current library and raise an Exception otherwise. + :param root: Root node of a parsed lxml.etree. Teh root tag has to + contain a supported odML version number, otherwise it is not + accepted as a valid odML file. + """ + if root.tag != 'odML': + raise ParserException("Expecting start tag but got <%s>.\n" % root.tag) + elif 'version' not in root.attrib: + raise ParserException("Could not find format version attribute " + "in odML start tag.\n") + elif root.attrib['version'] != FORMAT_VERSION: + msg = ("Invalid odML document format version '%s'. " + "Supported versions: '%s'." + % (root.attrib['version'], FORMAT_VERSION)) + raise ParserException(msg) + def fromFile(self, xml_file): """ parse the datastream from a file like object *xml_file* @@ -158,6 +178,8 @@ def fromFile(self, xml_file): root = ET.parse(xml_file, self.parser).getroot() except ET.XMLSyntaxError as e: raise ParserException(e.msg) + + self._handle_version(root) return self.parse_element(root) def fromString(self, string): @@ -165,6 +187,8 @@ def fromString(self, string): root = ET.XML(string, self.parser) except ET.XMLSyntaxError as e: raise ParserException(e.msg) + + self._handle_version(root) return self.parse_element(root) def check_mandatory_arguments(self, data, ArgClass, tag_name, node): From 713e72048eacc6573aabdd37be454d6751ad3f25 Mon Sep 17 00:00:00 2001 From: "M. Sonntag" Date: Wed, 6 Dec 2017 19:25:53 +0100 Subject: [PATCH 13/23] [odmlparser] Add odml format version check --- odml/tools/odmlparser.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/odml/tools/odmlparser.py b/odml/tools/odmlparser.py index 125c22db..9d52add1 100644 --- a/odml/tools/odmlparser.py +++ b/odml/tools/odmlparser.py @@ -17,6 +17,10 @@ allowed_parsers = ['ODML', 'XML', 'YAML', 'JSON'] +class ParserException(Exception): + pass + + class ODMLWriter: """ A generic odML document writer, for XML, YAML and JSON. @@ -168,7 +172,16 @@ def is_valid_attribute(self, attr, fmt): return None def to_odml(self): - self.odml_version = self.parsed_doc.get('odml-version', FORMAT_VERSION) + # Parse only odML documents of supported format versions. + if 'odml-version' not in self.parsed_doc: + raise ParserException("Invalid odML document: Could not find odml-version.") + elif self.parsed_doc.get('odml-version') != FORMAT_VERSION: + msg = ("Invalid odML document format version '%s'. " + "Supported versions: '%s'." + % (self.parsed_doc.get('odml-version'), FORMAT_VERSION)) + raise ParserException(msg) + + self.odml_version = self.parsed_doc.get('odml-version') self.parsed_doc = self.parsed_doc['Document'] doc_attrs = {} From c9c4dd627183754f30343c12aa5b053d20bf2b0d Mon Sep 17 00:00:00 2001 From: "M. Sonntag" Date: Thu, 7 Dec 2017 11:23:13 +0100 Subject: [PATCH 14/23] [init/setup] Update packaging information Added author list, contact, classifiers, license text and updated the provided hompage for future packaging. --- odml/info.py | 13 +++++++++++++ setup.py | 14 ++++++++++---- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/odml/info.py b/odml/info.py index e6a9f827..a1a6267f 100644 --- a/odml/info.py +++ b/odml/info.py @@ -1,2 +1,15 @@ VERSION = '1.3.3' FORMAT_VERSION = '1' +AUTHOR = 'Hagen Fritsch, Christian Kellner, Jan Grewe, ' \ + 'Achilleas Koutsou, Michael Sonntag, Lyuba Zehl' +COPYRIGHT = '(c) 2011-2017, German Neuroinformatics Node' +CONTACT = 'dev@g-node.org' +HOMEPAGE = 'https://github.com/G-Node/python-odml' +CLASSIFIERS = [ + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 3', + 'License :: OSI Approved :: BSD License', + 'Development Status :: 5 - Production/Stable', + 'Topic :: Scientific/Engineering', + 'Intended Audience :: Science/Research' +] diff --git a/setup.py b/setup.py index 16271c87..ccfcdf51 100644 --- a/setup.py +++ b/setup.py @@ -3,6 +3,7 @@ from setuptools import setup except ImportError as ex: from distutils.core import setup +from odml.info import AUTHOR, CONTACT, CLASSIFIERS, HOMEPAGE, VERSION packages = [ 'odml', @@ -12,19 +13,24 @@ with open('README.rst') as f: description_text = f.read() +with open("LICENSE") as f: + license_text = f.read() + install_req = ["lxml", "pyyaml"] if sys.version_info < (3, 4): install_req += ["enum34"] setup( name='odML', - version='1.3.2', + version=VERSION, description='open metadata Markup Language', - author='G-Node', - author_email='dev@g-node.org', - url='http://www.g-node.org/projects/odml', + author=AUTHOR, + author_email=CONTACT, + url=HOMEPAGE, packages=packages, test_suite='test', install_requires=install_req, long_description=description_text, + classifiers=CLASSIFIERS, + license=license_text ) From 577b8ff9e4163f69e06b2e92fc2f51f8431a565d Mon Sep 17 00:00:00 2001 From: "M. Sonntag" Date: Thu, 7 Dec 2017 14:27:04 +0100 Subject: [PATCH 15/23] [odmlparser] Save only valid odML documents --- odml/tools/odmlparser.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/odml/tools/odmlparser.py b/odml/tools/odmlparser.py index 9d52add1..60f4dfec 100644 --- a/odml/tools/odmlparser.py +++ b/odml/tools/odmlparser.py @@ -122,6 +122,17 @@ def get_values(self, value_list): return value_seq def write_file(self, odml_document, filename): + # Write document only if it does not contain validation errors. + from ..validation import Validation # disgusting import problems + validation = Validation(odml_document) + msg = "" + for e in validation.errors: + if e.is_error: + msg += "\n\t- %s %s: %s" % (e.obj, e.type, e.msg) + if msg != "": + msg = "Resolve document validation errors before saving %s" % msg + raise ParserException(msg) + file = open(filename, 'w') file.write(self.to_string(odml_document)) file.close() From 9b7baac08cf3c7815beb1759e24c08750a0728ad Mon Sep 17 00:00:00 2001 From: "M. Sonntag" Date: Thu, 7 Dec 2017 15:25:35 +0100 Subject: [PATCH 16/23] [odmlparser] Use xmlparser with ignore_errors For now when xmlparser is used from odmlparser, all unsupported or missing element errors are warnings to allow parsing of partially invalid odML documents. --- odml/tools/odmlparser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/odml/tools/odmlparser.py b/odml/tools/odmlparser.py index 60f4dfec..afca9a21 100644 --- a/odml/tools/odmlparser.py +++ b/odml/tools/odmlparser.py @@ -277,7 +277,7 @@ def parse_values(self, value_list): def from_file(self, file): if self.parser == 'XML' or self.parser == 'ODML': - odml_doc = xmlparser.XMLReader().fromFile(file) + odml_doc = xmlparser.XMLReader(ignore_errors=True).fromFile(file) self.doc = odml_doc return odml_doc From 5f29fcdc98e06e0de482a81dfaea73b6510c2617 Mon Sep 17 00:00:00 2001 From: "M. Sonntag" Date: Thu, 7 Dec 2017 15:43:15 +0100 Subject: [PATCH 17/23] [xmlparser] Add parser warning list --- odml/tools/xmlparser.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/odml/tools/xmlparser.py b/odml/tools/xmlparser.py index 42a82b76..dae594f7 100644 --- a/odml/tools/xmlparser.py +++ b/odml/tools/xmlparser.py @@ -6,7 +6,6 @@ python -m odml.tools.xmlparser file.odml """ -#TODO make this module a parser class, allow arguments (e.g. skip_errors=1 to parse even broken documents) import sys from odml import format from lxml import etree as ET @@ -148,6 +147,7 @@ def __init__(self, ignore_errors=False, filename=None): self.tags = dict([(obj._xml_name, obj) for obj in format.__all__]) self.ignore_errors = ignore_errors self.filename = filename + self.warnings = [] @staticmethod def _handle_version(root): @@ -212,6 +212,7 @@ def warn(self, msg, elem): msg = "warning[%s:%d:<%s>]: %s\n" % (self.filename, elem.sourceline, elem.tag, msg) else: msg = "warning: %s\n" % msg + self.warnings.append(msg) sys.stderr.write(msg) def parse_element(self, node): From 5d2701b48b66e246c23b7a9057c995f8bfc578b0 Mon Sep 17 00:00:00 2001 From: "M. Sonntag" Date: Thu, 7 Dec 2017 15:43:31 +0100 Subject: [PATCH 18/23] [odmlparser] Add parser warning list --- odml/tools/odmlparser.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/odml/tools/odmlparser.py b/odml/tools/odmlparser.py index afca9a21..2a7c0ccd 100644 --- a/odml/tools/odmlparser.py +++ b/odml/tools/odmlparser.py @@ -173,13 +173,16 @@ def __init__(self, parser='XML'): if parser not in allowed_parsers: raise NotImplementedError("'%s' odML parser does not exist!" % parser) self.parser = parser + self.warnings = [] def is_valid_attribute(self, attr, fmt): if attr in fmt._args: return attr if fmt.revmap(attr): return attr - print("Invalid element <%s> inside <%s> tag" % (attr, fmt.__class__.__name__)) + msg = "Invalid element <%s> inside <%s> tag" % (attr, fmt.__class__.__name__) + print(msg) + self.warnings.append(msg) return None def to_odml(self): @@ -277,7 +280,9 @@ def parse_values(self, value_list): def from_file(self, file): if self.parser == 'XML' or self.parser == 'ODML': - odml_doc = xmlparser.XMLReader(ignore_errors=True).fromFile(file) + par = xmlparser.XMLReader(ignore_errors=True) + self.warnings = par.warnings + odml_doc = par.fromFile(file) self.doc = odml_doc return odml_doc From 64a91d555d6e62d5c0e9c23ec65f617ed2e5ebfa Mon Sep 17 00:00:00 2001 From: "M. Sonntag" Date: Thu, 7 Dec 2017 17:04:54 +0100 Subject: [PATCH 19/23] [travis] Use master branch travis file --- .travis.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index e64716e4..80b17400 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,13 +14,10 @@ matrix: - python: "3.4" - python: "3.5" -before_install: - - sudo apt-get -qq update - - sudo apt-get install -y python-enum python-lxml - install: - export PYVER=${TRAVIS_PYTHON_VERSION:0:1} - pip install --upgrade coveralls + - pip install lxml enum34 pyyaml script: - python setup.py build From 24150553fd5b2431c9e5ec5a19a3fdfc2e9065ed Mon Sep 17 00:00:00 2001 From: "M. Sonntag" Date: Mon, 11 Dec 2017 14:58:01 +0100 Subject: [PATCH 20/23] [parser] Update invalid odML format version text --- odml/tools/odmlparser.py | 4 ++-- odml/tools/xmlparser.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/odml/tools/odmlparser.py b/odml/tools/odmlparser.py index 2a7c0ccd..52b9971b 100644 --- a/odml/tools/odmlparser.py +++ b/odml/tools/odmlparser.py @@ -190,8 +190,8 @@ def to_odml(self): if 'odml-version' not in self.parsed_doc: raise ParserException("Invalid odML document: Could not find odml-version.") elif self.parsed_doc.get('odml-version') != FORMAT_VERSION: - msg = ("Invalid odML document format version '%s'. " - "Supported versions: '%s'." + msg = ("Cannot read file: invalid odML document format version '%s'. \n" + "This package supports odML format versions: '%s'." % (self.parsed_doc.get('odml-version'), FORMAT_VERSION)) raise ParserException(msg) diff --git a/odml/tools/xmlparser.py b/odml/tools/xmlparser.py index dae594f7..28f935d9 100644 --- a/odml/tools/xmlparser.py +++ b/odml/tools/xmlparser.py @@ -164,8 +164,8 @@ def _handle_version(root): raise ParserException("Could not find format version attribute " "in odML start tag.\n") elif root.attrib['version'] != FORMAT_VERSION: - msg = ("Invalid odML document format version '%s'. " - "Supported versions: '%s'." + msg = ("Cannot read file: invalid odML document format version '%s'. \n" + "This package supports odML format versions: '%s'." % (root.attrib['version'], FORMAT_VERSION)) raise ParserException(msg) From f45e9bccfab55ce2092bb1d2152c30c6b420eade Mon Sep 17 00:00:00 2001 From: "M. Sonntag" Date: Tue, 12 Dec 2017 15:43:53 +0100 Subject: [PATCH 21/23] [fileio] Define parsers only in odmlparser --- odml/fileio.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/odml/fileio.py b/odml/fileio.py index be2a3d1a..65db69a8 100644 --- a/odml/fileio.py +++ b/odml/fileio.py @@ -1,6 +1,6 @@ -from .tools.odmlparser import ODMLReader, ODMLWriter +from .tools.odmlparser import ODMLReader, ODMLWriter, allowed_parsers -parsers = ["xml", "json", "yaml"] +PARSERS = allowed_parsers def load(filename, backend="xml"): From c8e6157a30e22e77a2737bb7915945b4a58e5b33 Mon Sep 17 00:00:00 2001 From: "M. Sonntag" Date: Tue, 12 Dec 2017 15:44:05 +0100 Subject: [PATCH 22/23] [fileio] Add docstrings --- odml/fileio.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/odml/fileio.py b/odml/fileio.py index 65db69a8..1ce29cc9 100644 --- a/odml/fileio.py +++ b/odml/fileio.py @@ -4,15 +4,38 @@ def load(filename, backend="xml"): + """ + Load an odML document from file. + :param filename: Path and filename from where the odML document + is to be loaded and parsed. + :param backend: File format of the file containing the odML document. + The default format is XML. + :return: The parsed odML document. + """ reader = ODMLReader(backend) return reader.from_file(filename) def save(obj, filename, backend="xml"): + """ + Save an open odML document to file of a specified format. + :param obj: odML document do be saved. + :param filename: Filename and path where the odML document + should be saved. + :param backend: Format in which the odML document is to be saved. + The default format is XML. + """ writer = ODMLWriter(backend) return writer.write_file(obj, filename) def display(obj, backend="xml"): + """ + Print an open odML document to the command line, formatted in the + specified format. + :param obj: odML document to be displayed. + :param backend: Format in which the odML document is to be displayed. + The default format is XML. + """ writer = ODMLWriter(backend) print(writer.to_string(obj)) From c2ae4b50c33ec2b0b0139266abb5fe6bc1a5e472 Mon Sep 17 00:00:00 2001 From: "M. Sonntag" Date: Tue, 12 Dec 2017 16:47:35 +0100 Subject: [PATCH 23/23] [xmlparser] Fixing typos --- odml/tools/xmlparser.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/odml/tools/xmlparser.py b/odml/tools/xmlparser.py index 28f935d9..09586c3c 100644 --- a/odml/tools/xmlparser.py +++ b/odml/tools/xmlparser.py @@ -154,15 +154,15 @@ def _handle_version(root): """ Check if the odML version of a handed in parsed lxml.etree is supported by the current library and raise an Exception otherwise. - :param root: Root node of a parsed lxml.etree. Teh root tag has to + :param root: Root node of a parsed lxml.etree. The root tag has to contain a supported odML version number, otherwise it is not accepted as a valid odML file. """ if root.tag != 'odML': - raise ParserException("Expecting start tag but got <%s>.\n" % root.tag) + raise ParserException("Expecting tag but got <%s>.\n" % root.tag) elif 'version' not in root.attrib: raise ParserException("Could not find format version attribute " - "in odML start tag.\n") + "in tag.\n") elif root.attrib['version'] != FORMAT_VERSION: msg = ("Cannot read file: invalid odML document format version '%s'. \n" "This package supports odML format versions: '%s'."