diff --git a/.travis.yml b/.travis.yml index e64716e4..80b17400 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,13 +14,10 @@ matrix: - python: "3.4" - python: "3.5" -before_install: - - sudo apt-get -qq update - - sudo apt-get install -y python-enum python-lxml - install: - export PYVER=${TRAVIS_PYTHON_VERSION:0:1} - pip install --upgrade coveralls + - pip install lxml enum34 pyyaml script: - python setup.py build diff --git a/odml/__init__.py b/odml/__init__.py index ebc41066..84e5b2c5 100644 --- a/odml/__init__.py +++ b/odml/__init__.py @@ -5,10 +5,10 @@ from . import section from .dtypes import DType from .fileio import load, save, display +from .info import VERSION from .tools.odmlparser import allowed_parsers as parsers - -__version__ = '1.3.2' +__version__ = VERSION # the original property-function is overwritten # so get it back! diff --git a/odml/fileio.py b/odml/fileio.py index be2a3d1a..1ce29cc9 100644 --- a/odml/fileio.py +++ b/odml/fileio.py @@ -1,18 +1,41 @@ -from .tools.odmlparser import ODMLReader, ODMLWriter +from .tools.odmlparser import ODMLReader, ODMLWriter, allowed_parsers -parsers = ["xml", "json", "yaml"] +PARSERS = allowed_parsers def load(filename, backend="xml"): + """ + Load an odML document from file. + :param filename: Path and filename from where the odML document + is to be loaded and parsed. + :param backend: File format of the file containing the odML document. + The default format is XML. + :return: The parsed odML document. + """ reader = ODMLReader(backend) return reader.from_file(filename) def save(obj, filename, backend="xml"): + """ + Save an open odML document to file of a specified format. + :param obj: odML document do be saved. + :param filename: Filename and path where the odML document + should be saved. + :param backend: Format in which the odML document is to be saved. + The default format is XML. + """ writer = ODMLWriter(backend) return writer.write_file(obj, filename) def display(obj, backend="xml"): + """ + Print an open odML document to the command line, formatted in the + specified format. + :param obj: odML document to be displayed. + :param backend: Format in which the odML document is to be displayed. + The default format is XML. + """ writer = ODMLWriter(backend) print(writer.to_string(obj)) diff --git a/odml/info.py b/odml/info.py new file mode 100644 index 00000000..a1a6267f --- /dev/null +++ b/odml/info.py @@ -0,0 +1,15 @@ +VERSION = '1.3.3' +FORMAT_VERSION = '1' +AUTHOR = 'Hagen Fritsch, Christian Kellner, Jan Grewe, ' \ + 'Achilleas Koutsou, Michael Sonntag, Lyuba Zehl' +COPYRIGHT = '(c) 2011-2017, German Neuroinformatics Node' +CONTACT = 'dev@g-node.org' +HOMEPAGE = 'https://github.com/G-Node/python-odml' +CLASSIFIERS = [ + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 3', + 'License :: OSI Approved :: BSD License', + 'Development Status :: 5 - Production/Stable', + 'Topic :: Scientific/Engineering', + 'Intended Audience :: Science/Research' +] diff --git a/odml/section.py b/odml/section.py index fdefc315..3ae09ac1 100644 --- a/odml/section.py +++ b/odml/section.py @@ -25,12 +25,17 @@ class BaseSection(base.sectionable, mapping.mapableSection, Section): _format = format.Section - def __init__(self, name, type="undefined", parent=None, definition=None, mapping=None): + def __init__(self, name, type="undefined", parent=None, definition=None, + mapping=None, reference=None, repository=None, link=None, include=None): self._parent = parent self._name = name self._props = base.SmartList() self._definition = definition self._mapping = mapping + self._reference = reference + self._repository = repository + self._link = link + self._include = include super(BaseSection, self).__init__() # this may fire a change event, so have the section setup then self.type = type diff --git a/odml/tools/odmlparser.py b/odml/tools/odmlparser.py index bdddbe31..52b9971b 100644 --- a/odml/tools/odmlparser.py +++ b/odml/tools/odmlparser.py @@ -7,26 +7,28 @@ """ -import yaml import json +import yaml + from .. import format +from ..info import FORMAT_VERSION from . import xmlparser -# FIX ME: Version should not be hardcoded here. Import from odML module after -# fixing the circular imports issue. -odml_version = '1' - allowed_parsers = ['ODML', 'XML', 'YAML', 'JSON'] +class ParserException(Exception): + pass + + class ODMLWriter: - ''' + """ A generic odML document writer, for XML, YAML and JSON. Usage: xml_writer = ODMLWriter(parser='XML') xml_writer.write_file(odml_document, filepath) - ''' + """ def __init__(self, parser='XML'): self.doc = None # odML document @@ -55,7 +57,6 @@ def to_dict(self, odml_document): self.parsed_doc = parsed_doc def get_sections(self, section_list): - section_seq = [] for section in section_list: @@ -81,7 +82,6 @@ def get_sections(self, section_list): return section_seq def get_properties(self, props_list): - props_seq = [] for prop in props_list: @@ -100,7 +100,7 @@ def get_properties(self, props_list): prop_dict[attr] = t props_seq.append(prop_dict) - + return props_seq def get_values(self, value_list): @@ -122,6 +122,17 @@ def get_values(self, value_list): return value_seq def write_file(self, odml_document, filename): + # Write document only if it does not contain validation errors. + from ..validation import Validation # disgusting import problems + validation = Validation(odml_document) + msg = "" + for e in validation.errors: + if e.is_error: + msg += "\n\t- %s %s: %s" % (e.obj, e.type, e.msg) + if msg != "": + msg = "Resolve document validation errors before saving %s" % msg + raise ParserException(msg) + file = open(filename, 'w') file.write(self.to_string(odml_document)) file.close() @@ -135,7 +146,7 @@ def to_string(self, odml_document): self.to_dict(odml_document) odml_output = {} odml_output['Document'] = self.parsed_doc - odml_output['odml-version'] = odml_version + odml_output['odml-version'] = FORMAT_VERSION if self.parser == 'YAML': string_doc = yaml.dump(odml_output, default_flow_style=False) @@ -150,8 +161,8 @@ class ODMLReader: based on the given data exchange format, like XML, YAML or JSON. Usage: - yaml_odml_doc = ODMLReader(parser='YAML').fromFile(open("odml_doc.yaml")) - json_odml_doc = ODMLReader(parser='JSON').fromFile(open("odml_doc.json")) + yaml_odml_doc = ODMLReader(parser='YAML').from_file("odml_doc.yaml") + json_odml_doc = ODMLReader(parser='JSON').from_file("odml_doc.json") """ def __init__(self, parser='XML'): @@ -162,18 +173,29 @@ def __init__(self, parser='XML'): if parser not in allowed_parsers: raise NotImplementedError("'%s' odML parser does not exist!" % parser) self.parser = parser + self.warnings = [] def is_valid_attribute(self, attr, fmt): if attr in fmt._args: return attr if fmt.revmap(attr): return attr - print("Invalid element <%s> inside <%s> tag" % (attr, fmt.__class__.__name__)) + msg = "Invalid element <%s> inside <%s> tag" % (attr, fmt.__class__.__name__) + print(msg) + self.warnings.append(msg) return None def to_odml(self): - - self.odml_version = self.parsed_doc.get('odml-version', odml_version) + # Parse only odML documents of supported format versions. + if 'odml-version' not in self.parsed_doc: + raise ParserException("Invalid odML document: Could not find odml-version.") + elif self.parsed_doc.get('odml-version') != FORMAT_VERSION: + msg = ("Cannot read file: invalid odML document format version '%s'. \n" + "This package supports odML format versions: '%s'." + % (self.parsed_doc.get('odml-version'), FORMAT_VERSION)) + raise ParserException(msg) + + self.odml_version = self.parsed_doc.get('odml-version') self.parsed_doc = self.parsed_doc['Document'] doc_attrs = {} @@ -193,7 +215,6 @@ def to_odml(self): return self.doc def parse_sections(self, section_list): - odml_sections = [] for section in section_list: @@ -218,7 +239,6 @@ def parse_sections(self, section_list): return odml_sections - def parse_properties(self, props_list): odml_props = [] @@ -257,34 +277,34 @@ def parse_values(self, value_list): return odml_values - def from_file(self, file): if self.parser == 'XML' or self.parser == 'ODML': - odml_doc = xmlparser.XMLReader().fromFile(file) + par = xmlparser.XMLReader(ignore_errors=True) + self.warnings = par.warnings + odml_doc = par.fromFile(file) self.doc = odml_doc return odml_doc elif self.parser == 'YAML': - try: - self.parsed_doc = yaml.load(file) - except yaml.parser.ParserError as e: - print(e) - return - finally: - file.close() + with open(file) as yaml_data: + try: + self.parsed_doc = yaml.load(yaml_data) + except yaml.parser.ParserError as e: + print(e) + return + return self.to_odml() elif self.parser == 'JSON': - try: - self.parsed_doc = json.load(file) - except json.decoder.JSONDecodeError as e: - print(e) - return - finally: - file.close() - return self.to_odml() + with open(file) as json_data: + try: + self.parsed_doc = json.load(json_data) + except ValueError as e: # Python 2 does not support JSONDecodeError + print("JSON Decoder Error: %s" % e) + return + return self.to_odml() def from_string(self, string): @@ -292,17 +312,19 @@ def from_string(self, string): odml_doc = xmlparser.XMLReader().fromString(string) self.doc = odml_doc return self.doc + elif self.parser == 'YAML': try: - odml_doc = yaml.load(string) + self.parsed_doc = yaml.load(string) except yaml.parser.ParserError as e: print(e) return return self.to_odml() + elif self.parser == 'JSON': try: - odml_doc = json.loads(string) - except json.decoder.JSONDecodeError as e: - print(e) + self.parsed_doc = json.loads(string) + except ValueError as e: # Python 2 does not support JSONDecodeError + print("JSON Decoder Error: %s" % e) return return self.to_odml() diff --git a/odml/tools/xmlparser.py b/odml/tools/xmlparser.py index 8a34026a..09586c3c 100644 --- a/odml/tools/xmlparser.py +++ b/odml/tools/xmlparser.py @@ -6,13 +6,13 @@ python -m odml.tools.xmlparser file.odml """ -#TODO make this module a parser class, allow arguments (e.g. skip_errors=1 to parse even broken documents) import sys from odml import format from lxml import etree as ET from lxml.builder import E # this is needed for py2exe to include lxml completely from lxml import _elementpath as _dummy +from ..info import FORMAT_VERSION try: unicode = unicode @@ -35,8 +35,6 @@ format.Value._xml_attributes = {} format.Value._xml_content = 'value' -XML_VERSION = "1" - class XMLWriter: """ @@ -66,7 +64,7 @@ def save_element(e): # generate attributes if isinstance(fmt, format.Document.__class__): - cur.attrib['version'] = XML_VERSION + cur.attrib['version'] = FORMAT_VERSION for k, v in fmt._xml_attributes.items(): if not v or not hasattr(e, fmt.map(v)): @@ -149,6 +147,27 @@ def __init__(self, ignore_errors=False, filename=None): self.tags = dict([(obj._xml_name, obj) for obj in format.__all__]) self.ignore_errors = ignore_errors self.filename = filename + self.warnings = [] + + @staticmethod + def _handle_version(root): + """ + Check if the odML version of a handed in parsed lxml.etree is supported + by the current library and raise an Exception otherwise. + :param root: Root node of a parsed lxml.etree. The root tag has to + contain a supported odML version number, otherwise it is not + accepted as a valid odML file. + """ + if root.tag != 'odML': + raise ParserException("Expecting tag but got <%s>.\n" % root.tag) + elif 'version' not in root.attrib: + raise ParserException("Could not find format version attribute " + "in tag.\n") + elif root.attrib['version'] != FORMAT_VERSION: + msg = ("Cannot read file: invalid odML document format version '%s'. \n" + "This package supports odML format versions: '%s'." + % (root.attrib['version'], FORMAT_VERSION)) + raise ParserException(msg) def fromFile(self, xml_file): """ @@ -159,6 +178,8 @@ def fromFile(self, xml_file): root = ET.parse(xml_file, self.parser).getroot() except ET.XMLSyntaxError as e: raise ParserException(e.msg) + + self._handle_version(root) return self.parse_element(root) def fromString(self, string): @@ -166,6 +187,8 @@ def fromString(self, string): root = ET.XML(string, self.parser) except ET.XMLSyntaxError as e: raise ParserException(e.msg) + + self._handle_version(root) return self.parse_element(root) def check_mandatory_arguments(self, data, ArgClass, tag_name, node): @@ -189,6 +212,7 @@ def warn(self, msg, elem): msg = "warning[%s:%d:<%s>]: %s\n" % (self.filename, elem.sourceline, elem.tag, msg) else: msg = "warning: %s\n" % msg + self.warnings.append(msg) sys.stderr.write(msg) def parse_element(self, node): diff --git a/setup.py b/setup.py index 16271c87..ccfcdf51 100644 --- a/setup.py +++ b/setup.py @@ -3,6 +3,7 @@ from setuptools import setup except ImportError as ex: from distutils.core import setup +from odml.info import AUTHOR, CONTACT, CLASSIFIERS, HOMEPAGE, VERSION packages = [ 'odml', @@ -12,19 +13,24 @@ with open('README.rst') as f: description_text = f.read() +with open("LICENSE") as f: + license_text = f.read() + install_req = ["lxml", "pyyaml"] if sys.version_info < (3, 4): install_req += ["enum34"] setup( name='odML', - version='1.3.2', + version=VERSION, description='open metadata Markup Language', - author='G-Node', - author_email='dev@g-node.org', - url='http://www.g-node.org/projects/odml', + author=AUTHOR, + author_email=CONTACT, + url=HOMEPAGE, packages=packages, test_suite='test', install_requires=install_req, long_description=description_text, + classifiers=CLASSIFIERS, + license=license_text ) diff --git a/test/test_parser.py b/test/test_parser.py index 8f9156af..e593dc6f 100644 --- a/test/test_parser.py +++ b/test/test_parser.py @@ -38,14 +38,14 @@ def tearDown(self): def test_xml(self): self.xml_writer.write_file(self.odml_doc, self.xml_file) - xml_doc = self.xml_reader.from_file(open(self.xml_file)) + xml_doc = self.xml_reader.from_file(self.xml_file) self.assertEqual(xml_doc, self.odml_doc) def test_yaml(self): self.yaml_writer.write_file(self.odml_doc, self.yaml_file) - yaml_doc = self.yaml_reader.from_file(open(self.yaml_file)) + yaml_doc = self.yaml_reader.from_file(self.yaml_file) self.assertEqual(yaml_doc, self.odml_doc) @@ -53,7 +53,7 @@ def test_yaml(self): def test_json(self): self.json_writer.write_file(self.odml_doc, self.json_file) - json_doc = self.json_reader.from_file(open(self.json_file)) + json_doc = self.json_reader.from_file(self.json_file) self.assertEqual(json_doc, self.odml_doc) @@ -61,13 +61,13 @@ def test_json(self): def test_json_yaml_xml(self): self.json_writer.write_file(self.odml_doc, self.json_file) - json_doc = self.json_reader.from_file(open(self.json_file)) + json_doc = self.json_reader.from_file(self.json_file) self.yaml_writer.write_file(json_doc, self.yaml_file) - yaml_doc = self.yaml_reader.from_file(open(self.yaml_file)) + yaml_doc = self.yaml_reader.from_file(self.yaml_file) self.xml_writer.write_file(yaml_doc, self.xml_file) - xml_doc = self.xml_reader.from_file(open(self.xml_file)) + xml_doc = self.xml_reader.from_file(self.xml_file) self.assertEqual(json_doc, self.odml_doc) self.assertEqual(json_doc, yaml_doc) diff --git a/test/test_samplefile.py b/test/test_samplefile.py index 9b8f547e..79405e3a 100644 --- a/test/test_samplefile.py +++ b/test/test_samplefile.py @@ -5,6 +5,7 @@ import sys import re +from odml.info import FORMAT_VERSION from odml.tools import xmlparser from odml.tools import jsonparser from odml.tools import dumper @@ -141,10 +142,10 @@ def test_xml_writer_version(self): val = unicode(xmlparser.XMLWriter(doc)) else: val = str(xmlparser.XMLWriter(doc)) - self.assertIn('version="%s"' % xmlparser.XML_VERSION, val) + self.assertIn('version="%s"' % FORMAT_VERSION, val) doc = xmlparser.XMLReader().fromString(val) # this test is switched off until the XML versioning support is implemented - # self.assertEqual(doc._xml_version, xmlparser.XML_VERSION) + # self.assertEqual(doc._xml_version, FORMAT_VERSION) def test_save(self): for module in [xmlparser.XMLWriter, jsonparser.JSONWriter]: