Merge branch 'release-v0.1.0' into 'master'

Release v0.1.0

See merge request !5

.gitignore

+__pycache__
+build
+dist
+namelist.egg-info
+*.pyc
+.coverage

.gitlab-ci.yml

+image: continuumio/miniconda3:latest
+
+before_script:
+  - env
+  - source activate root
+  - conda install pytest pytest-cov
+
+test:
+  stage: test
+  script:
+    - pytest -v --cov --cov-report term-missing .

README.md

+[![pipeline status](https://git.geomar.de/martin-claus/py-namelist/badges/develop/pipeline.svg)](https://git.geomar.de/martin-claus/py-namelist/commits/develop)
+[![coverage report](https://git.geomar.de/martin-claus/py-namelist/badges/develop/coverage.svg)](https://git.geomar.de/martin-claus/py-namelist/commits/develop)
+
 py-namelist
 ===========
 Parse Fortran Namelists to dict-like objects and back
 
 ## Download
-To get the latest git version do
+To get the latest release do
+```bash
+git clone https://git.geomar.de:martin-claus/py-namelist.git --branch v0.1.0 --depth=1
 ```
-$ git clone https://github.com/martinclaus/py-namelist.git
+or, if you prefer the latest unstable version
+```bash
+git clone https://git.geomar.de:martin-claus/py-namelist.git
 ```
 
+
 ## Usage
-To parse a namelist file you call `parse_namelist(fobj)` where fobj is a file-like object offering a `read()` and `seek()` method (usually the standard python file object).
-```
+To parse a namelist file you call `parse_namelist_file(fobj)` where fobj is a file-like object offering a `read()` and `seek()` method
+(usually the standard python file object). Alternatively, you can parse a string using `parse_namelist_string(str)`.
+```python
 import namelist
 with open(your_nml_file) as fobj:
 	nmls = namelist.parse_namelist_file(fobj)
 ```
 `nmls` will be a list of instances of the `Namelist` class.
 
-`Namelist` is a subclass of `dict` or `OrderedDict` if you use Python >= 2.7. A `Namelist` instance, `nml`, is initialized with an name and optionally with initial values.
-```
+`Namelist` is a subclass of `OrderedDict` (or `dict` if you use Python < 2.7).
+A `Namelist` instance, `nml`, is initialized with an name and optionally with initial values.
+
+```python
 nml = Namelist("param", (("key1", val1), ...))
 ```
 
-The name attribute will set the read-only property name of `nml`. To change, add or delete values the same methods are available as for dict.
-```
+The name attribute will set the read-only property name of `nml`. To change, add or delete values the
+same methods are available as for `dict`.
+```python
 print nml.name
 nml.update({"eggs": 1, "spam": [1, 2, 3]})
 del(nml["param"])
 ```
 
-To create a Fortran readable string representation of the Namelist instance, just use the `str()` build-in
-```
+To create a Fortran readable string representation of the `Namelist` instance, just use the `str()` build-in
+```python
 s = str(nml)
 ```
 or just
-```
+```python
 print(nml)
 ```
 
-A string representation of the Namelist instance that can be used by `eval()` to create copies of it can be created by `repr()`
-```
+A string representation of the `Namelist` instance that can be used by `eval()` to create copies of it can be created by `repr()`
+```python
 print repr(nml)
 ```
 
-**Note**: The parsing of namelist does not have to strictly follow the Fortran standard. Hence, some namelists that are perfectly accepted by some Fortran version are not guaranteed to be correctly parsed by `parse_namelist()`. Always check the content of your Namelist object.
+**Note**: The parsing of namelist does not have to strictly follow the Fortran standard. Hence, some namelists that are perfectly accepted by some Fortran version are not guaranteed to be correctly parsed by `parse_namelist_string()`. Always check the content of your `Namelist` object. If you do find a namelist that does not work, please create an issue at <https://git.geomar.de/martin-claus/py-namelist/> together with the namelist that does not work.

namelist/__init__.py

+"""namelist: Parsing Fortran namelists to Python dictionaries and back."""
+
+__name__ = "namelist"
+__version__ = "0.0.1"
+
+from .namelist import (Namelist,
+                       parse_namelist_file,
+                       parse_namelist_string,
+                       )

namelist.py → namelist/namelist.py

@@ -4,56 +4,60 @@ Created on Tue Sep 23 10:27:57 2014
 
 @author: mclaus
 """
-import sys
 import re
 
-if sys.version_info[:2] < (2, 7):
-    DictClass = dict
-else:
-    # If Python 2.7 or higher use OrderedDict to preserve
-    # the order of the Namelist
+try:
     from collections import OrderedDict as DictClass
+except ImportError:  # pragma: no cover # Python < 2.7
+    DictClass = dict
 
 MODULE_NAME = "namelist"
 NML_LINE_LENGTH = 70
 # Config file parser, called from the class initialization
-varname   = r'[a-zA-Z][a-zA-Z0-9_]*'
-valueBool = re.compile(r"(\.(true|false|t|f)\.)",re.I)
-quote = re.compile(r"([\'\"]{1}.*[\'\"]{1})")
+varname = r'[a-zA-Z][a-zA-Z0-9_]*'
+valueBool = re.compile(r"(\.(true|false|t|f)\.)", re.I)
+quote = re.compile(r"([\']{1}[^\']*[\']{1}|[\"]{1}[^\"]*[\"]{1})",
+                   re.MULTILINE)
 namelistname = re.compile(r"&(" + varname + r")")
 paramname = re.compile(r"^(" + varname + r")")
-namlistend = re.compile(r"^\$(end)?", re.I)
-comment = re.compile(r"#.*")
+namlistend = re.compile(r'^(&(end)?|/)$', re.I)
+comment = re.compile(r"!.*$", re.MULTILINE)
 equalsign = re.compile(r"^=$")
-computation = re.compile(r"^([0-9\.e]+\s*[\*\+\-/]{1}\s*)+[0-9\.e]+", re.I)
+computation = re.compile(
+    r"([\(]*[0-9\.e]+[\)\s]*([\*\+\-/]{1}|[\*]{2})\s*)+[0-9\.e]+[\)]*", re.I
+)
 
 
 class Namelist(DictClass):
-    """ Class to handle Fortran Namelists
-    Namelist(string) -> new namelist with fortran nml identifier string
-    Namelist(string, init_val) -> new initialized namelist with nml identifier
-        string and init_val beeing a valid initialisation object for the parent
+    """Class to handle Fortran Namelists.
+
+    Namelist(string) -> new namelist with fortran namelist group name
+    Namelist(string, init_val) -> new initialized namelist with namelist group
+        name and init_val beeing a valid initialisation object for the parent
         class (either OrderedDict for Python >= 2.7 or else dict).
     A fortran readable string representation of the namelist can be generated
     via str() build-in function. A string representation of the Python object
-    that can be used with eval or string.Template substitution can be obtained
+    that can be used with eval or string. Template substitution can be obtained
     by repr() build-in function.
     """
+
     @property
     def name(self):
-        """ Read only property name, representing the fortran namelist
-        identifier.
-        """
+        """Namelist group name."""
         return self._name
 
     def __init__(self, name, init_val=()):
-        """x.__init__(...) initializes x; see help(type(x)) for signature"""
+        """Create a `Namelist` instance.
+
+        See help(type(x)) for signature.
+        """
         self._name = name
         super(self.__class__, self).__init__(init_val)
 
     def __str__(self):
-        """x.__str__(self) -> Fortran readable string representation of the
-        namelist. If a value v is a sequence, an 1D fortran array representation
+        """Fortran readable string representation of the namelist.
+
+        If a value v is a sequence, an 1D fortran array representation
         is created using iter(v).
         """
         retstr = "&%s\n" % str(self.name)
@@ -75,7 +79,7 @@ class Namelist(DictClass):
                             tmpstr = tmpstr[:-1]
                         retstr += tmpstr + " &\n"
                         tmpstr = ""
-                retstr = retstr + tmpstr[:-1] + "/)\n"
+                retstr = retstr + tmpstr[:-1] + " /)\n"
             else:
                 if isinstance(v, bool):
                     if v:
@@ -85,32 +89,68 @@ class Namelist(DictClass):
                 else:
                     rv = repr(v)
                 retstr += "%s = %s\n" % (str(k), rv)
-        retstr += "&end\n"
+        retstr += "/\n"
         return retstr
 
     def __repr__(self):
-        """x.__repr__(self) -> string that can be used by eval to create a copy
-        of x.
-        """
+        """Return a string that can be used by eval to create a copy."""
         retstr = "%s.%s(%s, (" % (MODULE_NAME, self.__class__.__name__,
-                                 repr(self.name))
+                                  repr(self.name))
         for k, v in self.items():
             retstr += "%s, " % repr((k, v))
         retstr += "))"
         return retstr
 
     def has_name(self, name):
-        """x.hasname(self, name) <==> name==x.name"""
+        """Return `True` if `name` matches the namelist group name.
+
+        Parameters
+        ----------
+        name : str
+            name to test against.
+
+        Returns
+        -------
+        bool : `True` if `name` matches the namelist group name.
+
+        """
         return name == self.name
 
+
 def parse_namelist_file(in_file):
-    """ parse_namelist_file(fobj) -> list of Namelist instances. fobj can be
-    any object that implements pythons file object API, i.e. that offers a
-    read() method.
+    """Parse namelists from file object.
+
+    Parameters
+    ----------
+    in_file : :obj:
+        Any object that implements pythons file object API, i.e. that offers a
+        `read` and `seek` method.
+
+    Returns
+    -------
+    :obj:`List` of :obj:`Namelist`
+
     """
-    retlist = []
-    content = _tokenize(in_file.read())
+    namelist_string = in_file.read()
     in_file.seek(0, 0)
+    return parse_namelist_string(namelist_string)
+
+
+def parse_namelist_string(in_string):
+    """Parse namelists from string.
+
+    Parameters
+    ----------
+    in_string : str
+        String containing one or more namelist definitions.
+
+    Returns
+    -------
+    :obj:`List` of :obj:`Namelist`
+
+    """
+    retlist = []
+    content = _tokenize(in_string)
     for item in content:
         match = re.match(namelistname, item)
         if match:
@@ -123,13 +163,13 @@ def parse_namelist_file(in_file):
             pname = match.group(1)
             nml[pname] = []
             continue
-        for pattern in (namlistend, equalsign):
-            match = re.match(pattern, item)
-            if match:
-                continue
+        if re.match(namlistend, item):
+            continue
+        if re.match(equalsign, item):
+            continue
         match = re.match(valueBool, item)
         if match:
-            nml[pname].append(match.group(1)[1].lower()=="t")
+            nml[pname].append(match.group(1)[1].lower() == "t")
             continue
         match = re.match(quote, item)
         if match:
@@ -140,26 +180,53 @@ def parse_namelist_file(in_file):
         except ValueError:
             pass
         else:
-            continue
+            continue  # pragma: no cover
         try:
             nml[pname].append(float(item))
         except ValueError:
             pass
         else:
-            continue
+            continue  # pragma: no cover
         match = re.match(computation, item)
         if match:
             nml[pname].append(eval(item))
     for nml in retlist:
-        for k, v in nml.iteritems():
+        for k, v in nml.items():
             if len(v) == 1:
                 nml[k] = v[0]
     return retlist
 
+
 def _tokenize(text):
-    fs = "$FS$"
+    """Extract syntax tokens."""
+    fs = "$$$FS$$$"
+
+    # remove comments
     text = re.sub(comment, '', text)
-    for char, rep in zip(('\n', r',', ' ', '=', ), (fs, fs, fs, fs+'='+fs)):
+
+    hashed_tokens = {}
+
+    # replace quoted strings by hash
+    text = _hash_token(text, quote, hashed_tokens, fs)
+
+    # replace numerical computations by hash
+    text = _hash_token(text, computation, hashed_tokens, fs)
+
+    for char, rep in zip(('\n', r',', ' ', '=', '(/', '/)'),
+                         (fs, fs, fs, fs+'='+fs, fs, fs)):
         text = text.replace(char, rep)
     text = text.split(fs)
-    return [token.strip() for token in text if token.strip() != '']
+    tokens = [token.strip() for token in text if token.strip() != '']
+    return [hashed_tokens[t] if t in hashed_tokens else t for t in tokens]
+
+
+def _hash_token(text, pattern, hashed_tokens, fs):
+    while True:
+        match = re.search(pattern, text)
+        if not match:
+            break
+        matched_str = match.group(0)
+        hashed = str(hash(matched_str))
+        hashed_tokens[hashed] = matched_str
+        text = text.replace(matched_str, fs+hashed+fs, 1)
+    return text

setup.py

+import setuptools
+
+with open("README.md", "r") as fh:
+    long_description = fh.read()
+
+setuptools.setup(
+    name="namelist",
+    version="0.1.0",
+    author="Martin Claus",
+    author_email="mclaus@geomar.de",
+    description="Parsing Fortran namelists to Python dictionaries and back.",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    url="https://git.geomar.de/martin-claus/py-namelist",
+    packages=setuptools.find_packages(),
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: GNU General Public License v2 (GPLv2)",
+        "Operating System :: OS Independent",
+        "Development Status :: 4 - Beta",
+    ],
+)

tests/context.py

+"""Context for tests."""
+
+import os
+import sys
+sys.path.insert(
+    0,
+    os.path.abspath(
+        os.path.join(os.path.dirname(__file__), '..')
+    )
+)
+
+
+import namelist
+import io

tests/test_parsing.py

+"""Test paring of namelists to dictionary."""
+
+import pytest
+
+from context import namelist, io
+
+
+@pytest.mark.parametrize(
+    "string,res",
+    [("&nml3 &end", ["&nml3", "&end"]),
+     ("&n val1=34,\nval2=35/",
+      ['&n', 'val1', '=', '34', 'val2', '=', '35/']),
+     ("&n val1=34,\nval2=35 /",
+      ['&n', 'val1', '=', '34', 'val2', '=', '35', '/']),
+     ("&n val1=34,!this is a comment\nval2=35 /",
+      ['&n', 'val1', '=', '34', 'val2', '=', '35', '/']),
+     ]
+)
+def test_tokenize(string, res):
+    nml = namelist.namelist._tokenize(string)
+    assert nml == res
+
+
+@pytest.mark.parametrize(
+    "string",
+    ["&nml &end",
+     "&nml\n&end",
+     "&nml\n&end\n",
+     "&nml &",
+     "&nml /", "&nml/",
+     ]
+)
+def test_parse_string(string):
+    nml = namelist.parse_namelist_string(string)[0]
+    assert nml.name == "nml"
+
+
+@pytest.mark.parametrize(
+    "string",
+    ["&nml &end",
+     "&nml\n&end",
+     "&nml\n&end\n",
+     "&nml &",
+     "&nml /", "&nml/",
+     ]
+)
+def test_parse_file(string):
+    f = io.StringIO(string)
+    nml = namelist.parse_namelist_file(f)[0]
+    assert nml.name == "nml"
+
+
+@pytest.mark.parametrize(
+    "string",
+    ["&nml\n/\n",
+     "&nml\nval1 = 3\n/\n",
+     "&nml\nval1 = .TRUE.\n/\n",
+     "&nml\nval1 = .FALSE.\n/\n",
+     "&nml\nval1 = (/ 1,2,3,4,5,6 /)\n/\n",
+     "&nml\nval1 = (/ .TRUE.,.FALSE.,.TRUE.,.TRUE. /)\n/\n",
+     ]
+)
+def test_string_out(string):
+    nml = namelist.parse_namelist_string(string)[0]
+    assert str(nml) == string
+
+
+@pytest.mark.parametrize(
+    "string,arr",
+    [("&nml\nval = (/ {} /)\n/\n", range(1, 30)),
+     ("&nml\nval = (/ {} /)\n/\n", range(1, 100)),
+     ]
+)
+def test_string_out_linebreak(string, arr):
+    arr_string = ",".join([str(a) for a in arr])
+    nml_string = string.format(arr_string)
+    assert_arr_string = ""
+    while True:
+        lnbrk = namelist.namelist.NML_LINE_LENGTH
+        if len(arr_string) <= lnbrk:
+            assert_arr_string += arr_string
+            break
+        lnbrk_offset = arr_string[lnbrk:].find(',')
+        lnbrk += max(lnbrk_offset, 0) + 1
+        assert_arr_string += arr_string[:lnbrk] + " &\n"
+        arr_string = arr_string[lnbrk:]
+
+    assert_string = string.format(assert_arr_string)
+    nml = namelist.parse_namelist_string(nml_string)[0]
+    print(str(nml), assert_string)
+    assert str(nml) == assert_string
+
+
+@pytest.mark.parametrize(
+    "string",
+    ["&nml\n/\n",
+     "&nml\nval1 = 3\n/\n",
+     "&nml\nval1 = .TRUE., val2 = 1.\n/\n",
+     "&nml\nval1 = .FALSE.\n/\n",
+     "&nml\nval1 = (/ 1,2,3,4,5,6 /)\n/\n",
+     "&nml\nval1 = (/ .TRUE.,.FALSE.,.TRUE.,.TRUE. /)\n/\n",
+     "&nml\nval = (/ {} /)\n/\n".format(range(1, 100)),
+     ]
+)
+def test_repr(string):
+    nml = namelist.parse_namelist_string(string)[0]
+    nml_copy = eval(repr(nml))
+    assert nml == nml_copy
+    assert nml.name == nml_copy.name
+
+
+@pytest.mark.parametrize(
+    "name",
+    ["nml",
+     "nml_other",
+     "nml2",
+     ]
+)
+def test_has_name(name):
+    nml_string = "&{} /".format(name)
+    nml = namelist.parse_namelist_string(nml_string)[0]
+    assert nml.has_name(name)
+
+
+@pytest.mark.parametrize(
+    "string",
+    ["&nml2 val=34 &end",
+     "&nml2\n val=34 \n&end",
+     ]
+)
+def test_match_name_val(string):
+    nml = namelist.parse_namelist_string(string)[0]
+    assert nml.name == "nml2"
+    assert nml["val"] == 34
+
+
+@pytest.mark.parametrize(
+    "string, val",
+    [("&nml2 val=34 &end", 34),
+     ("&nml2\n val=34. \n&end", 34.),
+     ]
+)
+def test_val_conversion(string, val):
+    nml = namelist.parse_namelist_string(string)[0]
+    assert nml.name == "nml2"
+    assert nml["val"] == val
+    assert type(nml["val"]) == type(val)
+
+
+@pytest.mark.parametrize(
+    "string",
+    ["&nml3 val1=34, val2=35 &end",
+     "&nml3 val1=34 val2=35 &end",
+     "&nml3\nval1=34\nval2=35\n&end",
+     ]
+)
+def test_match_multiple_name_val(string):
+    nml = namelist.parse_namelist_string(string)[0]
+    assert nml.name == "nml3"
+    assert nml["val1"] == 34
+    assert nml["val2"] == 35
+
+
+@pytest.mark.parametrize(
+    "string,val",
+    [("&nml val=.T./", True),
+     ("&nml val=.t./", True),
+     ("&nml val=.TRUE./", True),
+     ("&nml val=.true./", True),
+     ("&nml val=.F./", False),
+     ("&nml val=.f./", False),
+     ("&nml val=.FALSE./", False),
+     ("&nml val=.false./", False),
+     ]
+)
+def test_var_bool(string, val):
+    nml = namelist.parse_namelist_string(string)[0]
+    assert nml["val"] is val
+
+
+@pytest.mark.parametrize(
+    "string,val",
+    [("&nml val='this is a string'/", "this is a string"),
+     ("&nml val='this is \na string'/", "this is \na string"),
+     ("&nml val=\"this is \na string\"/", "this is \na string"),
+     ("&nml val=\"this 'is' \na string\"/", "this 'is' \na string"),
+     ("&nml val='this \"is\" \na string'/", "this \"is\" \na string"),
+     ]
+)
+def test_var_string(string, val):
+    nml = namelist.parse_namelist_string(string)[0]
+    assert nml["val"] == val
+
+
+@pytest.mark.parametrize(
+    "string,arr",
+    [("&nml val=(/ {} /)/", list(range(1, 10))),
+     ("&nml val=(/{}/)/", list(range(1, 10))),
+     ("&nml val=(/{}/)/", [float(n) for n in range(1, 10)]),
+     ]
+)
+def test_var_array(string, arr):
+    nml_string = string.format(",".join([str(a) for a in arr]))
+    nml = namelist.parse_namelist_string(nml_string)[0]
+
+    # do elementwise identity check to also check types
+    for a, b in zip(nml["val"], arr):
+        assert a == b
+        assert type(a) == type(b)
+
+    nml_string = string.format(" , ".join([str(a) for a in arr]))
+    nml = namelist.parse_namelist_string(nml_string)[0]
+    for a, b in zip(nml["val"], arr):
+        assert a == b
+        assert type(a) == type(b)
+
+
+@pytest.mark.parametrize("string", ["&nml val= {}, val2='lsl'/"])
+@pytest.mark.parametrize("op", ["+", "-", "/", "*", "**"])
+@pytest.mark.parametrize(
+    "expression",
+    ["2.0 {} 2",
+     "1e3 {} 1e2",
+     "1 {} 2 - 3",
+     "(1 + 2) {} 3",
+     "((1 + 2)) {} 3",
+     "(((1 {} 2))* 3)",
+     ]
+)
+def test_var_expression(string, op, expression):
+    nml_string = string.format(expression.format(op))
+    nml = namelist.parse_namelist_string(nml_string)[0]
+    assert nml["val"] == eval(expression.format(op))