Merge branch '2-create-test-cases' into 'develop'

Resolve "Create test cases"

Closes #2

See merge request !3

.gitignore

@@ -2,3 +2,5 @@ __pycache__
 build
 dist
 namelist.egg-info
+*.pyc
+.coverage

README.md

@@ -5,7 +5,7 @@ Parse Fortran Namelists to dict-like objects and back
 ## Download
 To get the latest git version do
 ```
-$ git clone https://github.com/martinclaus/py-namelist.git
+$ git clone https://git.geomar.de:martin-claus/py-namelist.git
 ```
 
 ## Usage

namelist/__init__.py

@@ -3,4 +3,7 @@
 __name__ = "namelist"
 __version__ = "0.0.1"
 
-from .namelist import Namelist, parse_namelist_file
+from .namelist import (Namelist,
+                       parse_namelist_file,
+                       parse_namelist_string,
+                       )

namelist/namelist.py

@@ -4,56 +4,60 @@ Created on Tue Sep 23 10:27:57 2014
 
 @author: mclaus
 """
-import sys
 import re
 
-if sys.version_info[:2] < (2, 7):
-    DictClass = dict
-else:
-    # If Python 2.7 or higher use OrderedDict to preserve
-    # the order of the Namelist
+try:
     from collections import OrderedDict as DictClass
+except ImportError:  # pragma: no cover # Python < 2.7
+    DictClass = dict
 
 MODULE_NAME = "namelist"
 NML_LINE_LENGTH = 70
 # Config file parser, called from the class initialization
-varname   = r'[a-zA-Z][a-zA-Z0-9_]*'
-valueBool = re.compile(r"(\.(true|false|t|f)\.)",re.I)
-quote = re.compile(r"([\'\"]{1}.*[\'\"]{1})")
+varname = r'[a-zA-Z][a-zA-Z0-9_]*'
+valueBool = re.compile(r"(\.(true|false|t|f)\.)", re.I)
+quote = re.compile(r"([\']{1}[^\']*[\']{1}|[\"]{1}[^\"]*[\"]{1})",
+                   re.MULTILINE)
 namelistname = re.compile(r"&(" + varname + r")")
 paramname = re.compile(r"^(" + varname + r")")
-namlistend = re.compile(r"^\$(end)?", re.I)
-comment = re.compile(r"#.*")
+namlistend = re.compile(r'^(&(end)?|/)$', re.I)
+comment = re.compile(r"!.*$", re.MULTILINE)
 equalsign = re.compile(r"^=$")
-computation = re.compile(r"^([0-9\.e]+\s*[\*\+\-/]{1}\s*)+[0-9\.e]+", re.I)
+computation = re.compile(
+    r"([\(]*[0-9\.e]+[\)\s]*([\*\+\-/]{1}|[\*]{2})\s*)+[0-9\.e]+[\)]*", re.I
+)
 
 
 class Namelist(DictClass):
-    """ Class to handle Fortran Namelists
-    Namelist(string) -> new namelist with fortran nml identifier string
-    Namelist(string, init_val) -> new initialized namelist with nml identifier
-        string and init_val beeing a valid initialisation object for the parent
+    """Class to handle Fortran Namelists.
+
+    Namelist(string) -> new namelist with fortran namelist group name
+    Namelist(string, init_val) -> new initialized namelist with namelist group
+        name and init_val beeing a valid initialisation object for the parent
         class (either OrderedDict for Python >= 2.7 or else dict).
     A fortran readable string representation of the namelist can be generated
     via str() build-in function. A string representation of the Python object
-    that can be used with eval or string.Template substitution can be obtained
+    that can be used with eval or string. Template substitution can be obtained
     by repr() build-in function.
     """
+
     @property
     def name(self):
-        """ Read only property name, representing the fortran namelist
-        identifier.
-        """
+        """Namelist group name."""
         return self._name
 
     def __init__(self, name, init_val=()):
-        """x.__init__(...) initializes x; see help(type(x)) for signature"""
+        """Create a `Namelist` instance.
+
+        See help(type(x)) for signature.
+        """
         self._name = name
         super(self.__class__, self).__init__(init_val)
 
     def __str__(self):
-        """x.__str__(self) -> Fortran readable string representation of the
-        namelist. If a value v is a sequence, an 1D fortran array representation
+        """Fortran readable string representation of the namelist.
+
+        If a value v is a sequence, an 1D fortran array representation
         is created using iter(v).
         """
         retstr = "&%s\n" % str(self.name)
@@ -75,7 +79,7 @@ class Namelist(DictClass):
                             tmpstr = tmpstr[:-1]
                         retstr += tmpstr + " &\n"
                         tmpstr = ""
-                retstr = retstr + tmpstr[:-1] + "/)\n"
+                retstr = retstr + tmpstr[:-1] + " /)\n"
             else:
                 if isinstance(v, bool):
                     if v:
@@ -85,32 +89,68 @@ class Namelist(DictClass):
                 else:
                     rv = repr(v)
                 retstr += "%s = %s\n" % (str(k), rv)
-        retstr += "&end\n"
+        retstr += "/\n"
         return retstr
 
     def __repr__(self):
-        """x.__repr__(self) -> string that can be used by eval to create a copy
-        of x.
-        """
+        """Return a string that can be used by eval to create a copy."""
         retstr = "%s.%s(%s, (" % (MODULE_NAME, self.__class__.__name__,
-                                 repr(self.name))
+                                  repr(self.name))
         for k, v in self.items():
             retstr += "%s, " % repr((k, v))
         retstr += "))"
         return retstr
 
     def has_name(self, name):
-        """x.hasname(self, name) <==> name==x.name"""
+        """Return `True` if `name` matches the namelist group name.
+
+        Parameters
+        ----------
+        name : str
+            name to test against.
+
+        Returns
+        -------
+        bool : `True` if `name` matches the namelist group name.
+
+        """
         return name == self.name
 
+
 def parse_namelist_file(in_file):
-    """ parse_namelist_file(fobj) -> list of Namelist instances. fobj can be
-    any object that implements pythons file object API, i.e. that offers a
-    read() method.
+    """Parse namelists from file object.
+
+    Parameters
+    ----------
+    in_file : :obj:
+        Any object that implements pythons file object API, i.e. that offers a
+        `read` and `seek` method.
+
+    Returns
+    -------
+    :obj:`List` of :obj:`Namelist`
+
     """
-    retlist = []
-    content = _tokenize(in_file.read())
+    namelist_string = in_file.read()
     in_file.seek(0, 0)
+    return parse_namelist_string(namelist_string)
+
+
+def parse_namelist_string(in_string):
+    """Parse namelists from string.
+
+    Parameters
+    ----------
+    in_string : str
+        String containing one or more namelist definitions.
+
+    Returns
+    -------
+    :obj:`List` of :obj:`Namelist`
+
+    """
+    retlist = []
+    content = _tokenize(in_string)
     for item in content:
         match = re.match(namelistname, item)
         if match:
@@ -123,13 +163,13 @@ def parse_namelist_file(in_file):
             pname = match.group(1)
             nml[pname] = []
             continue
-        for pattern in (namlistend, equalsign):
-            match = re.match(pattern, item)
-            if match:
-                continue
+        if re.match(namlistend, item):
+            continue
+        if re.match(equalsign, item):
+            continue
         match = re.match(valueBool, item)
         if match:
-            nml[pname].append(match.group(1)[1].lower()=="t")
+            nml[pname].append(match.group(1)[1].lower() == "t")
             continue
         match = re.match(quote, item)
         if match:
@@ -140,26 +180,53 @@ def parse_namelist_file(in_file):
         except ValueError:
             pass
         else:
-            continue
+            continue  # pragma: no cover
         try:
             nml[pname].append(float(item))
         except ValueError:
             pass
         else:
-            continue
+            continue  # pragma: no cover
         match = re.match(computation, item)
         if match:
             nml[pname].append(eval(item))
     for nml in retlist:
-        for k, v in nml.iteritems():
+        for k, v in nml.items():
             if len(v) == 1:
                 nml[k] = v[0]
     return retlist
 
+
 def _tokenize(text):
-    fs = "$FS$"
+    """Extract syntax tokens."""
+    fs = "$$$FS$$$"
+
+    # remove comments
     text = re.sub(comment, '', text)
-    for char, rep in zip(('\n', r',', ' ', '=', ), (fs, fs, fs, fs+'='+fs)):
+
+    hashed_tokens = {}
+
+    # replace quoted strings by hash
+    text = _hash_token(text, quote, hashed_tokens, fs)
+
+    # replace numerical computations by hash
+    text = _hash_token(text, computation, hashed_tokens, fs)
+
+    for char, rep in zip(('\n', r',', ' ', '=', '(/', '/)'),
+                         (fs, fs, fs, fs+'='+fs, fs, fs)):
         text = text.replace(char, rep)
     text = text.split(fs)
-    return [token.strip() for token in text if token.strip() != '']
+    tokens = [token.strip() for token in text if token.strip() != '']
+    return [hashed_tokens[t] if t in hashed_tokens else t for t in tokens]
+
+
+def _hash_token(text, pattern, hashed_tokens, fs):
+    while True:
+        match = re.search(pattern, text)
+        if not match:
+            break
+        matched_str = match.group(0)
+        hashed = str(hash(matched_str))
+        hashed_tokens[hashed] = matched_str
+        text = text.replace(matched_str, fs+hashed+fs, 1)
+    return text

tests/context.py

+"""Context for tests."""
+
+import os
+import sys
+sys.path.insert(
+    0,
+    os.path.abspath(
+        os.path.join(os.path.dirname(__file__), '..')
+    )
+)
+
+
+import namelist
+import io

tests/test_parsing.py

+"""Test paring of namelists to dictionary."""
+
+import pytest
+
+from context import namelist, io
+
+
+@pytest.mark.parametrize(
+    "string,res",
+    [("&nml3 &end", ["&nml3", "&end"]),
+     ("&n val1=34,\nval2=35/",
+      ['&n', 'val1', '=', '34', 'val2', '=', '35/']),
+     ("&n val1=34,\nval2=35 /",
+      ['&n', 'val1', '=', '34', 'val2', '=', '35', '/']),
+     ("&n val1=34,!this is a comment\nval2=35 /",
+      ['&n', 'val1', '=', '34', 'val2', '=', '35', '/']),
+     ]
+)
+def test_tokenize(string, res):
+    nml = namelist.namelist._tokenize(string)
+    assert nml == res
+
+
+@pytest.mark.parametrize(
+    "string",
+    ["&nml &end",
+     "&nml\n&end",
+     "&nml\n&end\n",
+     "&nml &",
+     "&nml /", "&nml/",
+     ]
+)
+def test_parse_string(string):
+    nml = namelist.parse_namelist_string(string)[0]
+    assert nml.name == "nml"
+
+
+@pytest.mark.parametrize(
+    "string",
+    ["&nml &end",
+     "&nml\n&end",
+     "&nml\n&end\n",
+     "&nml &",
+     "&nml /", "&nml/",
+     ]
+)
+def test_parse_file(string):
+    f = io.StringIO(string)
+    nml = namelist.parse_namelist_file(f)[0]
+    assert nml.name == "nml"
+
+
+@pytest.mark.parametrize(
+    "string",
+    ["&nml\n/\n",
+     "&nml\nval1 = 3\n/\n",
+     "&nml\nval1 = .TRUE.\n/\n",
+     "&nml\nval1 = .FALSE.\n/\n",
+     "&nml\nval1 = (/ 1,2,3,4,5,6 /)\n/\n",
+     "&nml\nval1 = (/ .TRUE.,.FALSE.,.TRUE.,.TRUE. /)\n/\n",
+     ]
+)
+def test_string_out(string):
+    nml = namelist.parse_namelist_string(string)[0]
+    assert str(nml) == string
+
+
+@pytest.mark.parametrize(
+    "string,arr",
+    [("&nml\nval = (/ {} /)\n/\n", range(1, 30)),
+     ("&nml\nval = (/ {} /)\n/\n", range(1, 100)),
+     ]
+)
+def test_string_out_linebreak(string, arr):
+    arr_string = ",".join([str(a) for a in arr])
+    nml_string = string.format(arr_string)
+    assert_arr_string = ""
+    while True:
+        lnbrk = namelist.namelist.NML_LINE_LENGTH
+        if len(arr_string) <= lnbrk:
+            assert_arr_string += arr_string
+            break
+        lnbrk_offset = arr_string[lnbrk:].find(',')
+        lnbrk += max(lnbrk_offset, 0) + 1
+        assert_arr_string += arr_string[:lnbrk] + " &\n"
+        arr_string = arr_string[lnbrk:]
+
+    assert_string = string.format(assert_arr_string)
+    nml = namelist.parse_namelist_string(nml_string)[0]
+    print(str(nml), assert_string)
+    assert str(nml) == assert_string
+
+
+@pytest.mark.parametrize(
+    "string",
+    ["&nml\n/\n",
+     "&nml\nval1 = 3\n/\n",
+     "&nml\nval1 = .TRUE., val2 = 1.\n/\n",
+     "&nml\nval1 = .FALSE.\n/\n",
+     "&nml\nval1 = (/ 1,2,3,4,5,6 /)\n/\n",
+     "&nml\nval1 = (/ .TRUE.,.FALSE.,.TRUE.,.TRUE. /)\n/\n",
+     "&nml\nval = (/ {} /)\n/\n".format(range(1, 100)),
+     ]
+)
+def test_repr(string):
+    nml = namelist.parse_namelist_string(string)[0]
+    nml_copy = eval(repr(nml))
+    assert nml == nml_copy
+    assert nml.name == nml_copy.name
+
+
+@pytest.mark.parametrize(
+    "name",
+    ["nml",
+     "nml_other",
+     "nml2",
+     ]
+)
+def test_has_name(name):
+    nml_string = "&{} /".format(name)
+    nml = namelist.parse_namelist_string(nml_string)[0]
+    assert nml.has_name(name)
+
+
+@pytest.mark.parametrize(
+    "string",
+    ["&nml2 val=34 &end",
+     "&nml2\n val=34 \n&end",
+     ]
+)
+def test_match_name_val(string):
+    nml = namelist.parse_namelist_string(string)[0]
+    assert nml.name == "nml2"
+    assert nml["val"] == 34
+
+
+@pytest.mark.parametrize(
+    "string, val",
+    [("&nml2 val=34 &end", 34),
+     ("&nml2\n val=34. \n&end", 34.),
+     ]
+)
+def test_val_conversion(string, val):
+    nml = namelist.parse_namelist_string(string)[0]
+    assert nml.name == "nml2"
+    assert nml["val"] == val
+    assert type(nml["val"]) == type(val)
+
+
+@pytest.mark.parametrize(
+    "string",
+    ["&nml3 val1=34, val2=35 &end",
+     "&nml3 val1=34 val2=35 &end",
+     "&nml3\nval1=34\nval2=35\n&end",
+     ]
+)
+def test_match_multiple_name_val(string):
+    nml = namelist.parse_namelist_string(string)[0]
+    assert nml.name == "nml3"
+    assert nml["val1"] == 34
+    assert nml["val2"] == 35
+
+
+@pytest.mark.parametrize(
+    "string,val",
+    [("&nml val=.T./", True),
+     ("&nml val=.t./", True),
+     ("&nml val=.TRUE./", True),
+     ("&nml val=.true./", True),
+     ("&nml val=.F./", False),
+     ("&nml val=.f./", False),
+     ("&nml val=.FALSE./", False),
+     ("&nml val=.false./", False),
+     ]
+)
+def test_var_bool(string, val):
+    nml = namelist.parse_namelist_string(string)[0]
+    assert nml["val"] is val
+
+
+@pytest.mark.parametrize(
+    "string,val",
+    [("&nml val='this is a string'/", "this is a string"),
+     ("&nml val='this is \na string'/", "this is \na string"),
+     ("&nml val=\"this is \na string\"/", "this is \na string"),
+     ("&nml val=\"this 'is' \na string\"/", "this 'is' \na string"),
+     ("&nml val='this \"is\" \na string'/", "this \"is\" \na string"),
+     ]
+)
+def test_var_string(string, val):
+    nml = namelist.parse_namelist_string(string)[0]
+    assert nml["val"] == val
+
+
+@pytest.mark.parametrize(
+    "string,arr",
+    [("&nml val=(/ {} /)/", list(range(1, 10))),
+     ("&nml val=(/{}/)/", list(range(1, 10))),
+     ("&nml val=(/{}/)/", [float(n) for n in range(1, 10)]),
+     ]
+)
+def test_var_array(string, arr):
+    nml_string = string.format(",".join([str(a) for a in arr]))
+    nml = namelist.parse_namelist_string(nml_string)[0]
+
+    # do elementwise identity check to also check types
+    for a, b in zip(nml["val"], arr):
+        assert a == b
+        assert type(a) == type(b)
+
+    nml_string = string.format(" , ".join([str(a) for a in arr]))
+    nml = namelist.parse_namelist_string(nml_string)[0]
+    for a, b in zip(nml["val"], arr):
+        assert a == b
+        assert type(a) == type(b)
+
+
+@pytest.mark.parametrize("string", ["&nml val= {}, val2='lsl'/"])
+@pytest.mark.parametrize("op", ["+", "-", "/", "*", "**"])
+@pytest.mark.parametrize(
+    "expression",
+    ["2.0 {} 2",
+     "1e3 {} 1e2",
+     "1 {} 2 - 3",
+     "(1 + 2) {} 3",
+     "((1 + 2)) {} 3",
+     "(((1 {} 2))* 3)",
+     ]
+)
+def test_var_expression(string, op, expression):
+    nml_string = string.format(expression.format(op))
+    nml = namelist.parse_namelist_string(nml_string)[0]
+    assert nml["val"] == eval(expression.format(op))