Commit a3112832 authored by Willi Rath's avatar Willi Rath

Merge branch 'develop' into 'master'

Develop

See merge request !92
parents b597001d af989196
"""data_repo_renderer.""" """data_repo_renderer."""
import argparse import argparse
import logging
from pathlib import Path from pathlib import Path
from pkg_resources import get_distribution, DistributionNotFound from pkg_resources import get_distribution, DistributionNotFound
import shutil import shutil
...@@ -8,12 +9,31 @@ import stat ...@@ -8,12 +9,31 @@ import stat
import textwrap import textwrap
import yaml import yaml
def maybe_get_installed_version(pkg_name):
"""If `pkg_name` is installed, set version from there.
Attributes
----------
pkg_name : str
Package name.
Returns
-------
version : str
Version number from installed package or 'vX.X.X'
"""
try:
version = get_distribution(pkg_name).version
except DistributionNotFound:
# package is not installed
version = "vX.X.X"
return version
# Set version string # Set version string
try: __version__ = maybe_get_installed_version(__name__)
__version__ = get_distribution(__name__).version
except DistributionNotFound:
# package is not installed
__version__ = "vX.X.X"
class Renderer(object): class Renderer(object):
...@@ -439,6 +459,10 @@ class ReadmeRenderer(Renderer): ...@@ -439,6 +459,10 @@ class ReadmeRenderer(Renderer):
"""\ """\
# {{repo_name}} # {{repo_name}}
{{repo_description}}
-----
**Note:** *This dataset is just a mirror of an external source. **Note:** *This dataset is just a mirror of an external source.
Please make sure to properly credit the original creators of the data Please make sure to properly credit the original creators of the data
set just as you would do if you'd obtained the data directly from set just as you would do if you'd obtained the data directly from
...@@ -447,11 +471,6 @@ class ReadmeRenderer(Renderer): ...@@ -447,11 +471,6 @@ class ReadmeRenderer(Renderer):
To learn more about how to use and access the data, check the central To learn more about how to use and access the data, check the central
documentation: <https://git.geomar.de/data/docs/> documentation: <https://git.geomar.de/data/docs/>
-----
{{repo_description}}
{{maybe_acknowledgements}} {{maybe_acknowledgements}}
{{maybe_citations}} {{maybe_citations}}
...@@ -571,6 +590,13 @@ def cli_run_renderer(argvec=None): ...@@ -571,6 +590,13 @@ def cli_run_renderer(argvec=None):
"""Run renderer.""" """Run renderer."""
# parse command line args # parse command line args
parser = argparse.ArgumentParser(description="Data repo renderer") parser = argparse.ArgumentParser(description="Data repo renderer")
parser.add_argument("--verbose", action='store_true',
help="Print debugging output.")
parser.add_argument("--in-place", action='store_true',
help=(
"Render in place? This will render everything in "
"the parent directory of `yaml_file`. "
"Incompatible with --prefix and --util."))
parser.add_argument("--prefix", help=("Custom path to a directory that " parser.add_argument("--prefix", help=("Custom path to a directory that "
"will contain the rendered repo. " "will contain the rendered repo. "
"Defaults to: './rendered/'")) "Defaults to: './rendered/'"))
...@@ -581,10 +607,37 @@ def cli_run_renderer(argvec=None): ...@@ -581,10 +607,37 @@ def cli_run_renderer(argvec=None):
help="Use this YAML file to render repo.") help="Use this YAML file to render repo.")
args = parser.parse_args(argvec) args = parser.parse_args(argvec)
# extract arguments # are we verbose?
util_src = Path(args.util if args.util is not None else "./util/") verbose = (args.verbose is True)
prefix = Path(args.prefix if args.prefix is not None else "./rendered/")
# do we do in-place rerendering?
in_place = (args.in_place is True)
# check for incompatible args
if in_place and args.prefix is not None:
raise ValueError("--prefix and --in-place are incompatible.")
if in_place and args.util is not None:
raise ValueError("--util and --in-place are incompatible.")
# Read paths from cli
yaml_file = Path(args.yaml_file) yaml_file = Path(args.yaml_file)
if in_place:
prefix = yaml_file.parent
util_src = prefix / "util"
else:
util_src = Path(args.util if args.util is not None else "./util/")
prefix = Path(args.prefix if args.prefix is not None
else "./rendered/")
# set up logging
logger = logging.getLogger("data_repo_renderer")
if verbose:
_log_level = logging.DEBUG
else:
_log_level = logging.INFO
logging.basicConfig(
level=_log_level,
format='%(asctime)s %(name)-16s: %(levelname)-8s %(message)s')
# load YAML file # load YAML file
with yaml_file.open() as stream: with yaml_file.open() as stream:
...@@ -602,6 +655,7 @@ def cli_run_renderer(argvec=None): ...@@ -602,6 +655,7 @@ def cli_run_renderer(argvec=None):
try: try:
f.unlink() f.unlink()
except Exception as e: except Exception as e:
logger.debug("failed to unlink {} with: {}".format(f, e))
pass pass
_unlink_files([update_script, init_script, readme_file]) _unlink_files([update_script, init_script, readme_file])
...@@ -635,18 +689,30 @@ def cli_run_renderer(argvec=None): ...@@ -635,18 +689,30 @@ def cli_run_renderer(argvec=None):
ReadmeRenderer(yaml_dict, output_file=readme_file) ReadmeRenderer(yaml_dict, output_file=readme_file)
# copy yaml file # copy yaml file
try: if not in_place:
shutil.copy(str(yaml_file), str(prefix / "meta.yaml")) try:
except shutil.SameFileError as e: shutil.copy(str(yaml_file), str(prefix / "meta.yaml"))
pass except shutil.SameFileError as e:
logger.debug("failed to copy `meta.yaml` with: {}".format(e))
pass
# copy util scripts (if any) # copy util scripts (if any): First, remove targed util, and then copy
try: # source util directory.
shutil.copytree(str(util_src), str(util_dst)) if not in_place:
except Exception as e: try:
pass shutil.rmtree(str(util_dst))
except Exception as e:
logger.debug("failed to unlink {} with: {}".format(util_dst, e))
pass
try:
shutil.copytree(str(util_src), str(util_dst))
except Exception as e:
logger.debug("failed to copy {} to {} with: {}".format(
util_src, util_dst, e))
pass
# make scripts executable # make scripts executable
_make_file_executable(update_script) _make_file_executable(update_script)
_make_file_executable(init_script) _make_file_executable(init_script)
list(map(_make_file_executable, util_dst.glob("*.*"))) if not in_place:
list(map(_make_file_executable, util_dst.glob("*.*")))
...@@ -236,8 +236,10 @@ def util(tmp_path): ...@@ -236,8 +236,10 @@ def util(tmp_path):
return util_path return util_path
@pytest.mark.parametrize("cli_args_verbose", [[], ["--verbose"]])
@pytest.mark.parametrize("util_after", [True, False]) @pytest.mark.parametrize("util_after", [True, False])
def test_full_yaml_example_01(util, yaml_example, tmp_path, util_after): def test_full_yaml_example_01(util, yaml_example, tmp_path, util_after,
cli_args_verbose):
# Set up command line args # Set up command line args
cli_args = ["--prefix", str(tmp_path / "rendered"), cli_args = ["--prefix", str(tmp_path / "rendered"),
...@@ -248,6 +250,9 @@ def test_full_yaml_example_01(util, yaml_example, tmp_path, util_after): ...@@ -248,6 +250,9 @@ def test_full_yaml_example_01(util, yaml_example, tmp_path, util_after):
if util_after: if util_after:
cli_args.insert(2, cli_args.pop(-1)) cli_args.insert(2, cli_args.pop(-1))
# cycle over verbose flag
cli_args = cli_args_verbose + cli_args
# and run the renderer # and run the renderer
data_repo_renderer.cli_run_renderer(cli_args) data_repo_renderer.cli_run_renderer(cli_args)
...@@ -260,3 +265,15 @@ def test_full_yaml_example_01(util, yaml_example, tmp_path, util_after): ...@@ -260,3 +265,15 @@ def test_full_yaml_example_01(util, yaml_example, tmp_path, util_after):
assert all( assert all(
(tmp_path / "rendered" / "util" / file_name).exists() (tmp_path / "rendered" / "util" / file_name).exists()
for file_name in ["postprocessing_01.sh", "postprocessing_02.sh"]) for file_name in ["postprocessing_01.sh", "postprocessing_02.sh"])
@pytest.mark.parametrize("cli_args_verbose", [[], ["--verbose"]])
@pytest.mark.parametrize(
"cli_args_i_p_u",
[["--in-place", "--prefix", "some_prefix"],
["--in-place", "--util", "some_util"],
["--in-place", "--prefix", "some_prefix", "--util", "some_util"]])
def test_incompatible_cli_args(cli_args_i_p_u, cli_args_verbose):
cli_args = cli_args_verbose + cli_args_i_p_u + ["meta.yaml"]
with pytest.raises(ValueError):
data_repo_renderer.cli_run_renderer(cli_args)
# -*- coding:utf-8 -*-
import data_repo_renderer
import pytest
def test_generic_version_if_not_installed():
vers = data_repo_renderer.maybe_get_installed_version("datareporenderer")
assert vers == "vX.X.X"
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment