test_full_yaml_file_and_cli.py 4.54 KB
Newer Older
Willi Rath's avatar
Willi Rath committed
1 2
# -*- coding:utf-8 -*-

Willi Rath's avatar
Willi Rath committed
3 4
import data_repo_renderer
from pathlib import Path
Willi Rath's avatar
Willi Rath committed
5 6
import pytest

7
yaml_example_file = """
8 9
repo_name: test_repo

Willi Rath's avatar
Willi Rath committed
10 11
people:  Jane Doe (<jane.doe@example.com>), John Doe (<john.doe@example.com>)

12 13 14
http_path_remote: http://www.example.com/git/group/reponame

git_path_remote: http://www.example.com/git/group/reponame.git
15 16 17 18 19 20

repo_description: |
    A sample description.

    With multiple lines.

Willi Rath's avatar
Willi Rath committed
21

22 23 24 25 26 27
    ## And some markdown.

    asdf asdf asdf asdf asdf asdf asdf asdf asdf asdf asdf asdf asdf asdf asdf
    asdf asdf asdf asdf asdf asdf asdf asdf asdf asdf asdf asdf asdf asdf asdf
    asdf asdf asdf asdf asdf asdf

Willi Rath's avatar
Willi Rath committed
28 29
prefixes: data doc

Willi Rath's avatar
Willi Rath committed
30 31 32 33
pre_processing:

    - echo "Not doing anything for pre-processing."

34 35
credential_files:
    - "~/.data_repo_creds/STH.cred"
36
    - "~/.data_repo_creds/SOTH.cred"
37

38
data:
Willi Rath's avatar
Willi Rath committed
39

40 41 42 43
    - url: https://www.example.com/files/nao_station_monthly.txt
      prefix: data
      file_name: nao_station_monthly.txt
      method: !!python/name:data_repo_renderer.CurlSingleFile
44

45 46 47 48
    - url: https://www.example.com/files/nao_station_djfm.txt
      prefix: data
      file_name: nao_station_djfm.txt
      method: !!python/name:data_repo_renderer.CurlSingleFile
49

50 51 52 53
    - url: https://www.example.com/files/nao_station_annual.txt
      prefix: data
      file_name: nao_station_annual.txt
      method: !!python/name:data_repo_renderer.CurlSingleFile
54

55 56 57 58 59
    - url: https://www.example.com/
      cut_dirs: 2
      prefix: data
      accept_files: "*.nc,*.cdf,*.nc.gz"
      method: !!python/name:data_repo_renderer.WgetRecursive
60

61 62 63 64 65 66 67 68
    - url: https://www.example.com/restricted/
      cut_dirs: 2
      prefix: data
      accept_files: "*.nc"
      username_var: "STH_USER"
      password_var: "STH_PWD"
      method: !!python/name:data_repo_renderer.WgetRecursiveCred

69
doc:
Willi Rath's avatar
Willi Rath committed
70

71 72 73 74 75 76 77 78 79 80 81 82 83
    - url: https://www.example.com/doc_01.html
      file_name: doc_01.html
      prefix: doc
      method: !!python/name:data_repo_renderer.CurlSingleFile

    - url: https://www.example.com/doc_02.html
      file_name: doc_02.html
      prefix: doc
      method: !!python/name:data_repo_renderer.CurlSingleFile

post_processing:

    - gunzip data/*.gz
Willi Rath's avatar
Willi Rath committed
84 85
    - util/postprocessing_01.sh
    - util/postprocessing_02.sh
86 87 88 89 90

"""

script_example_file = """#!/bin/bash

Willi Rath's avatar
Willi Rath committed
91 92
# Rendered with data_repo_renderer {version}

93 94 95 96
mkdir -p log
exec &> >(tee -a "log/update.log")
date -I'ns'

Willi Rath's avatar
Willi Rath committed
97
mkdir -p data doc
98

99
source "~/.data_repo_creds/STH.cred"
100
source "~/.data_repo_creds/SOTH.cred"
101

Willi Rath's avatar
Willi Rath committed
102 103
echo "Not doing anything for pre-processing."

104 105 106 107
git remote set-head origin -a
default_branch=`git symbolic-ref \\
    --short refs/remotes/origin/HEAD | cut -d/ -f2-`
git checkout ${{default_branch}}
Willi Rath's avatar
Willi Rath committed
108 109 110 111
git pull
git lfs pull
git lfs track "data/**"

112 113 114 115 116 117
curl -o "data/nao_station_monthly.txt" \
"https://www.example.com/files/nao_station_monthly.txt"
curl -o "data/nao_station_djfm.txt" \
"https://www.example.com/files/nao_station_djfm.txt"
curl -o "data/nao_station_annual.txt" \
"https://www.example.com/files/nao_station_annual.txt"
Willi Rath's avatar
Willi Rath committed
118
wget -nv -r -c -np -nH --cut-dirs=2 --accept "*.nc,*.cdf,*.nc.gz" -P "data" \
119
"https://www.example.com/"
Willi Rath's avatar
Willi Rath committed
120
wget -nv -r -c -np -nH --cut-dirs=2 --user="$STH_USER" --password="$STH_PWD" \
121
--accept "*.nc" -P "data" "https://www.example.com/restricted/"
122 123 124 125
curl -o "doc/doc_01.html" \
"https://www.example.com/doc_01.html"
curl -o "doc/doc_02.html" \
"https://www.example.com/doc_02.html"
126 127

gunzip data/*.gz
Willi Rath's avatar
Willi Rath committed
128 129
util/postprocessing_01.sh
util/postprocessing_02.sh
Willi Rath's avatar
Willi Rath committed
130 131 132 133 134 135

target_branch=`git describe 2> /dev/null`_update_`date +%s%N`
git checkout -b ${{target_branch}}
git add .
git commit -m "Auto-update data"
git push -u origin ${{target_branch}}
Willi Rath's avatar
Willi Rath committed
136
""".format(version=data_repo_renderer.__version__)
137

Willi Rath's avatar
Willi Rath committed
138

Willi Rath's avatar
Willi Rath committed
139
@pytest.fixture
Willi Rath's avatar
Willi Rath committed
140 141 142 143 144 145
def tmp_path(tmpdir):
    return Path(str(tmpdir))


@pytest.fixture
def yaml_example(tmp_path):
Willi Rath's avatar
Willi Rath committed
146 147
    yaml_file = tmp_path / "example.yaml"
    with yaml_file.open(mode="w") as f:
148
        f.write(yaml_example_file)
Willi Rath's avatar
Willi Rath committed
149 150 151 152

    return yaml_file


Willi Rath's avatar
Willi Rath committed
153 154 155 156
@pytest.fixture
def util(tmp_path):
    util_path = tmp_path / "util"
    util_path.mkdir(parents=True, exist_ok=True)
157
    (util_path / "postprocessing_01.sh").touch()
Willi Rath's avatar
Willi Rath committed
158
    (util_path / "postprocessing_02.sh").touch()
Willi Rath's avatar
Willi Rath committed
159 160 161 162
    return util_path


def test_full_yaml_example_01(util, yaml_example, tmp_path):
Willi Rath's avatar
Willi Rath committed
163

Willi Rath's avatar
Willi Rath committed
164 165 166
    data_repo_renderer.cli_run_renderer(["--prefix", str(tmp_path /
                                                         "rendered"),
                                         "--util", str(util),
Willi Rath's avatar
Willi Rath committed
167
                                         str(yaml_example)])
Willi Rath's avatar
Willi Rath committed
168
    with (tmp_path / "rendered" / "update.sh").open() as stream:
169 170
        written_script = stream.read()
    assert written_script == script_example_file