Commit 123c506c authored by Willi Rath's avatar Willi Rath

Merge branch '66-add-cera-downloads' into 'develop'

Resolve "Add a JBLOB renderer"

See merge request !78
parents 6a328f2f ba157ea3
......@@ -273,6 +273,61 @@ class WgetRecursiveCred(Renderer):
"--accept \"{accept_files}\" -P \"{prefix}\" \"{url}\"\n")
class JblobCred(Renderer):
"""Download and install Jblob and download data set from CERA.
Parameters
----------
prefix : str
Full path to the directory that should contain the downloaded data in
the end.
cera_data_sets : list
List of cera-dataset acronyms. Example: ['ERA40_SFC06_MM_EWSS',
'ERA40_SFC06_MM_NSSS']
username_var : str
Variable name in the credential file to hold the user name.
password_var : str
Variable name in the credential file to hold the password.
"""
template = textwrap.dedent("""\
# == Get data from CERA ==
# Download Jblob to a tmp directory and use it to download a list of
# given data sets from CERA:
jblob_tmp_dir=$(mktemp -d)
pushd "${{jblob_tmp_dir}}"
curl \\
"http://cera-www.dkrz.de/jblob/Jblob-2.0.10.zip" \\
-o JBlob-2.0.10.zip
unzip JBlob-2.0.10.zip
popd
jblob_jar="${{jblob_tmp_dir}}"/Jblob-2.0.10/Jblob.jar
commons_jar="${{jblob_tmp_dir}}"/Jblob-2.0.10/commons-codec-1.4.jar
java -cp "${{jblob_jar}}":"${{commons_jar}}" \\
de.dkrz.cera.applications.JblobClient \\
--username "${{{username_var}}}" \\
--password "${{{password_var}}}" \\
--dataset {cera_data_sets_string} \\
--dir {prefix}
rm -rf "${{jblob_tmp_dir}}"
# == Get data from CERA ==
""")
def __init__(self, yaml_dict=None, *args, **kwargs):
"""Construct `cera_data_sets_string`."""
if "cera_data_sets" not in yaml_dict:
yaml_dict["cera_data_sets_string"] = ""
else:
yaml_dict["cera_data_sets_string"] = " ".join(
yaml_dict["cera_data_sets"])
super(JblobCred, self).__init__(yaml_dict=yaml_dict, *args,
**kwargs)
class PreProcessing(Renderer):
"""Add all lines from the `pre_processing` field."""
......
......@@ -95,6 +95,15 @@ data:
password_var: "STH_PWD"
method: !!python/name:data_repo_renderer.WgetRecursiveCred
- cera_data_sets:
- ERAIN_SFC12_MM_EWSS_180
- ERAIN_SFC12_MM_NSSS_181
prefix: data
username_var: CERA_USER
password_var: CERA_PWD
method: !!python/name:data_repo_renderer.JblobCred
doc:
- url: https://www.example.com/doc_01.html
......@@ -148,6 +157,28 @@ wget -nv -r -c -np -nH --cut-dirs=2 --accept "*.nc,*.cdf,*.nc.gz" -P "data" \
"https://www.example.com/"
wget -nv -r -c -np -nH --cut-dirs=2 --user="$STH_USER" --password="$STH_PWD" \
--accept "*.nc" -P "data" "https://www.example.com/restricted/"
# == Get data from CERA ==
# Download Jblob to a tmp directory and use it to download a list of
# given data sets from CERA:
jblob_tmp_dir=$(mktemp -d)
pushd "${{jblob_tmp_dir}}"
curl \\
"http://cera-www.dkrz.de/jblob/Jblob-2.0.10.zip" \\
-o JBlob-2.0.10.zip
unzip JBlob-2.0.10.zip
popd
jblob_jar="${{jblob_tmp_dir}}"/Jblob-2.0.10/Jblob.jar
commons_jar="${{jblob_tmp_dir}}"/Jblob-2.0.10/commons-codec-1.4.jar
java -cp "${{jblob_jar}}":"${{commons_jar}}" \\
de.dkrz.cera.applications.JblobClient \\
--username "${{CERA_USER}}" \\
--password "${{CERA_PWD}}" \\
--dataset ERAIN_SFC12_MM_EWSS_180 ERAIN_SFC12_MM_NSSS_181 \\
--dir data
rm -rf "${{jblob_tmp_dir}}"
# == Get data from CERA ==
curl -L -o "doc/doc_01.html" \
"https://www.example.com/doc_01.html"
curl -L -o "doc/doc_02.html" \
......
......@@ -55,3 +55,35 @@ def test_wget_recursive_cred_rendering():
renderer = data_repo_renderer.WgetRecursiveCred(yaml_dict=yaml_dict)
print(renderer.yaml_dict, renderer.template)
assert renderer.__str__ == target_string
def test_jblob_cred_rendering():
yaml_dict = {"prefix": "pref", "cera_data_sets": ["example_ds_01",
"example_ds_02"],
"username_var": "ASDF_USER", "password_var": "ASDF_PWD"}
target_string = textwrap.dedent("""\
# == Get data from CERA ==
# Download Jblob to a tmp directory and use it to download a list of
# given data sets from CERA:
jblob_tmp_dir=$(mktemp -d)
pushd "${jblob_tmp_dir}"
curl \\
"http://cera-www.dkrz.de/jblob/Jblob-2.0.10.zip" \\
-o JBlob-2.0.10.zip
unzip JBlob-2.0.10.zip
popd
jblob_jar="${jblob_tmp_dir}"/Jblob-2.0.10/Jblob.jar
commons_jar="${jblob_tmp_dir}"/Jblob-2.0.10/commons-codec-1.4.jar
java -cp "${jblob_jar}":"${commons_jar}" \\
de.dkrz.cera.applications.JblobClient \\
--username "${ASDF_USER}" \\
--password "${ASDF_PWD}" \\
--dataset example_ds_01 example_ds_02 \\
--dir pref
rm -rf "${jblob_tmp_dir}"
# == Get data from CERA ==
""")
renderer = data_repo_renderer.JblobCred(yaml_dict=yaml_dict)
assert renderer.__str__ == target_string
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment