Merge branch '66-add-cera-downloads' into 'develop'

Resolve "Add a JBLOB renderer"

See merge request !78

data_repo_renderer/__init__.py

@@ -273,6 +273,61 @@ class WgetRecursiveCred(Renderer):
                 "--accept \"{accept_files}\" -P \"{prefix}\" \"{url}\"\n")
 
 
+class JblobCred(Renderer):
+    """Download and install Jblob and download data set from CERA.
+
+    Parameters
+    ----------
+    prefix : str
+        Full path to the directory that should contain the downloaded data in
+        the end.
+    cera_data_sets : list
+        List of cera-dataset acronyms.  Example: ['ERA40_SFC06_MM_EWSS',
+                                                  'ERA40_SFC06_MM_NSSS']
+    username_var : str
+        Variable name in the credential file to hold the user name.
+    password_var : str
+        Variable name in the credential file to hold the password.
+
+    """
+
+    template = textwrap.dedent("""\
+
+        # == Get data from CERA ==
+        # Download Jblob to a tmp directory and use it to download a list of
+        # given data sets from CERA:
+        jblob_tmp_dir=$(mktemp -d)
+        pushd "${{jblob_tmp_dir}}"
+        curl \\
+            "http://cera-www.dkrz.de/jblob/Jblob-2.0.10.zip" \\
+            -o JBlob-2.0.10.zip
+        unzip JBlob-2.0.10.zip
+        popd
+        jblob_jar="${{jblob_tmp_dir}}"/Jblob-2.0.10/Jblob.jar
+        commons_jar="${{jblob_tmp_dir}}"/Jblob-2.0.10/commons-codec-1.4.jar
+        java -cp "${{jblob_jar}}":"${{commons_jar}}" \\
+            de.dkrz.cera.applications.JblobClient \\
+            --username "${{{username_var}}}" \\
+            --password "${{{password_var}}}" \\
+            --dataset {cera_data_sets_string} \\
+            --dir {prefix}
+        rm -rf "${{jblob_tmp_dir}}"
+        # == Get data from CERA ==
+
+        """)
+
+    def __init__(self, yaml_dict=None, *args, **kwargs):
+        """Construct `cera_data_sets_string`."""
+        if "cera_data_sets" not in yaml_dict:
+            yaml_dict["cera_data_sets_string"] = ""
+        else:
+            yaml_dict["cera_data_sets_string"] = " ".join(
+                yaml_dict["cera_data_sets"])
+
+        super(JblobCred, self).__init__(yaml_dict=yaml_dict, *args,
+                                        **kwargs)
+
+
 class PreProcessing(Renderer):
     """Add all lines from the `pre_processing` field."""
 

tests/test_full_yaml_file_and_cli.py

@@ -95,6 +95,15 @@ data:
       password_var: "STH_PWD"
       method: !!python/name:data_repo_renderer.WgetRecursiveCred
 
+    - cera_data_sets:
+        - ERAIN_SFC12_MM_EWSS_180
+        - ERAIN_SFC12_MM_NSSS_181
+      prefix: data
+      username_var: CERA_USER
+      password_var: CERA_PWD
+      method: !!python/name:data_repo_renderer.JblobCred
+
+
 doc:
 
     - url: https://www.example.com/doc_01.html
@@ -148,6 +157,28 @@ wget -nv -r -c -np -nH --cut-dirs=2 --accept "*.nc,*.cdf,*.nc.gz" -P "data" \
 "https://www.example.com/"
 wget -nv -r -c -np -nH --cut-dirs=2 --user="$STH_USER" --password="$STH_PWD" \
 --accept "*.nc" -P "data" "https://www.example.com/restricted/"
+
+# == Get data from CERA ==
+# Download Jblob to a tmp directory and use it to download a list of
+# given data sets from CERA:
+jblob_tmp_dir=$(mktemp -d)
+pushd "${{jblob_tmp_dir}}"
+curl \\
+    "http://cera-www.dkrz.de/jblob/Jblob-2.0.10.zip" \\
+    -o JBlob-2.0.10.zip
+unzip JBlob-2.0.10.zip
+popd
+jblob_jar="${{jblob_tmp_dir}}"/Jblob-2.0.10/Jblob.jar
+commons_jar="${{jblob_tmp_dir}}"/Jblob-2.0.10/commons-codec-1.4.jar
+java -cp "${{jblob_jar}}":"${{commons_jar}}" \\
+    de.dkrz.cera.applications.JblobClient \\
+    --username "${{CERA_USER}}" \\
+    --password "${{CERA_PWD}}" \\
+    --dataset ERAIN_SFC12_MM_EWSS_180 ERAIN_SFC12_MM_NSSS_181 \\
+    --dir data
+rm -rf "${{jblob_tmp_dir}}"
+# == Get data from CERA ==
+
 curl -L -o "doc/doc_01.html" \
 "https://www.example.com/doc_01.html"
 curl -L -o "doc/doc_02.html" \

tests/test_sub_classes.py

@@ -55,3 +55,35 @@ def test_wget_recursive_cred_rendering():
     renderer = data_repo_renderer.WgetRecursiveCred(yaml_dict=yaml_dict)
     print(renderer.yaml_dict, renderer.template)
     assert renderer.__str__ == target_string
+
+
+def test_jblob_cred_rendering():
+    yaml_dict = {"prefix": "pref", "cera_data_sets": ["example_ds_01",
+                                                      "example_ds_02"],
+                 "username_var": "ASDF_USER", "password_var": "ASDF_PWD"}
+    target_string = textwrap.dedent("""\
+
+        # == Get data from CERA ==
+        # Download Jblob to a tmp directory and use it to download a list of
+        # given data sets from CERA:
+        jblob_tmp_dir=$(mktemp -d)
+        pushd "${jblob_tmp_dir}"
+        curl \\
+            "http://cera-www.dkrz.de/jblob/Jblob-2.0.10.zip" \\
+            -o JBlob-2.0.10.zip
+        unzip JBlob-2.0.10.zip
+        popd
+        jblob_jar="${jblob_tmp_dir}"/Jblob-2.0.10/Jblob.jar
+        commons_jar="${jblob_tmp_dir}"/Jblob-2.0.10/commons-codec-1.4.jar
+        java -cp "${jblob_jar}":"${commons_jar}" \\
+            de.dkrz.cera.applications.JblobClient \\
+            --username "${ASDF_USER}" \\
+            --password "${ASDF_PWD}" \\
+            --dataset example_ds_01 example_ds_02 \\
+            --dir pref
+        rm -rf "${jblob_tmp_dir}"
+        # == Get data from CERA ==
+
+        """)
+    renderer = data_repo_renderer.JblobCred(yaml_dict=yaml_dict)
+    assert renderer.__str__ == target_string