Commit 123c506c authored by Willi Rath's avatar Willi Rath
Browse files

Merge branch '66-add-cera-downloads' into 'develop' 

Resolve "Add a JBLOB renderer"

See merge request !78
parents 6a328f2f ba157ea3

data_repo_renderer/__init__.py

+55 −0
Original line number Diff line number Diff line
@@ -273,6 +273,61 @@ class WgetRecursiveCred(Renderer): 
                "--accept \"{accept_files}\" -P \"{prefix}\" \"{url}\"\n")





class JblobCred(Renderer):

    """Download and install Jblob and download data set from CERA.



    Parameters

    ----------

    prefix : str

        Full path to the directory that should contain the downloaded data in

        the end.

    cera_data_sets : list

        List of cera-dataset acronyms.  Example: ['ERA40_SFC06_MM_EWSS',

                                                  'ERA40_SFC06_MM_NSSS']

    username_var : str

        Variable name in the credential file to hold the user name.

    password_var : str

        Variable name in the credential file to hold the password.



    """



    template = textwrap.dedent("""\



        # == Get data from CERA ==

        # Download Jblob to a tmp directory and use it to download a list of

        # given data sets from CERA:

        jblob_tmp_dir=$(mktemp -d)

        pushd "${{jblob_tmp_dir}}"

        curl \\

            "http://cera-www.dkrz.de/jblob/Jblob-2.0.10.zip" \\

            -o JBlob-2.0.10.zip

        unzip JBlob-2.0.10.zip

        popd

        jblob_jar="${{jblob_tmp_dir}}"/Jblob-2.0.10/Jblob.jar

        commons_jar="${{jblob_tmp_dir}}"/Jblob-2.0.10/commons-codec-1.4.jar

        java -cp "${{jblob_jar}}":"${{commons_jar}}" \\

            de.dkrz.cera.applications.JblobClient \\

            --username "${{{username_var}}}" \\

            --password "${{{password_var}}}" \\

            --dataset {cera_data_sets_string} \\

            --dir {prefix}

        rm -rf "${{jblob_tmp_dir}}"

        # == Get data from CERA ==



        """)



    def __init__(self, yaml_dict=None, *args, **kwargs):

        """Construct `cera_data_sets_string`."""

        if "cera_data_sets" not in yaml_dict:

            yaml_dict["cera_data_sets_string"] = ""

        else:

            yaml_dict["cera_data_sets_string"] = " ".join(

                yaml_dict["cera_data_sets"])



        super(JblobCred, self).__init__(yaml_dict=yaml_dict, *args,

                                        **kwargs)





class PreProcessing(Renderer):

    """Add all lines from the `pre_processing` field."""

tests/test_full_yaml_file_and_cli.py

+31 −0
Original line number Diff line number Diff line
@@ -95,6 +95,15 @@ data: 
      password_var: "STH_PWD"

      method: !!python/name:data_repo_renderer.WgetRecursiveCred



    - cera_data_sets:

        - ERAIN_SFC12_MM_EWSS_180

        - ERAIN_SFC12_MM_NSSS_181

      prefix: data

      username_var: CERA_USER

      password_var: CERA_PWD

      method: !!python/name:data_repo_renderer.JblobCred





doc:



    - url: https://www.example.com/doc_01.html
@@ -148,6 +157,28 @@ wget -nv -r -c -np -nH --cut-dirs=2 --accept "*.nc,*.cdf,*.nc.gz" -P "data" \ 
"https://www.example.com/"

wget -nv -r -c -np -nH --cut-dirs=2 --user="$STH_USER" --password="$STH_PWD" \

--accept "*.nc" -P "data" "https://www.example.com/restricted/"



# == Get data from CERA ==

# Download Jblob to a tmp directory and use it to download a list of

# given data sets from CERA:

jblob_tmp_dir=$(mktemp -d)

pushd "${{jblob_tmp_dir}}"

curl \\

    "http://cera-www.dkrz.de/jblob/Jblob-2.0.10.zip" \\

    -o JBlob-2.0.10.zip

unzip JBlob-2.0.10.zip

popd

jblob_jar="${{jblob_tmp_dir}}"/Jblob-2.0.10/Jblob.jar

commons_jar="${{jblob_tmp_dir}}"/Jblob-2.0.10/commons-codec-1.4.jar

java -cp "${{jblob_jar}}":"${{commons_jar}}" \\

    de.dkrz.cera.applications.JblobClient \\

    --username "${{CERA_USER}}" \\

    --password "${{CERA_PWD}}" \\

    --dataset ERAIN_SFC12_MM_EWSS_180 ERAIN_SFC12_MM_NSSS_181 \\

    --dir data

rm -rf "${{jblob_tmp_dir}}"

# == Get data from CERA ==



curl -L -o "doc/doc_01.html" \

"https://www.example.com/doc_01.html"

curl -L -o "doc/doc_02.html" \

tests/test_sub_classes.py

+32 −0
Original line number Diff line number Diff line
@@ -55,3 +55,35 @@ def test_wget_recursive_cred_rendering(): 
    renderer = data_repo_renderer.WgetRecursiveCred(yaml_dict=yaml_dict)

    print(renderer.yaml_dict, renderer.template)

    assert renderer.__str__ == target_string





def test_jblob_cred_rendering():

    yaml_dict = {"prefix": "pref", "cera_data_sets": ["example_ds_01",

                                                      "example_ds_02"],

                 "username_var": "ASDF_USER", "password_var": "ASDF_PWD"}

    target_string = textwrap.dedent("""\



        # == Get data from CERA ==

        # Download Jblob to a tmp directory and use it to download a list of

        # given data sets from CERA:

        jblob_tmp_dir=$(mktemp -d)

        pushd "${jblob_tmp_dir}"

        curl \\

            "http://cera-www.dkrz.de/jblob/Jblob-2.0.10.zip" \\

            -o JBlob-2.0.10.zip

        unzip JBlob-2.0.10.zip

        popd

        jblob_jar="${jblob_tmp_dir}"/Jblob-2.0.10/Jblob.jar

        commons_jar="${jblob_tmp_dir}"/Jblob-2.0.10/commons-codec-1.4.jar

        java -cp "${jblob_jar}":"${commons_jar}" \\

            de.dkrz.cera.applications.JblobClient \\

            --username "${ASDF_USER}" \\

            --password "${ASDF_PWD}" \\

            --dataset example_ds_01 example_ds_02 \\

            --dir pref

        rm -rf "${jblob_tmp_dir}"

        # == Get data from CERA ==



        """)

    renderer = data_repo_renderer.JblobCred(yaml_dict=yaml_dict)

    assert renderer.__str__ == target_string