Commit 31316cc9 authored by Willi Rath's avatar Willi Rath

Baby-sit breaking Jblob connections

parent 112095bc
......@@ -284,6 +284,9 @@ class JblobCred(Renderer):
cera_data_sets : list
List of cera-dataset acronyms. Example: ['ERA40_SFC06_MM_EWSS',
'ERA40_SFC06_MM_NSSS']
max_dl_time : str
Maximum time (in seconds and per data set) allowed for download before
a timeout kicks in to sort of detect a failure.
username_var : str
Variable name in the credential file to hold the user name.
password_var : str
......@@ -304,14 +307,27 @@ class JblobCred(Renderer):
unzip JBlob-2.0.10.zip
popd
jblob_jar="${{jblob_tmp_dir}}"/Jblob-2.0.10/Jblob.jar
commons_jar="${{jblob_tmp_dir}}"/Jblob-2.0.10/commons-codec-1.4.jar
for ds in {cera_data_sets_string}; do
java -cp "${{jblob_jar}}":"${{commons_jar}}" \\
cc14_jar="${{jblob_tmp_dir}}"/Jblob-2.0.10/commons-codec-1.4.jar
function get_cera_data {{
data_set=$1
time_out=$2
timeout ${{time_out}} java -cp "${{jblob_jar}}":"${{cc14_jar}}" \\
de.dkrz.cera.applications.JblobClient \\
--username "${{{username_var}}}" \\
--password "${{{password_var}}}" \\
--dataset ${{ds}} \\
--dataset ${{data_set}} \\
--dir {prefix}
}}
data_set_list=({cera_data_sets_string})
for ds in "${{data_set_list[@]}}"; do
get_cera_data ${{ds}} 3 &> /dev/null;
echo "Touched ${{ds}}"
done
sleep 60;
for ds in "${{data_set_list[@]}}"; do
get_cera_data ${{ds}} {max_dl_time} \\
|| {{ sleep 10; get_cera_data ${{ds}} {max_dl_time}; }} \\
|| echo "${{ds}} failed"
done
rm -rf "${{jblob_tmp_dir}}"
# == Get data from CERA ==
......
......@@ -99,6 +99,7 @@ data:
- ERAIN_SFC12_MM_EWSS_180
- ERAIN_SFC12_MM_NSSS_181
prefix: data
max_dl_time: 100
username_var: CERA_USER
password_var: CERA_PWD
method: !!python/name:data_repo_renderer.JblobCred
......@@ -170,14 +171,27 @@ curl \\
unzip JBlob-2.0.10.zip
popd
jblob_jar="${{jblob_tmp_dir}}"/Jblob-2.0.10/Jblob.jar
commons_jar="${{jblob_tmp_dir}}"/Jblob-2.0.10/commons-codec-1.4.jar
for ds in ERAIN_SFC12_MM_EWSS_180 ERAIN_SFC12_MM_NSSS_181; do
java -cp "${{jblob_jar}}":"${{commons_jar}}" \\
cc14_jar="${{jblob_tmp_dir}}"/Jblob-2.0.10/commons-codec-1.4.jar
function get_cera_data {{
data_set=$1
time_out=$2
timeout ${{time_out}} java -cp "${{jblob_jar}}":"${{cc14_jar}}" \\
de.dkrz.cera.applications.JblobClient \\
--username "${{CERA_USER}}" \\
--password "${{CERA_PWD}}" \\
--dataset ${{ds}} \\
--dataset ${{data_set}} \\
--dir data
}}
data_set_list=(ERAIN_SFC12_MM_EWSS_180 ERAIN_SFC12_MM_NSSS_181)
for ds in "${{data_set_list[@]}}"; do
get_cera_data ${{ds}} 3 &> /dev/null;
echo "Touched ${{ds}}"
done
sleep 60;
for ds in "${{data_set_list[@]}}"; do
get_cera_data ${{ds}} 100 \\
|| {{ sleep 10; get_cera_data ${{ds}} 100; }} \\
|| echo "${{ds}} failed"
done
rm -rf "${{jblob_tmp_dir}}"
# == Get data from CERA ==
......
......@@ -60,7 +60,8 @@ def test_wget_recursive_cred_rendering():
def test_jblob_cred_rendering():
yaml_dict = {"prefix": "pref", "cera_data_sets": ["example_ds_01",
"example_ds_02"],
"username_var": "ASDF_USER", "password_var": "ASDF_PWD"}
"max_dl_time": 100, "username_var": "ASDF_USER",
"password_var": "ASDF_PWD"}
target_string = textwrap.dedent("""\
# == Get data from CERA ==
......@@ -74,14 +75,27 @@ def test_jblob_cred_rendering():
unzip JBlob-2.0.10.zip
popd
jblob_jar="${jblob_tmp_dir}"/Jblob-2.0.10/Jblob.jar
commons_jar="${jblob_tmp_dir}"/Jblob-2.0.10/commons-codec-1.4.jar
for ds in example_ds_01 example_ds_02; do
java -cp "${jblob_jar}":"${commons_jar}" \\
cc14_jar="${jblob_tmp_dir}"/Jblob-2.0.10/commons-codec-1.4.jar
function get_cera_data {
data_set=$1
time_out=$2
timeout ${time_out} java -cp "${jblob_jar}":"${cc14_jar}" \\
de.dkrz.cera.applications.JblobClient \\
--username "${ASDF_USER}" \\
--password "${ASDF_PWD}" \\
--dataset ${ds} \\
--dataset ${data_set} \\
--dir pref
}
data_set_list=(example_ds_01 example_ds_02)
for ds in "${data_set_list[@]}"; do
get_cera_data ${ds} 3 &> /dev/null;
echo "Touched ${ds}"
done
sleep 60;
for ds in "${data_set_list[@]}"; do
get_cera_data ${ds} 100 \\
|| { sleep 10; get_cera_data ${ds} 100; } \\
|| echo "${ds} failed"
done
rm -rf "${jblob_tmp_dir}"
# == Get data from CERA ==
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment