Merge branch 'develop' into 'master'

Add input data

See merge request !63

input_data/AO_index/meta.yaml

+repo_name: AO_index
+
+people:  Willi Rath (<wrath@geomar.de>)
+
+http_path_remote: https://git.geomar.de/data/AO_index
+
+git_path_remote: git@git.geomar.de:data/AO_index
+
+repo_description: |
+    Arctic Oscillation time series from
+    <http://www.cpc.ncep.noaa.gov/products/precip/CWlink/daily_ao_index/ao.shtml>.
+
+prefixes: data doc
+
+data:
+
+    - url: "http://www.cpc.ncep.noaa.gov/products/precip/CWlink/daily_ao_index/monthly.ao.index.b50.current.ascii"
+      prefix: data
+      file_name: monthly.ao.index.b50.current.ascii
+      method: !!python/name:data_repo_renderer.CurlSingleFile
+
+doc:
+
+    - url: "http://www.cpc.ncep.noaa.gov/products/precip/CWlink/daily_ao_index/ao.shtml"
+      file_name: www.cpc.ncep.noaa.gov_products_precip_CWlink_daily_ao_index_ao.shtml
+      prefix: doc
+      method: !!python/name:data_repo_renderer.CurlSingleFile
+
+    - url: "http://journals.ametsoc.org/doi/pdf/10.1175/1520-0442%282000%29013%3C3994%3ADFITSP%3E2.0.CO%3B2"
+      file_name: Higgins_et_al_2000.pdf
+      prefix: doc
+      method: !!python/name:data_repo_renderer.CurlSingleFile
+
+    - url: "http://onlinelibrary.wiley.com/doi/10.1029/2001GL013660/pdf"
+      file_name: Zhou_et_al_2001.pdf
+      prefix: doc
+      method: !!python/name:data_repo_renderer.CurlSingleFile

input_data/NSIDC_Sea_Ice/meta.yaml

+repo_name: NSIDC_Sea_Ice
+
+people: Willi Rath (<wrath@geomar.de>)
+
+http_path_remote: https://git.geomar.de/data/NSIDC_Sea_Ice
+
+git_path_remote: git@git.geomar.de:data/NSIDC_Sea_Ice
+
+repo_description: |
+     NOAA/NSIDC Climate Data Record of Passive Microwave Sea Ice Concentration,
+     Version 3
+
+     <http://nsidc.org/data/G02202/versions/3>
+
+prefixes: data doc
+
+data:
+
+    - url: "ftp://sidads.colorado.edu/pub/DATASETS/NOAA/G02202_V3/"
+      cut_dirs: 4
+      prefix: data
+      accept_files: "*.*"
+      method: !!python/name:data_repo_renderer.WgetRecursive
+
+doc:
+
+    - url: "http://nsidc.org/sites/nsidc.org/files/technical-references/SeaIce_CDR_CATBD_final_Rev-5.pdf"
+      file_name: SeaIce_CDR_CATBD_final_Rev-5.pdf
+      prefix: doc
+      method: !!python/name:data_repo_renderer.CurlSingleFile
+
+    - url: "https://www.earth-syst-sci-data.net/5/311/2013/essd-5-311-2013.pdf"
+      file_name: essd-5-311-2013.pdf
+      prefix: doc
+      method: !!python/name:data_repo_renderer.CurlSingleFile
+
+    - url: "http://nsidc.org/data/G02202/versions/3/print/"
+      file_name: nsidc.org_data_G02202_versions_3_print.html
+      prefix: doc
+      method: !!python/name:data_repo_renderer.CurlSingleFile
+
+post_processing:
+
+    - util/convert_to_netCDF4_classic.sh

input_data/NSIDC_Sea_Ice/util/convert_to_netCDF4_classic.sh

+#!/bin/bash
+
+# define and export convert function
+function convert_to_nc4c {
+    file_name=$1
+    echo "converting ${file_name}"
+    nccopy -7 -d 1 -s -c "time/1,ygrid/100,xgrid/100" \
+        ${file_name} ${file_name}_tmp_nc4
+    mv ${file_name}_tmp_nc4 ${file_name}
+}
+export -f convert_to_nc4c
+
+# bunzip and convert using 4 threads
+find data/ -type f -iname \*.nc -print0 | \
+    xargs -0 -n1 -P4 -I {} bash -c "convert_to_nc4c {}"

input_data/SAM_index/meta.yaml

+repo_name: SAM_index
+
+people:  Willi Rath (<wrath@geomar.de>)
+
+http_path_remote: https://git.geomar.de/data/SAM_index
+
+git_path_remote: git@git.geomar.de:data/SAM_index
+
+repo_description: |
+    Marshall Southern Annular Mode (SAM) Index (Station-based)
+
+    <https://climatedataguide.ucar.edu/climate-data/marshall-southern-annular-mode-sam-index-station-based>
+
+prefixes: data doc
+
+data:
+
+    - url: "http://www.nerc-bas.ac.uk/public/icd/gjma/newsam.1957.2007.txt"
+      prefix: data
+      file_name: sam.1957.current.txt
+      method: !!python/name:data_repo_renderer.CurlSingleFile
+
+doc:
+
+    - url: "https://climatedataguide.ucar.edu/climate-data/marshall-southern-annular-mode-sam-index-station-based"
+      file_name: climatedataguide.ucar.edu_climate-data_marshall-southern-annular-mode-sam-index-station-based.html
+      prefix: doc
+      method: !!python/name:data_repo_renderer.CurlSingleFile
+
+    # Disable PDF download for now.
+    # - url: "http://journals.ametsoc.org/doi/pdf/10.1175/1520-0442%282003%29016%3C4134%3ATITSAM%3E2.0.CO%3B2?cookieSet=1"
+    #   file_name: Marshall_2003.pdf
+    #   prefix: doc
+    #   method: !!python/name:data_repo_renderer.CurlSingleFile

input_data/SRTM15_PLUS/meta.yaml

+repo_name: SRTM15_PLUS
+
+people:  Willi Rath (<wrath@geomar.de>)
+
+http_path_remote: https://git.geomar.de/data/SRTM15_PLUS
+
+git_path_remote: git@git.geomar.de:data/SRTM15_PLUS
+
+repo_description: |
+    A new 15-arc second resolution global topography/bathymetry grid
+    (SRTM15_PLUS) developed from a wide variety of data sources.
+
+    <http://topex.ucsd.edu/WWW_html/srtm30_plus.html>
+
+prefixes: data doc
+
+data:
+
+    - url: "ftp://topex.ucsd.edu/pub/srtm15_plus/topo15_compressed.nc"
+      prefix: data
+      file_name: topo15.nc
+      method: !!python/name:data_repo_renderer.CurlSingleFile
+
+doc:
+
+    - url: "http://topex.ucsd.edu/WWW_html/srtm30_plus.html"
+      prefix: doc
+      file_name: topex.ucsd.edu_WWW_html_srtm30_plus.html
+      method: !!python/name:data_repo_renderer.CurlSingleFile
+
+    - url: "http://topex.ucsd.edu/sandwell/publications/124_MG_Becker.pdf"
+      prefix: doc
+      file_name: Becker_et_al_2009.pdf
+      method: !!python/name:data_repo_renderer.CurlSingleFile
+
+    - url: "ftp://topex.ucsd.edu/pub/srtm15_plus/README.V1.txt"
+      prefix: doc
+      file_name: README.V1.txt
+      method: !!python/name:data_repo_renderer.CurlSingleFile
+
+    - url: "ftp://topex.ucsd.edu/pub/srtm15_plus/README_PERMISSIONS.txt"
+      prefix: doc
+      file_name: README_PERMISSIONS.txt
+      method: !!python/name:data_repo_renderer.CurlSingleFile
+
+    - url: "ftp://topex.ucsd.edu/pub/srtm15_plus/REFERENCES_SRTM15_PLUS.txt"
+      prefix: doc
+      file_name: REFERENCES_SRTM15_PLUS.txt
+      method: !!python/name:data_repo_renderer.CurlSingleFile
+
+post_processing:
+
+    - util/convert_to_netCDF4_classic.sh

input_data/SRTM15_PLUS/util/convert_to_netCDF4_classic.sh

+#!/bin/bash
+
+# define and export convert function
+function convert_to_nc4c {
+    file_name=$1
+    echo "converting ${file_name}"
+    nccopy -7 -d 1 -s  \
+        ${file_name} ${file_name}_tmp_nc4
+    mv ${file_name}_tmp_nc4 ${file_name}
+}
+export -f convert_to_nc4c
+
+# bunzip and convert using 4 threads
+find data/ -type f -iname \*.nc -print0 | \
+    xargs -0 -n1 -P4 -I {} bash -c "convert_to_nc4c {}"

input_data/SRTM30_PLUS/meta.yaml

+repo_name: SRTM30_PLUS
+
+people:  Willi Rath (<wrath@geomar.de>)
+
+http_path_remote: https://git.geomar.de/data/SRTM30_PLUS
+
+git_path_remote: git@git.geomar.de:data/SRTM30_PLUS
+
+repo_description: |
+    A new 30-arc second resolution global topography/bathymetry grid
+    (SRTM30_PLUS) developed from a wide variety of data sources.
+
+    <http://topex.ucsd.edu/WWW_html/srtm30_plus.html>
+
+prefixes: data doc
+
+data:
+
+    - url: "ftp://topex.ucsd.edu/pub/srtm30_plus/topo30/topo30.grd"
+      prefix: data
+      file_name: topo30.nc
+      method: !!python/name:data_repo_renderer.CurlSingleFile
+
+    - url: "ftp://topex.ucsd.edu/pub/srtm30_plus/topo30/topo30_sid.grd"
+      prefix: data
+      file_name: topo30_sid.nc
+      method: !!python/name:data_repo_renderer.CurlSingleFile
+
+    - url: "ftp://topex.ucsd.edu/pub/srtm30_plus/sid_filelist_metadata_V11.txt"
+      prefix: data
+      file_name: sid_filelist_metadata_V11.txt
+      method: !!python/name:data_repo_renderer.CurlSingleFile
+
+doc:
+
+    - url: "http://topex.ucsd.edu/WWW_html/srtm30_plus.html"
+      prefix: doc
+      file_name: topex.ucsd.edu_WWW_html_srtm30_plus.html
+      method: !!python/name:data_repo_renderer.CurlSingleFile
+
+    - url: "http://topex.ucsd.edu/sandwell/publications/124_MG_Becker.pdf"
+      prefix: doc
+      file_name: Becker_et_al_2009.pdf
+      method: !!python/name:data_repo_renderer.CurlSingleFile
+
+    - url: "ftp://topex.ucsd.edu/pub/srtm30_plus/topo30/COPYRIGHT.txt"
+      prefix: doc
+      file_name: COPYRIGHT.txt
+      method: !!python/name:data_repo_renderer.CurlSingleFile
+
+    - url: "ftp://topex.ucsd.edu/pub/srtm30_plus/README.V11.txt"
+      prefix: doc
+      file_name: README.V11.txt
+      method: !!python/name:data_repo_renderer.CurlSingleFile
+
+    - url: "ftp://topex.ucsd.edu/pub/srtm30_plus/README_PERMISSIONS.txt"
+      prefix: doc
+      file_name: README_PERMISSIONS.txt
+      method: !!python/name:data_repo_renderer.CurlSingleFile
+
+    - url: "ftp://topex.ucsd.edu/pub/srtm30_plus/README_SRTM30_PLUS.txt"
+      prefix: doc
+      file_name: README_SRTM30_PLUS.txt
+      method: !!python/name:data_repo_renderer.CurlSingleFile
+
+post_processing:
+
+    - util/convert_to_netCDF4_classic.sh

input_data/SRTM30_PLUS/util/convert_to_netCDF4_classic.sh

+#!/bin/bash
+
+# define and export convert function
+function convert_to_nc4c {
+    file_name=$1
+    echo "converting ${file_name}"
+    nccopy -7 -d 1 -s  \
+        ${file_name} ${file_name}_tmp_nc4
+    mv ${file_name}_tmp_nc4 ${file_name}
+}
+export -f convert_to_nc4c
+
+# bunzip and convert using 4 threads
+find data/ -type f -iname \*.nc -print0 | \
+    xargs -0 -n1 -P4 -I {} bash -c "convert_to_nc4c {}"