Clean up index.html* and robots.txt

input_data/TROPFLUX/meta.yaml

@@ -47,3 +47,8 @@ doc:
       file_name: www.incois.gov.in_tropflux_publications.html
       prefix: doc
       method: !!python/name:data_repo_renderer.CurlSingleFile
+
+post_processing:
+
+    - find data/ -iname \*index.html\* -exec rm -fv {} \;
+    - find data/ -iname robots.txt -exec rm -fv {} \;