Merge pull request #476 from GaneshManal/PNDA-4483

PNDA-4483: Data management & cluster housekeeping
pndaproject · Apr 18, 2018 · 8608dbd · 8608dbd
2 parents b67a9c3 + a51ffc6
commit 8608dbd
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 3 deletions.
diff --git a/salt/hdfs-cleaner/init.sls b/salt/hdfs-cleaner/init.sls
@@ -10,17 +10,18 @@
 {% set hadoop_distro = grains['hadoop.distro'] %}
 {% set pnda_user  = pillar['pnda']['user'] %}
 {% set gobblin_work_dir = '/user/' + pnda_user + '/gobblin/work' %}
+{% set flink_job_dir = '/' + pnda_user + '/flink/completed-jobs' %}
 
 {% set install_dir = pillar['pnda']['homedir'] %}
 
 {% set virtual_env_dir = install_dir + "/" + app_directory_name + "/venv" %}
 {% set pip_index_url = pillar['pip']['index_url'] %}
 
 {% if grains['hadoop.distro'] == 'HDP' %}
-{% set streaming_dirs_to_clean = '"/user/*/.sparkStaging/", "/app-logs/*/logs/", "/spark-history/"' %}
+{% set streaming_dirs_to_clean = '"/user/*/.sparkStaging/", "/app-logs/*/logs/", "/spark-history/", "/user/*/.flink/"' %}
 {% set general_dirs_to_clean = '"/mr-history/done/"' %}
 {% else %}
-{% set streaming_dirs_to_clean = '"/user/*/.sparkStaging/", "/tmp/logs/*/logs/", "/user/spark/applicationHistory/"' %}
+{% set streaming_dirs_to_clean = '"/user/*/.sparkStaging/", "/tmp/logs/*/logs/", "/user/spark/applicationHistory/", "/user/*/.flink/"' %}
 {% set general_dirs_to_clean = '"/user/history/done/"' %}
 {% endif %}
 
@@ -65,6 +66,7 @@ hdfs-cleaner-copy_config:
         archive_type: '{{ archive_type }}'
         archive_service: '{{ archive_service }}'
         gobblin_work_dir: {{ gobblin_work_dir }}
+        flink_job_dir: {{ flink_job_dir }}
         streaming_dirs_to_clean: '{{ streaming_dirs_to_clean }}'
         general_dirs_to_clean: '{{ general_dirs_to_clean }}'
     - require:

diff --git a/salt/hdfs-cleaner/templates/properties.json.tpl b/salt/hdfs-cleaner/templates/properties.json.tpl
@@ -16,7 +16,8 @@
     "general_dirs_to_clean": [{{ general_dirs_to_clean }}],
     "old_dirs_to_clean": [
         {"name": "{{ gobblin_work_dir }}/metrics", "age_seconds": 172800},
-        {"name": "{{ gobblin_work_dir }}/state-store/PullFromKafkaMR", "age_seconds": 172800}
+        {"name": "{{ gobblin_work_dir }}/state-store/PullFromKafkaMR", "age_seconds": 172800},
+        {"name": "{{ flink_job_dir }}", "age_seconds": 172800}
     ],
     "swift_repo": "{{ archive_type }}://{{ container }}{{ archive_service }}/{{ repo_path }}",
     "container_name": "{{ container }}",