Merge pull request #58 from RSE-Cambridge/lvm

add lvm and DNE2 changes for prod dac
RSE-Cambridge · Mar 11, 2019 · b780e7a · b780e7a
2 parents 95b3f1d + 3b7d322
commit b780e7a
Show file tree

Hide file tree

Showing 4 changed files with 99 additions and 37 deletions.
diff --git a/fs-ansible/roles/lustre/tasks/format.yaml b/fs-ansible/roles/lustre/tasks/format.yaml
@@ -1,7 +1,7 @@
 ---
 - set_fact:
     mgs: "{{ vars[fs_name + '_mgs'] | default(omit) }}"
-    mgsnode: "{{ vars[fs_name + '_mgsnode'] }}"
+    mgsnode: "{{ vars[fs_name + '_mgsnode']}}"
     mdts: "{{ vars[fs_name + '_mdts'] | default({}) }}"
     osts: "{{ vars[fs_name + '_osts'] | default({}) }}"
     client_port: "{{ vars[fs_name + '_client_port'] }}"
@@ -23,61 +23,116 @@
     - mgs is defined
   tags: [ 'never', 'reformat_mgs']
 
-- name: Remove old MDT Partition
-  parted:
-     device: "/dev/{{ item }}"
-     number: 1
-     state: absent
+- name: Clean LVM
+  block:
+    - name: Remove LV for MDT
+      lvol:
+        lv: "mdt"
+        vg: "dac-{{ item }}"
+        state: absent
+        force: yes
+      loop: "{{ mdts.keys() }}"
+
+    - name: Remove VG
+      lvg:
+        vg: "dac-{{ item }}"
+        pvs: "/dev/{{ item }}"
+        force: yes
+        state: absent
+      loop: "{{ mdts.keys() }}"
+
+    - name: dmsetup remove lvm
+      command: "dmsetup remove dac--{{ item }}-mdt"
+      register: command_result
+      failed_when: "command_result.rc != 0 and ('device' not in command_result.stderr)"
+      loop: "{{ mdts.keys() }}"
+
+
+  when:
+    - mdts is defined
+  tags: [ 'never', 'reformat_mdts', 'format']
+
+- name: Add VG
+  lvg:
+    vg: "dac-{{ item }}"
+    pvs: "/dev/{{ item }}"
+    state: present
   loop: "{{ mdts.keys() }}"
   when:
     - mdts is defined
   tags: [ 'never', 'reformat_mdts', 'format']
 
-- name: Add MDT Partition
-  parted:
-     device: "/dev/{{ item }}"
-     number: 1
-     part_start: "0%"
-     part_end: "{{ mdt_size }}"
-     label: gpt
-     state: present
+- name: Add LV for MDT
+  lvol:
+    lv: "mdt"
+    vg: "dac-{{ item }}"
+    size: "{{ mdt_size }}"
+    state: present
   loop: "{{ mdts.keys() }}"
   when:
     - mdts is defined
   tags: [ 'never', 'reformat_mdts', 'format']
 
 - name: Reformat MDTs
-  command: "/usr/sbin/mkfs.lustre --mdt --reformat --fsname={{ fs_name }} --index={{ item.value }} --mgsnode={{ mgsnode }}{{ lnet_suffix }} /dev/{{ item.key }}p1"
+  command: "/usr/sbin/mkfs.lustre --mdt --reformat --fsname={{ fs_name }} --index={{ item.value }} --mgsnode={{ mgsnode }}{{ lnet_suffix }} --mkfsoptions=\"-O large_dir\" /dev/mapper/dac--{{ item.key }}-mdt"
   loop: "{{ mdts|dict2items }}"
   when:
     - mdts is defined
   tags: [ 'never', 'reformat_mdts', 'format']
 
-- name: Remove old OST Partition
-  parted:
-     device: "/dev/{{ item }}"
-     number: 2
-     state: absent
+
+- name: Clean LVM
+  block:
+    - name: Remove LV for OST
+      lvol:
+        lv: "ost"
+        vg: "dac-{{ item }}"
+        state: absent
+        force: yes
+      loop: "{{ osts.keys() }}"
+      when:
+        - mdts is defined
+
+    - name: Remove VG OST
+      lvg:
+        vg: "dac-{{ item }}"
+        pvs: "/dev/{{ item }}"
+        state: absent
+      loop: "{{ osts.keys() }}"
+      when:
+        - osts is defined
+        - mdts is not defined
+
+    - name: dmsetup remove lvm
+      command: "dmsetup remove dac--{{ item }}-ost"
+      register: command_result
+      failed_when: "command_result.rc != 0 and ('device' not in command_result.stderr)"
+      loop: "{{ mdts.keys() }}"
+  tags: [ 'never', 'reformat_osts', 'format']
+
+- name: Add VG
+  lvg:
+    vg: "dac-{{ item }}"
+    pvs: "/dev/{{ item }}"
+    state: present
   loop: "{{ osts.keys() }}"
   when:
     - osts is defined
-  tags: [ 'never', 'reformat_mdts', 'format']
+  tags: [ 'never', 'reformat_osts', 'format']
 
-- name: Add OST Partition
-  parted:
-     device: "/dev/{{ item }}"
-     number: 2
-     part_start: "{{ mdt_size }}"
-     part_end: "100%"
-     label: gpt
-     state: present
+- name: Add LV for OST
+  lvol:
+    lv: "ost"
+    vg: "dac-{{ item }}"
+    size: +100%FREE
+    state: present
   loop: "{{ osts.keys() }}"
   when:
     - osts is defined
-  tags: [ 'never', 'reformat_mdts', 'format']
+  tags: [ 'never', 'reformat_osts', 'format']
 
 - name: Reformat OSTs
-  command: "/usr/sbin/mkfs.lustre --ost --reformat --fsname={{ fs_name }} --index={{ item.value }} --mgsnode={{ mgsnode }}{{ lnet_suffix }} /dev/{{ item.key }}p2"
+  command: "/usr/sbin/mkfs.lustre --ost --reformat --fsname={{ fs_name }} --index={{ item.value }} --mgsnode={{ mgsnode }}{{ lnet_suffix }} /dev/mapper/dac--{{ item.key }}-ost"
   loop: "{{ osts|dict2items }}"
   when:
     - osts is defined

diff --git a/fs-ansible/roles/lustre/tasks/mount.yaml b/fs-ansible/roles/lustre/tasks/mount.yaml
@@ -44,12 +44,19 @@
       with_items: "{{ mdts.keys() }}"
 
     - name: mount MDTs
-      command: mount -t lustre /dev/{{ item }}p1 /lustre/{{ fs_name }}/MDT/{{ item }}
+      command: mount -t lustre /dev/mapper/dac--{{ item }}-mdt /lustre/{{ fs_name }}/MDT/{{ item }}
       register: command_result
       failed_when: "command_result.rc != 0 and ('is already mounted' not in command_result.stderr)"
       changed_when: "command_result.rc == 0"
       with_items: "{{ mdts.keys() }}"
 
+    - name: Enable DNE2 remote dir
+      command: "lctl set_param mdt.*.enable_remote_dir=1"
+      register: command_result
+      failed_when: "command_result.rc != 0 and ('lctl' not in command_result.stderr)"
+
+    - name: Enable DNE2 all users
+      command: "lctl set_param mdt.*.enable_remote_dir_gid=-1"
   when:
     - mdts is defined
   tags: [ 'never', 'start_mdts', 'create_mdt']
@@ -65,7 +72,7 @@
       with_items: "{{ osts.keys() }}"
 
     - name: mount OSTs
-      command: mount -t lustre /dev/{{ item }}p2 /lustre/{{ fs_name }}/OST/{{ item }}
+      command: mount -t lustre /dev/mapper/dac--{{ item }}-ost /lustre/{{ fs_name }}/OST/{{ item }}
       register: command_result
       failed_when: "command_result.rc != 0 and ('is already mounted' not in command_result.stderr)"
       changed_when: "command_result.rc == 0"

diff --git a/internal/pkg/pfsprovider/ansible/ansible.go b/internal/pkg/pfsprovider/ansible/ansible.go
@@ -84,7 +84,7 @@ func getInventory(fsType FSType, volume registry.Volume, brickAllocations []regi
 			"mgsnode":     mgsnode,
 			"client_port": fmt.Sprintf("%d", volume.ClientPort),
 			"lnet_suffix": getLnetSuffix(),
-			"mdt_size":    fmt.Sprintf("%dGB", getMdtSize()),
+			"mdt_size":    fmt.Sprintf("%dg", getMdtSize()),
 		},
 		Hosts: hosts,
 	}

diff --git a/internal/pkg/pfsprovider/ansible/ansible_test.go b/internal/pkg/pfsprovider/ansible/ansible_test.go
@@ -39,7 +39,7 @@ func TestPlugin_GetInventory(t *testing.T) {
       vars:
         abcdefgh_client_port: "10002"
         lnet_suffix: ""
-        abcdefgh_mdt_size: 20GB
+        abcdefgh_mdt_size: 20g
         abcdefgh_mgsnode: dac1
 `
 	assert.Equal(t, expected, result)
@@ -66,7 +66,7 @@ func TestPlugin_GetInventory_withNoOstOnOneHost(t *testing.T) {
       vars:
         abcdefgh_client_port: "10002"
         lnet_suffix: ""
-        abcdefgh_mdt_size: 20GB
+        abcdefgh_mdt_size: 20g
         abcdefgh_mgsnode: dac1
 `
 	assert.Equal(t, expected, result)
@@ -205,7 +205,7 @@ func TestPlugin_GetInventory_MaxMDT(t *testing.T) {
       vars:
         abcdefgh_client_port: "10002"
         lnet_suffix: ""
-        abcdefgh_mdt_size: 20GB
+        abcdefgh_mdt_size: 20g
         abcdefgh_mgsnode: dac1
 `
 	assert.Equal(t, expected, result)