From 5f0fdf224a1912a02b3ff5f570a4911ce116c26e Mon Sep 17 00:00:00 2001 From: ajk203 Date: Mon, 11 Mar 2019 10:32:32 +0000 Subject: [PATCH 1/2] add lvm and DNE2 changes for prod dac --- fs-ansible/roles/lustre/tasks/format.yaml | 117 ++++++++++++++++------ fs-ansible/roles/lustre/tasks/mount.yaml | 11 +- 2 files changed, 95 insertions(+), 33 deletions(-) diff --git a/fs-ansible/roles/lustre/tasks/format.yaml b/fs-ansible/roles/lustre/tasks/format.yaml index 3565c456..88fa1616 100644 --- a/fs-ansible/roles/lustre/tasks/format.yaml +++ b/fs-ansible/roles/lustre/tasks/format.yaml @@ -1,7 +1,7 @@ --- - set_fact: mgs: "{{ vars[fs_name + '_mgs'] | default(omit) }}" - mgsnode: "{{ vars[fs_name + '_mgsnode'] }}" + mgsnode: "{{ vars[fs_name + '_mgsnode']}}" mdts: "{{ vars[fs_name + '_mdts'] | default({}) }}" osts: "{{ vars[fs_name + '_osts'] | default({}) }}" client_port: "{{ vars[fs_name + '_client_port'] }}" @@ -23,61 +23,116 @@ - mgs is defined tags: [ 'never', 'reformat_mgs'] -- name: Remove old MDT Partition - parted: - device: "/dev/{{ item }}" - number: 1 - state: absent +- name: Clean LVM + block: + - name: Remove LV for MDT + lvol: + lv: "mdt" + vg: "dac-{{ item }}" + state: absent + force: yes + loop: "{{ mdts.keys() }}" + + - name: Remove VG + lvg: + vg: "dac-{{ item }}" + pvs: "/dev/{{ item }}" + force: yes + state: absent + loop: "{{ mdts.keys() }}" + + - name: dmsetup remove lvm + command: "dmsetup remove dac--{{ item }}-mdt" + register: command_result + failed_when: "command_result.rc != 0 and ('device' not in command_result.stderr)" + loop: "{{ mdts.keys() }}" + + + when: + - mdts is defined + tags: [ 'never', 'reformat_mdts', 'format'] + +- name: Add VG + lvg: + vg: "dac-{{ item }}" + pvs: "/dev/{{ item }}" + state: present loop: "{{ mdts.keys() }}" when: - mdts is defined tags: [ 'never', 'reformat_mdts', 'format'] -- name: Add MDT Partition - parted: - device: "/dev/{{ item }}" - number: 1 - part_start: "0%" - part_end: "{{ mdt_size }}" - label: gpt - state: present +- name: Add LV for MDT + lvol: + lv: "mdt" + vg: "dac-{{ item }}" + size: "{{ mdt_size }}" + state: present loop: "{{ mdts.keys() }}" when: - mdts is defined tags: [ 'never', 'reformat_mdts', 'format'] - name: Reformat MDTs - command: "/usr/sbin/mkfs.lustre --mdt --reformat --fsname={{ fs_name }} --index={{ item.value }} --mgsnode={{ mgsnode }}{{ lnet_suffix }} /dev/{{ item.key }}p1" + command: "/usr/sbin/mkfs.lustre --mdt --reformat --fsname={{ fs_name }} --index={{ item.value }} --mgsnode={{ mgsnode }}{{ lnet_suffix }} --mkfsoptions=\"-O large_dir\" /dev/mapper/dac--{{ item.key }}-mdt" loop: "{{ mdts|dict2items }}" when: - mdts is defined tags: [ 'never', 'reformat_mdts', 'format'] -- name: Remove old OST Partition - parted: - device: "/dev/{{ item }}" - number: 2 - state: absent + +- name: Clean LVM + block: + - name: Remove LV for OST + lvol: + lv: "ost" + vg: "dac-{{ item }}" + state: absent + force: yes + loop: "{{ osts.keys() }}" + when: + - mdts is defined + + - name: Remove VG OST + lvg: + vg: "dac-{{ item }}" + pvs: "/dev/{{ item }}" + state: absent + loop: "{{ osts.keys() }}" + when: + - osts is defined + - mdts is not defined + + - name: dmsetup remove lvm + command: "dmsetup remove dac--{{ item }}-ost" + register: command_result + failed_when: "command_result.rc != 0 and ('device' not in command_result.stderr)" + loop: "{{ mdts.keys() }}" + tags: [ 'never', 'reformat_osts', 'format'] + +- name: Add VG + lvg: + vg: "dac-{{ item }}" + pvs: "/dev/{{ item }}" + state: present loop: "{{ osts.keys() }}" when: - osts is defined - tags: [ 'never', 'reformat_mdts', 'format'] + tags: [ 'never', 'reformat_osts', 'format'] -- name: Add OST Partition - parted: - device: "/dev/{{ item }}" - number: 2 - part_start: "{{ mdt_size }}" - part_end: "100%" - label: gpt - state: present +- name: Add LV for OST + lvol: + lv: "ost" + vg: "dac-{{ item }}" + size: +100%FREE + state: present loop: "{{ osts.keys() }}" when: - osts is defined - tags: [ 'never', 'reformat_mdts', 'format'] + tags: [ 'never', 'reformat_osts', 'format'] - name: Reformat OSTs - command: "/usr/sbin/mkfs.lustre --ost --reformat --fsname={{ fs_name }} --index={{ item.value }} --mgsnode={{ mgsnode }}{{ lnet_suffix }} /dev/{{ item.key }}p2" + command: "/usr/sbin/mkfs.lustre --ost --reformat --fsname={{ fs_name }} --index={{ item.value }} --mgsnode={{ mgsnode }}{{ lnet_suffix }} /dev/mapper/dac--{{ item.key }}-ost" loop: "{{ osts|dict2items }}" when: - osts is defined diff --git a/fs-ansible/roles/lustre/tasks/mount.yaml b/fs-ansible/roles/lustre/tasks/mount.yaml index 3c160483..2daa7132 100644 --- a/fs-ansible/roles/lustre/tasks/mount.yaml +++ b/fs-ansible/roles/lustre/tasks/mount.yaml @@ -44,12 +44,19 @@ with_items: "{{ mdts.keys() }}" - name: mount MDTs - command: mount -t lustre /dev/{{ item }}p1 /lustre/{{ fs_name }}/MDT/{{ item }} + command: mount -t lustre /dev/mapper/dac--{{ item }}-mdt /lustre/{{ fs_name }}/MDT/{{ item }} register: command_result failed_when: "command_result.rc != 0 and ('is already mounted' not in command_result.stderr)" changed_when: "command_result.rc == 0" with_items: "{{ mdts.keys() }}" + - name: Enable DNE2 remote dir + command: "lctl set_param mdt.*.enable_remote_dir=1" + register: command_result + failed_when: "command_result.rc != 0 and ('lctl' not in command_result.stderr)" + + - name: Enable DNE2 all users + command: "lctl set_param mdt.*.enable_remote_dir_gid=-1" when: - mdts is defined tags: [ 'never', 'start_mdts', 'create_mdt'] @@ -65,7 +72,7 @@ with_items: "{{ osts.keys() }}" - name: mount OSTs - command: mount -t lustre /dev/{{ item }}p2 /lustre/{{ fs_name }}/OST/{{ item }} + command: mount -t lustre /dev/mapper/dac--{{ item }}-ost /lustre/{{ fs_name }}/OST/{{ item }} register: command_result failed_when: "command_result.rc != 0 and ('is already mounted' not in command_result.stderr)" changed_when: "command_result.rc == 0" From 3b7d32226ea79543bcb6f761cfe5770ca315a838 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 11 Mar 2019 10:48:13 +0000 Subject: [PATCH 2/2] Update format of mdt_size to match what lvm uses --- internal/pkg/pfsprovider/ansible/ansible.go | 2 +- internal/pkg/pfsprovider/ansible/ansible_test.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/pkg/pfsprovider/ansible/ansible.go b/internal/pkg/pfsprovider/ansible/ansible.go index 3ba06c4f..a61a14aa 100644 --- a/internal/pkg/pfsprovider/ansible/ansible.go +++ b/internal/pkg/pfsprovider/ansible/ansible.go @@ -84,7 +84,7 @@ func getInventory(fsType FSType, volume registry.Volume, brickAllocations []regi "mgsnode": mgsnode, "client_port": fmt.Sprintf("%d", volume.ClientPort), "lnet_suffix": getLnetSuffix(), - "mdt_size": fmt.Sprintf("%dGB", getMdtSize()), + "mdt_size": fmt.Sprintf("%dg", getMdtSize()), }, Hosts: hosts, } diff --git a/internal/pkg/pfsprovider/ansible/ansible_test.go b/internal/pkg/pfsprovider/ansible/ansible_test.go index da4e085e..3a78d7c6 100644 --- a/internal/pkg/pfsprovider/ansible/ansible_test.go +++ b/internal/pkg/pfsprovider/ansible/ansible_test.go @@ -39,7 +39,7 @@ func TestPlugin_GetInventory(t *testing.T) { vars: abcdefgh_client_port: "10002" lnet_suffix: "" - abcdefgh_mdt_size: 20GB + abcdefgh_mdt_size: 20g abcdefgh_mgsnode: dac1 ` assert.Equal(t, expected, result) @@ -66,7 +66,7 @@ func TestPlugin_GetInventory_withNoOstOnOneHost(t *testing.T) { vars: abcdefgh_client_port: "10002" lnet_suffix: "" - abcdefgh_mdt_size: 20GB + abcdefgh_mdt_size: 20g abcdefgh_mgsnode: dac1 ` assert.Equal(t, expected, result) @@ -205,7 +205,7 @@ func TestPlugin_GetInventory_MaxMDT(t *testing.T) { vars: abcdefgh_client_port: "10002" lnet_suffix: "" - abcdefgh_mdt_size: 20GB + abcdefgh_mdt_size: 20g abcdefgh_mgsnode: dac1 ` assert.Equal(t, expected, result)