diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2aadf0b..952003f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -5,6 +5,8 @@ # Set the owner for all the individual samples, if the owner is known and they want to continue to maintain the sample: /AI/GenAI-ChatBot-application-sample/ @cbenbass /CloudFormation/deploy-fsx-ontap/ @kcantrel +/Ansible/fsx_inventory_report/ @kcantrel +/Ansible/snapmirror_report/ @kcantrel /EKS/FSxN-as-PVC-for-EKS/ @mickeysh /Management-Utilities/auto_create_sm_relationships/ @kcantrel /Management-Utilities/auto_set_fsxn_auto_grow/ @kcantrel diff --git a/.github/linters/.yaml-lint.yml b/.github/linters/.yaml-lint.yml index 2959ad2..1fc32a2 100644 --- a/.github/linters/.yaml-lint.yml +++ b/.github/linters/.yaml-lint.yml @@ -2,6 +2,7 @@ extends: default rules: line-length: disable + comments-indentation: disable trailing-spaces: disable truthy: disable braces: diff --git a/Ansible/README.md b/Ansible/README.md new file mode 100644 index 0000000..aedd3e4 --- /dev/null +++ b/Ansible/README.md @@ -0,0 +1,26 @@ +# Ansible Samples +Ansible is a general purpose automation engine. It can be used to automate most repetitive IT tasks. +The typical input you provide Ansible is a "Playbook." This folder holds various playbooks that can be used +to help automate tasks around the management of a FSx for ONTAP file system. +Click here for more information on [Ansible](https://www.ansible.com/). + +| Tool | Description | +| --- | --- | +| [Inventory Report](./fsx_inventory_report) | Creates a report of all the FSxN file systems within an AWS account.| +| [SnapMirror Report](./snapmirror_report) | Creates a report of all SnapMirror relationships within all the FSxN file systems within an AWS account.| + +## Author Information + +This repository is maintained by the contributors listed on [GitHub](https://github.com/NetApp/FSx-ONTAP-samples-scripts/graphs/contributors). + +## License + +Licensed under the Apache License, Version 2.0 (the "License"). + +You may obtain a copy of the License at [apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0). + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an _"AS IS"_ basis, without WARRANTIES or conditions of any kind, either express or implied. + +See the License for the specific language governing permissions and limitations under the License. + +© 2024 NetApp, Inc. All Rights Reserved. diff --git a/Ansible/fsx_inventory_report/README.md b/Ansible/fsx_inventory_report/README.md new file mode 100644 index 0000000..e880749 --- /dev/null +++ b/Ansible/fsx_inventory_report/README.md @@ -0,0 +1,68 @@ +# Ansible Inventory Report +This Ansible playbook generates a report of all the FSx for ONTAP file systems within an AWS account. +In includes all the SVMs and Volumes. The format of the report is as follows: +``` +Region: + File System ID: + SVM ID: + Volumes: + + + SVM ID: + Volumes: + + + File System ID: + SVM ID: + Volumes: + + + SVM ID: + Volumes: + + +``` +Where: + - \ is the provisioned size of the volume in megabytes. + - \ is the security style of the volume (e.g. UNIX, NTFS). + - \ is the type of the volume (e.g. RW, DP). + +## Requirements +- jq - A lightweight and flexible command-line JSON processor. Installation instructions can be found [here](https://jqlang.github.io/jq/download/) +- Ansible 2.9 or later. Installation instructions can be found [here](https://docs.ansible.com/ansible/latest/installation_guide/index.html) +- AWS Ansible collection. This should be included with the base installation of Ansible. + +## Installation +There are three files used to create the report: +- `generate_report.yaml`: The Ansible playbook that generates the report. +- `processs_region.yaml`: A collection of tasks that will process all the FSxNs in a region. +- `get_all_fsxn_regions.yaml`: A collection of tasks that retrieves all the AWS regions, that are enabled for the account, where FSx for ONTAP is available. + +## Configuration +There are a variable that can be changed at the top of the `generate_report.yaml` file: +- report\_name - Sets the file path of the report that will be generated. + +Since this script leverages the AWS Ansible collection as well as the `aws` cli, you will need to provide authentication credentials for them. +You can read more about how to do that [here](https://docs.ansible.com/ansible/latest/collections/amazon/aws/docsite/aws_ec2_guide.html#authentication). + +## Usage +To generate the report copy the three files mentioned above and run the following command: +```bash +ansible-playbook generate_report.yaml +``` + +## Author Information + +This repository is maintained by the contributors listed on [GitHub](https://github.com/NetApp/FSx-ONTAP-samples-scripts/graphs/contributors). + +## License + +Licensed under the Apache License, Version 2.0 (the "License"). + +You may obtain a copy of the License at [apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0). + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an _"AS IS"_ basis, without WARRANTIES or conditions of any kind, either express or implied. + +See the License for the specific language governing permissions and limitations under the License. + +© 2024 NetApp, Inc. All Rights Reserved. diff --git a/Ansible/fsx_inventory_report/generate_report.yaml b/Ansible/fsx_inventory_report/generate_report.yaml new file mode 100644 index 0000000..5d40b09 --- /dev/null +++ b/Ansible/fsx_inventory_report/generate_report.yaml @@ -0,0 +1,34 @@ +# +# This Ansible playbook generates an inventory report for all the FSXNs +# in all the AWS regions for the account that it is running under. +################################################################################# +--- +- vars: + report_name: output.txt +################################################################################# +# +# Don't change anything below this line. +# +################################################################################# + fsxn_regions: [] + opted_in_regions: [] + + name: Playbook to generate an inventory report on all the FSxNs. + hosts: localhost + collections: + - amazon.aws + gather_facts: false + + tasks: + - name: Make sure the report file is empty. + ansible.builtin.shell: + cmd: echo "" > {{ report_name }} + + - name: Get all the regions that support FSxN that are opted into. + include_tasks: get_fsxn_regions.yaml + + - name: Generate the report for all the FSxNs. + include_tasks: process_region.yaml + loop: "{{ fsxn_regions }}" + loop_control: + loop_var: region diff --git a/Ansible/fsx_inventory_report/get_fsxn_regions.yaml b/Ansible/fsx_inventory_report/get_fsxn_regions.yaml new file mode 100644 index 0000000..06270df --- /dev/null +++ b/Ansible/fsx_inventory_report/get_fsxn_regions.yaml @@ -0,0 +1,26 @@ +# +# These tasks are used to set a variable named 'fsnx_regions' that contains a +# list of regions that support FSxN and are opted-in. +################################################################################ +- name: Get all the opted-in regions + amazon.aws.aws_region_info: + register: region_info + +- name: Get region names + set_fact: + enabled_regions: "{{ region_info.regions | map(attribute='region_name') | list }}" + +- name: Get the capabilities of all regions + set_fact: + regions_capabilities: "{{ lookup('ansible.builtin.url', 'https://api.regional-table.region-services.aws.a2z.com/index.json', split_lines=false) }}" + +- name: Filter regions that support FSxN and are opted-in + set_fact: + fsxn_regions: >- + {{ + regions_capabilities.prices + | selectattr("attributes.aws:serviceName", "equalto", "Amazon FSx for NetApp ONTAP") + | selectattr("attributes.aws:region", "in", enabled_regions) + | map(attribute="attributes.aws:region") + | list + }} diff --git a/Ansible/fsx_inventory_report/process_region.yaml b/Ansible/fsx_inventory_report/process_region.yaml new file mode 100644 index 0000000..fd848ab --- /dev/null +++ b/Ansible/fsx_inventory_report/process_region.yaml @@ -0,0 +1,24 @@ +# +# Since Ansible can't handle nested loops, this is a block of tasked that is +# run for each region. It assume that the calling playbook used 'region' as +# its loop variable. +################################################################################ +--- +- name: Get all the FSxNs for the specified region. + ansible.builtin.shell: + cmd: aws fsx describe-file-systems --region {{ region }} --query 'FileSystems[*].{FileSystemId:FileSystemId}' --output text | sed -e '/^$/d' + register: fsxn_ids_per_region + +- name: Get all the SVMs and volumes for each FSxN. + ansible.builtin.shell: + cmd: | + echo "Region: {{ region }}" >> {{ report_name }}; + fs={{ item }}; + echo " File System ID: ${fs}" >> {{ report_name }}; + svms=$(aws fsx describe-storage-virtual-machines --filters Name=file-system-id,Values=${fs} --region {{ region }} --output=text --query 'StorageVirtualMachines[*].StorageVirtualMachineId'); + for svm in $svms; do + echo " SVM ID: ${svm}" >> {{ report_name }}; + echo " Volume IDs:" >> {{ report_name }}; + aws fsx describe-volumes --filters Name=storage-virtual-machine-id,Values=${svm} --region {{ region }} --output=json --query 'Volumes[*].{Size: OntapConfiguration.SizeInMegabytes, ID: VolumeId, Name: Name, Type: OntapConfiguration.OntapVolumeType, Style: OntapConfiguration.SecurityStyle}' | jq -r '.[] | " \(.ID) \(.Type) \(.Style) \(.Size) \(.Name)"' >> {{ report_name }}; + done + loop: "{{ fsxn_ids_per_region.stdout_lines }}" diff --git a/Ansible/snapmirror_report/README.md b/Ansible/snapmirror_report/README.md new file mode 100644 index 0000000..3e15f94 --- /dev/null +++ b/Ansible/snapmirror_report/README.md @@ -0,0 +1,72 @@ +# Ansible SnapMirror Report +This Ansible playbook generates a report of all the FSx for ONTAP SnapMirror relationships within an AWS account. +The output of the report is a CSV file with the following columns: +- File System ID +- Source Path (svm:volume) +- Destination Path (svm:volume) +- State (e.g. snapmirrored, broken-off) +- Healthy (true or false) +- lag\_time (in "P#DT#H#M#S" format. See below for an explanation) + +The lag\_time format is an ASCII string that always starts with the letter 'P' and if the lag time is more than 24 hours it is followed by +a number and the letter 'D'. The number represents the number days. The next character is always a 'T' and is followed by +number/letter pairs, where the letter is either an 'H', 'M', or 'S'. If the letter is 'H' then the number before it represents +the number of hours. If the letter is 'M' then the number before it represents the number of minutes. If the letter is 'S' then +the number before it represents the number of seconds. For example, 'P1DT2H3M4S' represents 1 day, 2 hours, 3 minutes, and 4 seconds. + +## Requirements +- jq - A lightweight and flexible command-line JSON processor. Installation instructions can be found [here](https://jqlang.github.io/jq/download/) +- Ansible 2.9 or later. Installation instructions can be found [here](https://docs.ansible.com/ansible/latest/installation_guide/index.html) +- AWS Ansible collection. This should be included with the base installation of Ansible. +- AWS secret(s) with the credentials necessary to run SnapMirror ONTAP APIs against the FSx for ONTAP file systems. The required format of the secret is described below. + +## Installation +There are three files used to create the report: +- `generate_report.yaml`: The Ansible playbook that generates the report. +- `processs_region.yaml`: A collection of tasks that will process all the FSxNs in a region. +- `get_all_fsxn_regions.yaml`: A collection of tasks that retrieves all the regions, that are enabled for the account, where FSx for ONTAP is available. + +You will also need to create a file named (by default) `secrets_list.csv` that list the secret name for each FSx file systems. +The format of the file should be: +``` +file_system_id,secret_name +``` +:warning: **NOTE:** Do not add any spaces before or after the file\_system\_id or secret\_name. + +Each secret should have two `keys`: +| Key | Value | +| --- | --- | +| `username` | The username to use to authenticate with the FSx for ONTAP file system. | +| `password` | The password to use to authenticate with the FSx for ONTAP file system. | + +## Configuration +There are a few variables that can be changed at the top of the `generate_report.yaml` file: +- report\_name - Sets the file path of the report that will be generated. +- secrets\_list\_file - Sets the file path of the file that contains the list of FSx file systems and their secrets. See above for more information. +- secrets\_region - Set the region where the secrets are stored. + +Since this script leverages the AWS Ansible collection as well as the `aws` cli, you will need to provide authentication credentials for them. +You can read more about how to do that [here](https://docs.ansible.com/ansible/latest/collections/amazon/aws/docsite/aws_ec2_guide.html#authentication). + +## Usage +To generate the report, run the following command: +```bash +ansible-playbook generate_report.yaml +``` +After a successful run, the report will be stored in the file specified by the `report_name` variable. + +## Author Information + +This repository is maintained by the contributors listed on [GitHub](https://github.com/NetApp/FSx-ONTAP-samples-scripts/graphs/contributors). + +## License + +Licensed under the Apache License, Version 2.0 (the "License"). + +You may obtain a copy of the License at [apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0). + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an _"AS IS"_ basis, without WARRANTIES or conditions of any kind, either express or implied. + +See the License for the specific language governing permissions and limitations under the License. + +© 2024 NetApp, Inc. All Rights Reserved. diff --git a/Ansible/snapmirror_report/generate_report.yaml b/Ansible/snapmirror_report/generate_report.yaml new file mode 100644 index 0000000..306ceab --- /dev/null +++ b/Ansible/snapmirror_report/generate_report.yaml @@ -0,0 +1,36 @@ +# +# This Ansible playbook generates a SnapMirrorreport for all the +# SnapMirror relationships, in all the FSxNs, in all regions, +################################################################################# +--- +- vars: + report_name: output.csv + secrets_list_file: secrets_list.csv + secrets_region: us-west-2 +################################################################################# +# +# Don't change anything below this line. +# +################################################################################# + fsxn_regions: [] + opted_in_regions: [] + + name: Playbook to generate a SnapMirror report on all the FSxNs. + hosts: localhost + collections: + - amazon.aws + gather_facts: false + + tasks: + - name: Delete previous report while adding the header line. + ansible.builtin.shell: + cmd: echo fs,source,destination,state,healthy,lag_time > {{ report_name }} + + - name: Get all the regions that support FSxN that I am opted into. + include_tasks: get_fsxn_regions.yaml + + - name: Generate the report for all the FSxNs. + include_tasks: process_region.yaml + loop: "{{ fsxn_regions }}" + loop_control: + loop_var: region diff --git a/Ansible/snapmirror_report/get_fsxn_regions.yaml b/Ansible/snapmirror_report/get_fsxn_regions.yaml new file mode 100644 index 0000000..06270df --- /dev/null +++ b/Ansible/snapmirror_report/get_fsxn_regions.yaml @@ -0,0 +1,26 @@ +# +# These tasks are used to set a variable named 'fsnx_regions' that contains a +# list of regions that support FSxN and are opted-in. +################################################################################ +- name: Get all the opted-in regions + amazon.aws.aws_region_info: + register: region_info + +- name: Get region names + set_fact: + enabled_regions: "{{ region_info.regions | map(attribute='region_name') | list }}" + +- name: Get the capabilities of all regions + set_fact: + regions_capabilities: "{{ lookup('ansible.builtin.url', 'https://api.regional-table.region-services.aws.a2z.com/index.json', split_lines=false) }}" + +- name: Filter regions that support FSxN and are opted-in + set_fact: + fsxn_regions: >- + {{ + regions_capabilities.prices + | selectattr("attributes.aws:serviceName", "equalto", "Amazon FSx for NetApp ONTAP") + | selectattr("attributes.aws:region", "in", enabled_regions) + | map(attribute="attributes.aws:region") + | list + }} diff --git a/Ansible/snapmirror_report/process_region.yaml b/Ansible/snapmirror_report/process_region.yaml new file mode 100644 index 0000000..aecce2c --- /dev/null +++ b/Ansible/snapmirror_report/process_region.yaml @@ -0,0 +1,36 @@ +# +# Since Ansible can't handle nested loops, this is a block of tasked that is +# run for each region. It assume that the calling playbook used 'region' as its loop variable. +################################################################################# +--- +- name: Get all the FSxNs for the specified region. + ansible.builtin.shell: + cmd: aws fsx describe-file-systems --region {{ region }} --query 'FileSystems[*].{FileSystemId:FileSystemId}' --output text | sed -e '/^$/d' + register: fsxn_ids_per_region + +- name: Get the SnapMirror relationships for each FSxN. + when: secret != 'n/a' + ansible.builtin.shell: + cmd: | + fs={{ item }}; + username="{{ lookup('amazon.aws.aws_secret', '{{ secret }}.username', region=secrets_region, nested=true, on_missing='skip') }}"; + password="{{ lookup('amazon.aws.aws_secret', '{{ secret }}.password', region=secrets_region, nested=true, on_missing='skip') }}"; + if [ "$username" = '[]' -o "$password" = '[]' ]; then + echo "Missing secret for file system $fs" 1>&2; + exit 0; + fi; + ip=$(aws fsx describe-file-systems --region {{ region }} --file-system-ids $fs --query 'FileSystems[0].OntapConfiguration.Endpoints.Management.IpAddresses[0]' --output=text); + curl -s -u "${username}:${password}" -k https://$ip/api/snapmirror/relationships?fields=source,destination,lag_time,state,healthy | jq -r '.records[] | "'${fs}',\(.source.path),\(.destination.path),\(.state),\(.healthy),\(.lag_time)"' + loop: "{{ fsxn_ids_per_region.stdout_lines }}" + register: snapmirror_relationships + vars: + secret: "{{ lookup('ansible.builtin.csvfile', item, file=secrets_list_file, delimiter=',', default='n/a') }}" + +- name: Write the SnapMirror relationships to a file. + when: item.stdout is defined + ansible.builtin.shell: + cmd: | + if [ "{{ item.stdout }}" != "" ]; then + echo "{{ item.stdout }}" >> {{ report_name }}; + fi + loop: "{{ snapmirror_relationships.results }}"