diff --git a/example/terraform/.gitignore b/example/terraform/.gitignore new file mode 100644 index 0000000..f665df4 --- /dev/null +++ b/example/terraform/.gitignore @@ -0,0 +1,2 @@ +*.tf +*-managed-zone/ diff --git a/setup.py b/setup.py index 8274081..196fae4 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ from os import path from setuptools import find_packages, setup -dependencies = ['easyzone3', 'ruamel.yaml', 'click'] +dependencies = ['easyzone3', 'ruamel.yaml', 'Jinja2', 'click'] this_directory = path.abspath(path.dirname(__file__)) with open(path.join(this_directory, 'README.md'), encoding='utf-8') as f: @@ -22,6 +22,7 @@ long_description_content_type='text/markdown', package_dir={'': 'src'}, packages=find_packages(where='src'), + package_data={'': ['terraform-modules/*']}, include_package_data=True, zip_safe=False, platforms='any', diff --git a/src/zonefile_migrate/dns_record_set.py b/src/zonefile_migrate/dns_record_set.py new file mode 100644 index 0000000..0cbcfc4 --- /dev/null +++ b/src/zonefile_migrate/dns_record_set.py @@ -0,0 +1,41 @@ +import logging +from easyzone import easyzone +from dns.rdatatype import _by_text as DNSRecordTypes + + +class DNSRecordSet: + """ + a simple DNS record set representation + """ + + def __init__(self, name: str, rectype: str, ttl: int, rrdatas: list[str]): + self.name = name + self.rectype = rectype + self.ttl = ttl + self.rrdatas = rrdatas + + @staticmethod + def create_from_easyzone( + name: easyzone.Name, records: easyzone.Records + ) -> "DNSRecordSet": + """ + create a simple DNS record from an easyzone record. + """ + rrdatas = records.items + if len(rrdatas) > 0 and isinstance(rrdatas[0], (tuple, list)): + rrdatas = list(map(lambda r: " ".join(map(lambda v: str(v), r)), rrdatas)) + + return DNSRecordSet(name.name, records.type, name.ttl, rrdatas) + + +def create_from_zone(zone: easyzone.Zone) -> [DNSRecordSet]: + result: [DNSRecordSet] = [] + for key, name in zone.names.items(): + for rectype in DNSRecordTypes.keys(): + records = name.records(rectype) + if not records: + continue + + result.append(DNSRecordSet.create_from_easyzone(name, records)) + + return result diff --git a/src/zonefile_migrate/terraform-modules/google-managed-zone.tf b/src/zonefile_migrate/terraform-modules/google-managed-zone.tf new file mode 100644 index 0000000..809b646 --- /dev/null +++ b/src/zonefile_migrate/terraform-modules/google-managed-zone.tf @@ -0,0 +1,27 @@ +variable domain_name { + description = "domain name to create zone for" + type = string +} + +variable dns_record_sets { + description = "DNS record sets in this domain" + type = list(object({ + name = string + type = string + ttl = number + rrdatas = list(string) + })) +} +resource "google_dns_managed_zone" "managed_zone" { + name = replace(trimsuffix(var.domain_name, "."), "/\\./", "-") + dns_name = var.domain_name +} + +resource "google_dns_record_set" "record" { + for_each = {for r in var.dns_record_sets: format("%s-%s", r.name, r.type) => r} + name = each.value.name + managed_zone = google_dns_managed_zone.managed_zone.name + type = each.value.type + ttl = each.value.ttl + rrdatas = each.value.rrdatas +} diff --git a/src/zonefile_migrate/to_cloudformation.py b/src/zonefile_migrate/to_cloudformation.py index 1817776..85e216c 100644 --- a/src/zonefile_migrate/to_cloudformation.py +++ b/src/zonefile_migrate/to_cloudformation.py @@ -1,15 +1,16 @@ import click -import json -import logging import re -import sys -import os -from dns.rdatatype import _by_text as DNSRecordTypes + from pathlib import Path from ruamel.yaml import YAML, CommentedMap -from zonefile_migrate.logger import logging +from zonefile_migrate.logger import log from easyzone import easyzone -from dns.exception import SyntaxError +from zonefile_migrate.dns_record_set import create_from_zone +from zonefile_migrate.utils import ( + get_all_zonefiles_in_path, + convert_zonefiles, + target_file, +) def logical_resource_id(name: str): @@ -64,67 +65,49 @@ def convert_to_cloudformation(zone: easyzone.Zone) -> dict: ) result["Resources"] = resources - for key, name in zone.names.items(): - for rectype in DNSRecordTypes.keys(): - records = name.records(rectype) - if not records: - continue - - if rectype == "NS" and key == zone.domain: - logging.warning("ignoring NS records for origin %s", key) - continue - if rectype == "SOA": - logging.warning("ignoring SOA records for origin %s", key) - continue - - logical_name = generate_unique_logical_resource_id( - re.sub( - r"[^0-9a-zA-Z]", - "", - logical_resource_id( - re.sub( - r"^\*", - "wildcard", - key.removesuffix("." + zone.domain) - if key != "@" - else "Origin", - ) - ), - ) - + records.type - + "Record", - resources, - ) - resource_records = records.items - if rectype == "MX": - resource_records = list(map(lambda r: f"{r[0]} {r[1]}", records.items)) - - resources[logical_name] = CommentedMap( - { - "Type": "AWS::Route53::RecordSet", - "Properties": { - "Name": key, - "Type": records.type, - "ResourceRecords": resource_records, - "TTL": name.ttl, - "HostedZoneId": {"Ref": "HostedZone"}, - }, - } + for record_set in create_from_zone(zone): + if record_set.rectype == "NS" and record_set.name == zone.domain: + log.debug("ignoring NS records for origin %s", zone.domain) + continue + if record_set.rectype == "SOA": + log.debug("ignoring SOA records for domain %s", record_set.name) + continue + + logical_name = generate_unique_logical_resource_id( + re.sub( + r"[^0-9a-zA-Z]", + "", + logical_resource_id( + re.sub( + r"^\*", + "wildcard", + record_set.name.removesuffix("." + zone.domain) + if record_set.name == zone.domain + else "Origin", + ) + ), ) + + record_set.rectype + + "Record", + resources, + ) + + resources[logical_name] = CommentedMap( + { + "Type": "AWS::Route53::RecordSet", + "Properties": { + "Name": record_set.name, + "Type": record_set.rectype, + "ResourceRecords": record_set.rrdatas, + "TTL": record_set.ttl, + "HostedZoneId": {"Ref": "HostedZone"}, + }, + } + ) return result -def target_file(src: Path, dst: Path) -> Path: - if dst.is_file(): - return dst - - if src.suffix == ".zone": - return dst.joinpath(src.name).with_suffix(".yaml") - - return dst.joinpath(src.name + ".yaml") - - def common_parent(one: Path, other: Path) -> Path: """ returns the commons parent of two paths @@ -203,18 +186,6 @@ def generate_sceptre_configuration( YAML().dump(config, file) -def is_zonefile(path: Path) -> bool: - """ - returns true if the file pointed to by `path` contains a $ORIGIN or $TTL pragma, otherwise False - """ - if path.exists() and path.is_file(): - with path.open("r") as file: - for line in file: - if re.search(r"^\s*\$(ORIGIN|TTL)\s+", line, re.IGNORECASE): - return True - return False - - @click.command(name="to-cloudformation") @click.option( "--sceptre-group", @@ -237,8 +208,6 @@ def command(sceptre_group, src, dst): The zonefiles must contain a $ORIGIN and $TTL statement. If the SRC points to a directory all files which contain one of these statements will be converted. If a $ORIGIN is missing, the name of the file will be used as the domain name. - - """ if sceptre_group: sceptre_group = Path(sceptre_group) @@ -246,17 +215,10 @@ def command(sceptre_group, src, dst): if not src: raise click.UsageError("no source files were specified") - inputs = [] - for filename in map(lambda s: Path(s), src): - if filename.is_dir(): - inputs.extend( - [f for f in filename.iterdir() if f.is_file() and is_zonefile(f)] - ) - else: - inputs.append(filename) + inputs = get_all_zonefiles_in_path(src) if len(inputs) == 0: - click.UsageError("no zone files were found") + click.UsageError("no zonefiles were found") dst = Path(dst) if len(inputs) > 1: @@ -265,38 +227,16 @@ def command(sceptre_group, src, dst): if not dst.exists(): dst.mkdir(parents=True, exist_ok=True) - outputs = list(map(lambda d: target_file(d, dst), inputs)) + outputs = list(map(lambda d: target_file(d, dst, ".yaml"), inputs)) - for i, input in enumerate(map(lambda s: Path(s), inputs)): - with input.open("r") as file: - content = file.read() - found = re.search( - r"\$ORIGIN\s+(?P.*)\s*", - content, - re.MULTILINE | re.IGNORECASE, - ) - if found: - domain_name = found.group("domain_name") - else: - domain_name = input.name.removesuffix(".zone") - logging.warning( - "could not find $ORIGIN from zone file %s, using %s", - input, - domain_name, - ) - - try: - logging.info("reading zonefile %s", input.as_posix()) - zone = easyzone.zone_from_file(domain_name, input.as_posix()) - except SyntaxError as error: - logging.error(error) - exit(1) - - with outputs[i].open("w") as file: + def transform_to_cloudformation(zone: easyzone.Zone, output: Path): + with output.open("w") as file: YAML().dump(convert_to_cloudformation(zone), stream=file) if sceptre_group: generate_sceptre_configuration(zone, outputs[i], sceptre_group) + convert_zonefiles(inputs, outputs, transform_to_cloudformation) + if __name__ == "__main__": command() diff --git a/src/zonefile_migrate/to_terraform.py b/src/zonefile_migrate/to_terraform.py new file mode 100644 index 0000000..56f6bae --- /dev/null +++ b/src/zonefile_migrate/to_terraform.py @@ -0,0 +1,123 @@ +import click +import json +import logging +import re +import sys +import os +import pkgutil +from zonefile_migrate.utils import convert_zonefiles, target_file + + +from pathlib import Path +from ruamel.yaml import YAML, CommentedMap +from zonefile_migrate.logger import logging +from easyzone import easyzone +from dns.exception import SyntaxError +from zonefile_migrate.dns_record_set import create_from_zone +from jinja2 import Template +from zonefile_migrate.utils import get_all_zonefiles_in_path + +tf_managed_zone_template = """ +module {{ resource_name }} { + source = "./{{ provider }}-managed-zone" + domain_name = "{{ domain_name }}" + resource_record_sets = [{% for record in resource_record_sets %} + { + name = "{{ record.name }}" + type = "{{ record.rectype }}" + ttl = {{ record.ttl }} + rrdatas = [{% for rrdata in record.rrdatas %} + "{{ rrdata }}",{% endfor %} + ] + },{% endfor %} + ] +} +""" + + +def convert_to_terraform(zone: easyzone.Zone, provider: str) -> str: + """ + Converts the zonefile into a terraform tempalte for Google + """ + domain_name = zone.domain + resource_name = re.sub(r"\.", "_", zone.domain.removesuffix(".")) + resource_record_sets = list( + filter( + lambda r: r.rectype == "SOA" + or (r.rectype == "NS" and r.name == zone.domain), + create_from_zone(zone), + ) + ) + template = Template(tf_managed_zone_template) + + return template.render( + { + "domain_name": domain_name, + "resource_name": resource_name, + "provider": provider, + "resource_record_sets": resource_record_sets, + } + ) + + +@click.command(name="to-terraform") +@click.option( + "--provider", + required=False, + default="google", + help="name of provider to generate the managed zone for", +) +@click.argument("src", nargs=-1, type=click.Path()) +@click.argument("dst", nargs=1, type=click.Path()) +def command(provider, src, dst): + """ + Converts one or more `SRC` zonefiles into Terraform templates in `DST`. + + Each generated Terraform template contains a single hosted zone and all + associated resource record sets. The SOA and NS records for the origin domain are not + copied into the template. + + The zonefiles must contain a $ORIGIN and $TTL statement. If the SRC points to a directory + all files which contain one of these statements will be converted. If a $ORIGIN is missing, + the name of the file will be used as the domain name. + """ + tf_module_template = Path(__file__).parent.joinpath( + f"terraform-modules/{provider}-managed-zone.tf" + ) + if not tf_module_template.exists(): + raise click.UsageError(f"provider {provider} is not supported") + + if not src: + raise click.UsageError("no source files were specified") + + try: + inputs = get_all_zonefiles_in_path(src) + if len(inputs) == 0: + raise click.UsageError("no zonefiles were found") + except ValueError as error: + raise click.UsageError(error) + + dst = Path(dst) + if len(inputs) > 1: + if dst.exists() and not dst.is_dir(): + raise click.UsageError(f"{dst} is not a directory") + if not dst.exists(): + dst.mkdir(parents=True, exist_ok=True) + + outputs = list(map(lambda d: target_file(d, dst, ".tf"), inputs)) + + if dst.is_dir(): + main_path = dst.joinpath(f"{provider}-managed-zone/main.tf") + if not main_path.exists(): + main_path.parent.mkdir(exist_ok=True) + main_path.write_bytes(tf_module_template.read_bytes()) + + def _transform_to_terraform(zone: easyzone.Zone, output: Path): + with output.open("w") as file: + file.write(convert_to_terraform(zone, provider)) + + convert_zonefiles(inputs, outputs, _transform_to_terraform) + + +if __name__ == "__main__": + command() diff --git a/src/zonefile_migrate/utils.py b/src/zonefile_migrate/utils.py new file mode 100644 index 0000000..b098580 --- /dev/null +++ b/src/zonefile_migrate/utils.py @@ -0,0 +1,78 @@ +import re +from pathlib import Path +from typing import Callable, List +from easyzone import easyzone +from zonefile_migrate.logger import log + + +def is_zonefile(path: Path) -> bool: + """ + returns true if the file pointed to by `path` contains a $ORIGIN or $TTL pragma, otherwise False + """ + if path.exists() and path.is_file(): + with path.open("r") as file: + for line in file: + if re.search(r"^\s*\$(ORIGIN|TTL)\s+", line, re.IGNORECASE): + return True + return False + + +def get_all_zonefiles_in_path(src: [Path]) -> [Path]: + """ + creates a list of filenames of zonefiles from a list of paths. If the path + is not a directory, the path will be added as is. If the path points to a + directory, the files of the directory will be scanned to determined if they + are a zonefile (see is_zonefile) + """ + inputs = [] + for filename in map(lambda s: Path(s), src): + if not filename.exists(): + raise ValueError(f"{filename} does not exist") + + if filename.is_dir(): + inputs.extend( + [f for f in filename.iterdir() if f.is_file() and is_zonefile(f)] + ) + else: + inputs.append(filename) + return inputs + + +def convert_zonefiles( + inputs: [Path], outputs: [Path], transform: Callable[[easyzone.Zone, Path], None] +): + for i, input in enumerate(map(lambda s: Path(s), inputs)): + with input.open("r") as file: + content = file.read() + found = re.search( + r"\$ORIGIN\s+(?P.*)\s*", + content, + re.MULTILINE | re.IGNORECASE, + ) + if found: + domain_name = found.group("domain_name") + else: + domain_name = input.name.removesuffix(".zone") + log.warning( + "could not find $ORIGIN from zone file %s, using %s", + input, + domain_name, + ) + + try: + log.info("reading zonefile %s", input.as_posix()) + zone = easyzone.zone_from_file(domain_name, input.as_posix()) + transform(zone, outputs[i]) + except SyntaxError as error: + log.error(error) + exit(1) + + +def target_file(src: Path, dst: Path, extension: str) -> Path: + if dst.is_file(): + return dst + + if src.suffix == ".zone": + return dst.joinpath(src.name).with_suffix(extension) + + return dst.joinpath(src.name + extension) diff --git a/tests/test_convert_zonefile_to_cloudformation.py b/tests/test_convert_zonefile_to_cloudformation.py index 396439b..cf2cc4d 100644 --- a/tests/test_convert_zonefile_to_cloudformation.py +++ b/tests/test_convert_zonefile_to_cloudformation.py @@ -89,7 +89,7 @@ def test_something(self): expected_resource_records = list( map( lambda i: " ".join(map(lambda j: str(j), i)) - if isinstance(i, tuple) + if isinstance(i, (list, tuple)) else i, records.items, ) diff --git a/tests/test_convert_zonefile_to_dns_record_sets.py b/tests/test_convert_zonefile_to_dns_record_sets.py new file mode 100644 index 0000000..d80b628 --- /dev/null +++ b/tests/test_convert_zonefile_to_dns_record_sets.py @@ -0,0 +1,83 @@ +import unittest +import tempfile +import os +from zonefile_migrate.dns_record_set import create_from_zone +from easyzone import easyzone + + +zonefile = """ +$ORIGIN asample.org. +$TTL 86400 +@ SOA dns1.asample.org. hostmaster.asample.org. ( + 2001062501 ; serial + 21600 ; refresh after 6 hours + 3600 ; retry after 1 hour + 604800 ; expire after 1 week + 86400 ) ; minimum TTL of 1 day + ; +; + NS dns1.asample.org. + NS dns2.asample.org. +dns1 A 10.0.1.1 + AAAA aaaa:bbbb::1 +dns2 A 10.0.1.2 + AAAA aaaa:bbbb::2 +; +; +@ MX 10 mail.asample.org. + MX 20 mail2.asample.org. +mail A 10.0.1.5 + AAAA aaaa:bbbb::5 +mail2 A 10.0.1.6 + AAAA aaaa:bbbb::6 +; +; +; This sample zone file illustrates sharing the same IP addresses for multiple services: +; +services 60 A 10.0.1.10 + AAAA aaaa:bbbb::10 + A 10.0.1.11 + AAAA aaaa:bbbb::11 + +ftp CNAME services.asample.org. +www CNAME services.asample.org. +@ TXT "txt=beautiful" +; +; +""" + + +class ConvertToDNSRecordSetTestCase(unittest.TestCase): + def setUp(self) -> None: + filename = tempfile.NamedTemporaryFile() + with open(filename.name, "w") as file: + file.write(zonefile) + file.flush() + self.zone = easyzone.zone_from_file("asample.org", filename.name) + + def test_create_from_zone(self): + record_sets = create_from_zone(self.zone) + self.assertEqual(16, len(record_sets)) + for record_set in record_sets: + name = self.zone.names.get(record_set.name) + self.assertTrue(name, f"{record_set.name} not found in zone") + + self.assertEqual(name.ttl, record_set.ttl) + records = name.records(record_set.rectype) + self.assertTrue( + records, + f"no records found of type {record_set.rectype} for {record_set.name} in zone", + ) + expected_resource_records = list( + map( + lambda i: " ".join(map(lambda j: str(j), i)) + if isinstance(i, (list, tuple)) + else i, + records.items, + ) + ) + self.assertEqual(expected_resource_records, record_set.rrdatas) + + +if __name__ == "__main__": + unittest.main()