diff --git a/README.md b/README.md index be2ddc0..5e7cb50 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ Not supported, yet: This Terraform module is the part of [serverless.tf framework](https://github.com/antonbabenko/serverless.tf), which aims to simplify all operations when working with the serverless in Terraform: -1. Build and install dependencies - [read more](#build). +1. Build and install dependencies - [read more](#build). Requires Python 3.6 or newer. 2. Create, store, and use deployment packages - [read more](#package). 3. Create, update, and publish AWS Lambda Function and Lambda Layer - [see usage](#usage). 4. Create static and dynamic aliases for AWS Lambda Function - [see usage](#usage), see [modules/alias](https://github.com/terraform-aws-modules/terraform-aws-lambda/tree/master/modules/alias). @@ -306,7 +306,7 @@ module "lambda" { This is one of the most complicated part done by the module and normally you don't have to know internals. -`package.py` is Python script which does it. Make sure, Python 3.7 or newer is installed. The main functions of the script are to generate a filename of zip-archive based on the content of the files, verify if zip-archive has been already created, and create zip-archive only when it is necessary (during `apply`, not `plan`). +`package.py` is Python script which does it. Make sure, Python 3.6 or newer is installed. The main functions of the script are to generate a filename of zip-archive based on the content of the files, verify if zip-archive has been already created, and create zip-archive only when it is necessary (during `apply`, not `plan`). Hash of zip-archive created with the same content of the files is always identical which prevents unnecessary force-updates of the Lambda resources unless content modifies. If you need to have different filenames for the same content you can specify extra string argument `hash_extra`. @@ -315,7 +315,7 @@ When calling this module multiple times in one execution to create packages with ## Debug -Building and packaging has been historically hard to debug (especially with Terraform), so we made an effort to make it easier for user to see debug info. There are 3 different debug levels: `DEBUG` - to see only what is happening during planning phase, `DEBUG2` - to see all logging values, `DEBUG3` - to see all logging values and env variables (be careful sharing your env variables as they may contain secrets!). +Building and packaging has been historically hard to debug (especially with Terraform), so we made an effort to make it easier for user to see debug info. There are 3 different debug levels: `DEBUG` - to see only what is happening during planning phase and how a zip file content filtering in case of applied patterns, `DEBUG2` - to see more logging output, `DEBUG3` - to see all logging values, `DUMP_ENV` - to see all logging values and env variables (be careful sharing your env variables as they may contain secrets!). User can specify debug level like this: diff --git a/package.py b/package.py index 7cc8c13..c7590b8 100644 --- a/package.py +++ b/package.py @@ -2,8 +2,8 @@ import sys -if sys.version_info < (3, 7): - raise RuntimeError("A python version 3.7 or newer is required") +if sys.version_info < (3, 6): + raise RuntimeError("A python version 3.6 or newer is required") import os import re @@ -26,12 +26,15 @@ import logging PY38 = sys.version_info >= (3, 8) +PY37 = sys.version_info >= (3, 7) +PY36 = sys.version_info >= (3, 6) ################################################################################ # Logging DEBUG2 = 9 DEBUG3 = 8 +DUMP_ENV = 1 log_handler = None log = logging.getLogger() @@ -43,6 +46,7 @@ def configure_logging(use_tf_stderr=False): logging.addLevelName(DEBUG2, 'DEBUG2') logging.addLevelName(DEBUG3, 'DEBUG3') + logging.addLevelName(DUMP_ENV, 'DUMP_ENV') class LogFormatter(logging.Formatter): default_format = '%(message)s' @@ -139,28 +143,29 @@ def list_files(top_path, log=None): return results -def dataclass(name, **fields): - typ = type(name, (object,), { - '__slots__': fields.keys(), - '__getattr__': lambda *_: None, +def dataclass(name): + typ = type(name, (dict,), { + '__getattr__': lambda self, x: self.get(x), + '__init__': lambda self, **k: self.update(k), }) - for k, v in fields.items(): - setattr(typ, k, v) return typ def datatree(name, **fields): - def decode_json(v): + def decode_json(k, v): if v and isinstance(v, str) and v[0] in '"[{': try: - return json.loads(v) + o = json.loads(v) + if isinstance(o, dict): + return dataclass(k)(**o) + return o except json.JSONDecodeError: pass return v - return dataclass(name, **dict((( - k, datatree(k, **v) if isinstance(v, dict) else decode_json(v)) - for k, v in fields.items())))() + return dataclass(name)(**dict((( + k, datatree(k, **v) if isinstance(v, dict) else decode_json(k, v)) + for k, v in fields.items()))) def timestamp_now_ns(): @@ -291,7 +296,11 @@ def __enter__(self): return self.open() def __exit__(self, exc_type, exc_val, exc_tb): - self.close(failed=exc_type is not None) + if exc_type is not None: + self._log.exception("Error during zip archive creation") + self.close(failed=True) + raise SystemExit(1) + self.close() def _ensure_open(self): if self._zip is not None: @@ -380,10 +389,11 @@ def _write_zinfo(self, zinfo, filename, else: zinfo.compress_type = self._compress_type - if compresslevel is not None: - zinfo._compresslevel = compresslevel - else: - zinfo._compresslevel = self._compresslevel + if PY37: + if compresslevel is not None: + zinfo._compresslevel = compresslevel + else: + zinfo._compresslevel = self._compresslevel if zinfo.is_dir(): with zip._lock: @@ -438,9 +448,9 @@ def _zinfo_from_file(filename, arcname=None, *, strict_timestamps=True): isdir = stat.S_ISDIR(st.st_mode) mtime = time.localtime(st.st_mtime) date_time = mtime[0:6] - if not strict_timestamps and date_time[0] < 1980: + if strict_timestamps and date_time[0] < 1980: date_time = (1980, 1, 1, 0, 0, 0) - elif not strict_timestamps and date_time[0] > 2107: + elif strict_timestamps and date_time[0] > 2107: date_time = (2107, 12, 31, 23, 59, 59) # Create ZipInfo instance to store file information if arcname is None: @@ -631,11 +641,19 @@ def pip_requirements_step(path, prefix=None, required=False): hash(requirements) def commands_step(path, commands): - path = os.path.normpath(path) + if path: + path = os.path.normpath(path) batch = [] for c in commands: if isinstance(c, str): if c.startswith(':zip'): + if path: + hash(path) + else: + # If path doesn't defined for a block with + # commands it will be set to Terraform's + # current working directory + path = query.paths.cwd if batch: step('sh', path, '\n'.join(batch)) batch.clear() @@ -644,15 +662,18 @@ def commands_step(path, commands): _, _path, prefix = c prefix = prefix.strip() _path = os.path.normpath(os.path.join(path, _path)) - step('zip', _path, prefix) + step('zip:embedded', _path, prefix) + elif len(c) == 2: + prefix = None + _, _path = c + step('zip:embedded', _path, prefix) elif len(c) == 1: prefix = None - step('zip', path, prefix) + step('zip:embedded', path, prefix) else: raise ValueError( - ':zip command can have zero ' - 'or 2 arguments: {}'.format(c)) - hash(path) + ":zip invalid call signature, use: " + "':zip [path [prefix_in_zip]]'") else: batch.append(c) @@ -706,30 +727,30 @@ def execute(self, build_plan, zip_stream, query): for action in build_plan: cmd = action[0] - if cmd == 'zip': + if cmd.startswith('zip'): + ts = 0 if cmd == 'zip:embedded' else None source_path, prefix = action[1:] if sh_work_dir: if source_path != sh_work_dir: source_path = sh_work_dir - if pf: - for path in pf.filter(source_path, prefix): - if os.path.isdir(source_path): - arcname = os.path.relpath(path, source_path) - else: - arcname = os.path.basename(path) - zs.write_file(path, prefix, arcname) + if pf: + self._zip_write_with_filter(zs, pf, source_path, prefix, + timestamp=ts) else: if os.path.isdir(source_path): - zs.write_dirs(source_path, prefix=prefix) + zs.write_dirs(source_path, prefix=prefix, timestamp=ts) else: - zs.write_file(source_path, prefix=prefix) + zs.write_file(source_path, prefix=prefix, timestamp=ts) elif cmd == 'pip': runtime, pip_requirements, prefix = action[1:] - with install_pip_requirements(query, zs, - pip_requirements) as rd: + with install_pip_requirements(query, pip_requirements) as rd: if rd: - # XXX: timestamp=0 - what actually do with it? - zs.write_dirs(rd, prefix=prefix, timestamp=0) + if pf: + self._zip_write_with_filter(zs, pf, rd, prefix, + timestamp=0) + else: + # XXX: timestamp=0 - what actually do with it? + zs.write_dirs(rd, prefix=prefix, timestamp=0) elif cmd == 'sh': r, w = os.pipe() side_ch = os.fdopen(r) @@ -750,9 +771,22 @@ def execute(self, build_plan, zip_stream, query): elif cmd == 'clear:filter': pf = None + @staticmethod + def _zip_write_with_filter(zip_stream, path_filter, source_path, prefix, + timestamp=None): + for path in path_filter.filter(source_path, prefix): + if os.path.isdir(source_path): + arcname = os.path.relpath(path, source_path) + else: + arcname = os.path.basename(path) + zip_stream.write_file(path, prefix, arcname, timestamp=timestamp) + @contextmanager -def install_pip_requirements(query, zip_stream, requirements_file): +def install_pip_requirements(query, requirements_file): + # TODO: + # 1. Emit files instead of temp_dir + if not os.path.exists(requirements_file): yield return @@ -892,10 +926,15 @@ def prepare_command(args): # Load the query. query_data = json.load(sys.stdin) - if log.isEnabledFor(DEBUG3): + if log.isEnabledFor(DUMP_ENV): log.debug('ENV: %s', json.dumps(dict(os.environ), indent=2)) if log.isEnabledFor(DEBUG2): - log.debug('QUERY: %s', json.dumps(query_data, indent=2)) + if log.isEnabledFor(DEBUG3): + log.debug('QUERY: %s', json.dumps(query_data, indent=2)) + else: + log_excludes = ('source_path', 'hash_extra_paths', 'paths') + qd = {k: v for k, v in query_data.items() if k not in log_excludes} + log.debug('QUERY (excerpt): %s', json.dumps(qd, indent=2)) query = datatree('prepare_query', **query_data) @@ -944,7 +983,6 @@ def prepare_command(args): build_data = { 'filename': filename, 'runtime': runtime, - 'source_path': source_path, 'artifacts_dir': artifacts_dir, 'build_plan': build_plan, } @@ -1050,7 +1088,7 @@ def zip_cmd(args): subprocess.call([zipinfo, args.zipfile]) log.debug('-' * 80) log.debug('Source code hash: %s', - source_code_hash(open(args.zipfile, 'rb').read())) + source_code_hash(open(args.zipfile, 'rb').read())) p = hidden_parser('zip', help='Zip folder with provided files timestamp') p.set_defaults(command=zip_cmd)