From 7d630293a632fa4b0ba2e12d2a05bffd4123fc14 Mon Sep 17 00:00:00 2001 From: Fantix King Date: Wed, 2 Nov 2022 17:49:35 -0400 Subject: [PATCH 01/40] edgedb-python 1.1.0 Codegen Fixes ============= * Add missing std::json (#387) * Support optional argument (#387) * Fix camelcase generation (#387) * Allow symlinks in project dir (#387) * Use Executor on generated code for Client/Tx (#390) --- edgedb/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edgedb/_version.py b/edgedb/_version.py index 275c66aa..fbf380d1 100644 --- a/edgedb/_version.py +++ b/edgedb/_version.py @@ -28,4 +28,4 @@ # supported platforms, publish the packages on PyPI, merge the PR # to the target branch, create a Git tag pointing to the commit. -__version__ = '1.0.0' +__version__ = '1.1.0' From 4b8bec67afacbab3df1e6e4e2480e569ba08230b Mon Sep 17 00:00:00 2001 From: Quin Lynch Date: Mon, 16 May 2022 01:47:53 -0300 Subject: [PATCH 02/40] remove references to unix-domain sockets --- docs/api/asyncio_client.rst | 14 ++------------ docs/api/blocking_client.rst | 14 ++------------ 2 files changed, 4 insertions(+), 24 deletions(-) diff --git a/docs/api/asyncio_client.rst b/docs/api/asyncio_client.rst index 54afccf5..97bf7623 100644 --- a/docs/api/asyncio_client.rst +++ b/docs/api/asyncio_client.rst @@ -44,26 +44,16 @@ Client the :ref:`DSN Specification `. :param host: - Database host address as one of the following: - - - an IP address or a domain name; - - an absolute path to the directory containing the database - server Unix-domain socket (not supported on Windows); - - a sequence of any of the above, in which case the addresses - will be tried in order, and the host of the first successful - connection will be used for the whole connection pool. + Database host address as an IP address or a domain name; If not specified, the following will be tried, in order: - host address(es) parsed from the *dsn* argument, - the value of the ``EDGEDB_HOST`` environment variable, - - on Unix, common directories used for EdgeDB Unix-domain - sockets: ``"/run/edgedb"`` and ``"/var/run/edgedb"``, - ``"localhost"``. :param port: - Port number to connect to at the server host - (or Unix-domain socket file extension). If multiple host + Port number to connect to at the server host. If multiple host addresses were specified, this parameter may specify a sequence of port numbers of the same length as the host sequence, or it may specify a single port number to be used for all host diff --git a/docs/api/blocking_client.rst b/docs/api/blocking_client.rst index f50456c9..7ab26507 100644 --- a/docs/api/blocking_client.rst +++ b/docs/api/blocking_client.rst @@ -44,26 +44,16 @@ Client the :ref:`DSN Specification `. :param host: - Database host address as one of the following: - - - an IP address or a domain name; - - an absolute path to the directory containing the database - server Unix-domain socket (not supported on Windows); - - a sequence of any of the above, in which case the addresses - will be tried in order, and the host of the first successful - connection will be used for the whole connection pool. + Database host address as an IP address or a domain name; If not specified, the following will be tried, in order: - host address(es) parsed from the *dsn* argument, - the value of the ``EDGEDB_HOST`` environment variable, - - on Unix, common directories used for EdgeDB Unix-domain - sockets: ``"/run/edgedb"`` and ``"/var/run/edgedb"``, - ``"localhost"``. :param port: - Port number to connect to at the server host - (or Unix-domain socket file extension). If multiple host + Port number to connect to at the server host. If multiple host addresses were specified, this parameter may specify a sequence of port numbers of the same length as the host sequence, or it may specify a single port number to be used for all host From 8b289473761389b0b4c28279c2a78039a9907aca Mon Sep 17 00:00:00 2001 From: Fantix King Date: Fri, 18 Nov 2022 09:05:05 -0500 Subject: [PATCH 03/40] Handle ErrorResponse in ping (#393) * Handle IdleSessionTimeoutError in blocking client reconnect --- edgedb/blocking_client.py | 4 ++-- edgedb/protocol/protocol.pyx | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/edgedb/blocking_client.py b/edgedb/blocking_client.py index 4d35c43f..05e8b95c 100644 --- a/edgedb/blocking_client.py +++ b/edgedb/blocking_client.py @@ -152,8 +152,8 @@ async def raw_query(self, query_context: abstract.QueryContext): time.monotonic() - self._protocol.last_active_timestamp > self._ping_wait_time ): - await self._protocol._sync() - except errors.ClientConnectionError: + await self._protocol.ping() + except (errors.IdleSessionTimeoutError, errors.ClientConnectionError): await self.connect() return await super().raw_query(query_context) diff --git a/edgedb/protocol/protocol.pyx b/edgedb/protocol/protocol.pyx index 6bc5cd75..856c96c5 100644 --- a/edgedb/protocol/protocol.pyx +++ b/edgedb/protocol/protocol.pyx @@ -711,6 +711,27 @@ cdef class SansIOProtocol: else: self.fallthrough() + async def ping(self): + cdef char mtype + self.write(WriteBuffer.new_message(SYNC_MSG).end_message()) + exc = None + while True: + if not self.buffer.take_message(): + await self.wait_for_message() + mtype = self.buffer.get_message_type() + + if mtype == READY_FOR_COMMAND_MSG: + self.parse_sync_message() + break + elif mtype == ERROR_RESPONSE_MSG: + exc = self.parse_error_message() + self.buffer.finish_message() + break + else: + self.fallthrough() + if exc is not None: + raise exc + async def restore(self, bytes header, data_gen): cdef: WriteBuffer buf From a2bec180bc8e34cc5892a9d5040db3d4332f412f Mon Sep 17 00:00:00 2001 From: Fantix King Date: Fri, 18 Nov 2022 12:38:21 -0500 Subject: [PATCH 04/40] Output pretty error if possible (#399) Error position and hint are now included by default if present, overridden by EDGEDB_ERROR_HINT (enabled/disabled). The output is aslo colored if the stderr refers to a terminal, overriden by EDGEDB_COLOR_OUTPUT (auto/enabled/disabled). --- edgedb/color.py | 60 +++++++++++++++ edgedb/errors/_base.py | 138 ++++++++++++++++++++++++++++++++++- edgedb/protocol/protocol.pyx | 2 + 3 files changed, 196 insertions(+), 4 deletions(-) create mode 100644 edgedb/color.py diff --git a/edgedb/color.py b/edgedb/color.py new file mode 100644 index 00000000..1e95c7bf --- /dev/null +++ b/edgedb/color.py @@ -0,0 +1,60 @@ +import os +import sys +import warnings + +COLOR = None + + +class Color: + HEADER = "" + BLUE = "" + CYAN = "" + GREEN = "" + WARNING = "" + FAIL = "" + ENDC = "" + BOLD = "" + UNDERLINE = "" + + +def get_color() -> Color: + global COLOR + + if COLOR is None: + COLOR = Color() + if type(USE_COLOR) is bool: + use_color = USE_COLOR + else: + try: + use_color = USE_COLOR() + except Exception: + use_color = False + if use_color: + COLOR.HEADER = '\033[95m' + COLOR.BLUE = '\033[94m' + COLOR.CYAN = '\033[96m' + COLOR.GREEN = '\033[92m' + COLOR.WARNING = '\033[93m' + COLOR.FAIL = '\033[91m' + COLOR.ENDC = '\033[0m' + COLOR.BOLD = '\033[1m' + COLOR.UNDERLINE = '\033[4m' + + return COLOR + + +try: + USE_COLOR = { + "default": lambda: sys.stderr.isatty(), + "auto": lambda: sys.stderr.isatty(), + "enabled": True, + "disabled": False, + }[ + os.getenv("EDGEDB_COLOR_OUTPUT", "default") + ] +except KeyError: + warnings.warn( + "EDGEDB_COLOR_OUTPUT can only be one of: " + "default, auto, enabled or disabled" + ) + USE_COLOR = False diff --git a/edgedb/errors/_base.py b/edgedb/errors/_base.py index 9dec14f6..5756f264 100644 --- a/edgedb/errors/_base.py +++ b/edgedb/errors/_base.py @@ -17,6 +17,12 @@ # +import io +import os +import traceback +import unicodedata +import warnings + __all__ = ( 'EdgeDBError', 'EdgeDBMessage', ) @@ -79,6 +85,7 @@ class EdgeDBErrorMeta(Meta): class EdgeDBError(Exception, metaclass=EdgeDBErrorMeta): _code = None + _query = None tags = frozenset() def __init__(self, *args, **kwargs): @@ -93,15 +100,25 @@ def _position(self): # not a stable API method return int(self._read_str_field(FIELD_POSITION_START, -1)) + @property + def _position_start(self): + # not a stable API method + return int(self._read_str_field(FIELD_CHARACTER_START, -1)) + + @property + def _position_end(self): + # not a stable API method + return int(self._read_str_field(FIELD_CHARACTER_END, -1)) + @property def _line(self): # not a stable API method - return int(self._read_str_field(FIELD_LINE, -1)) + return int(self._read_str_field(FIELD_LINE_START, -1)) @property def _col(self): # not a stable API method - return int(self._read_str_field(FIELD_COLUMN, -1)) + return int(self._read_str_field(FIELD_COLUMN_START, -1)) @property def _hint(self): @@ -127,6 +144,35 @@ def _from_code(code, *args, **kwargs): exc._code = code return exc + def __str__(self): + msg = super().__str__() + if SHOW_HINT and self._query and self._position_start >= 0: + try: + return _format_error( + msg, + self._query, + self._position_start, + max(1, self._position_end - self._position_start), + self._line if self._line > 0 else "?", + self._col if self._col > 0 else "?", + self._hint or "error", + ) + except Exception: + return "".join( + ( + msg, + LINESEP, + LINESEP, + "During formatting of the above exception, " + "another exception occurred:", + LINESEP, + LINESEP, + traceback.format_exc(), + ) + ) + else: + return msg + def _lookup_cls(code: int, *, meta: type, default: type): try: @@ -180,6 +226,68 @@ def _severity_name(severity): return 'PANIC' +def _format_error(msg, query, start, offset, line, col, hint): + c = get_color() + rv = io.StringIO() + rv.write(f"{c.BOLD}{msg}{c.ENDC}{LINESEP}") + lines = query.splitlines(keepends=True) + num_len = len(str(len(lines))) + rv.write(f"{c.BLUE}{'':>{num_len}} ┌─{c.ENDC} query:{line}:{col}{LINESEP}") + rv.write(f"{c.BLUE}{'':>{num_len}} │ {c.ENDC}{LINESEP}") + for num, line in enumerate(lines): + length = len(line) + line = line.rstrip() # we'll use our own line separator + if start >= length: + # skip lines before the error + start -= length + continue + + if start >= 0: + # Error starts in current line, write the line before the error + first_half = repr(line[:start])[1:-1] + line = line[start:] + length -= start + rv.write(f"{c.BLUE}{num + 1:>{num_len}} │ {c.ENDC}{first_half}") + start = _unicode_width(first_half) + else: + # Multi-line error continues + rv.write(f"{c.BLUE}{num + 1:>{num_len}} │ {c.FAIL}│ {c.ENDC}") + + if offset > length: + # Error is ending beyond current line + line = repr(line)[1:-1] + rv.write(f"{c.FAIL}{line}{c.ENDC}{LINESEP}") + if start >= 0: + # Multi-line error starts + rv.write(f"{c.BLUE}{'':>{num_len}} │ " + f"{c.FAIL}╭─{'─' * start}^{c.ENDC}{LINESEP}") + offset -= length + start = -1 # mark multi-line + else: + # Error is ending within current line + first_half = repr(line[:offset])[1:-1] + line = repr(line[offset:])[1:-1] + rv.write(f"{c.FAIL}{first_half}{c.ENDC}{line}{LINESEP}") + size = _unicode_width(first_half) + if start >= 0: + # Mark single-line error + rv.write(f"{c.BLUE}{'':>{num_len}} │ {' ' * start}" + f"{c.FAIL}{'^' * size} {hint}{c.ENDC}") + else: + # End of multi-line error + rv.write(f"{c.BLUE}{'':>{num_len}} │ " + f"{c.FAIL}╰─{'─' * (size - 1)}^ {hint}{c.ENDC}") + break + return rv.getvalue() + + +def _unicode_width(text): + return sum( + 2 if unicodedata.east_asian_width(c) == "W" else 1 + for c in unicodedata.normalize("NFC", text) + ) + + FIELD_HINT = 0x_00_01 FIELD_DETAILS = 0x_00_02 FIELD_SERVER_TRACEBACK = 0x_01_01 @@ -187,8 +295,14 @@ def _severity_name(severity): # XXX: Subject to be changed/deprecated. FIELD_POSITION_START = 0x_FF_F1 FIELD_POSITION_END = 0x_FF_F2 -FIELD_LINE = 0x_FF_F3 -FIELD_COLUMN = 0x_FF_F4 +FIELD_LINE_START = 0x_FF_F3 +FIELD_COLUMN_START = 0x_FF_F4 +FIELD_UTF16_COLUMN_START = 0x_FF_F5 +FIELD_LINE_END = 0x_FF_F6 +FIELD_COLUMN_END = 0x_FF_F7 +FIELD_UTF16_COLUMN_END = 0x_FF_F8 +FIELD_CHARACTER_START = 0x_FF_F9 +FIELD_CHARACTER_END = 0x_FF_FA EDGE_SEVERITY_DEBUG = 20 @@ -198,3 +312,19 @@ def _severity_name(severity): EDGE_SEVERITY_ERROR = 120 EDGE_SEVERITY_FATAL = 200 EDGE_SEVERITY_PANIC = 255 + + +LINESEP = os.linesep + +try: + SHOW_HINT = {"default": True, "enabled": True, "disabled": False}[ + os.getenv("EDGEDB_ERROR_HINT", "default") + ] +except KeyError: + warnings.warn( + "EDGEDB_ERROR_HINT can only be one of: default, enabled or disabled" + ) + SHOW_HINT = False + + +from edgedb.color import get_color diff --git a/edgedb/protocol/protocol.pyx b/edgedb/protocol/protocol.pyx index 856c96c5..b6b86295 100644 --- a/edgedb/protocol/protocol.pyx +++ b/edgedb/protocol/protocol.pyx @@ -305,6 +305,7 @@ cdef class SansIOProtocol: elif mtype == ERROR_RESPONSE_MSG: exc = self.parse_error_message() + exc._query = query exc = self._amend_parse_error( exc, output_format, expect_one, required_one) @@ -435,6 +436,7 @@ cdef class SansIOProtocol: elif mtype == ERROR_RESPONSE_MSG: exc = self.parse_error_message() + exc._query = query if exc.get_code() == parameter_type_mismatch_code: if not isinstance(in_dc, NullCodec): buf = WriteBuffer.new() From 26fb6d8e2d5001f5c16d361a158683e2aab467d6 Mon Sep 17 00:00:00 2001 From: Fantix King Date: Fri, 18 Nov 2022 18:31:27 -0500 Subject: [PATCH 05/40] Disallow None in elements of array argument --- edgedb/protocol/codecs/array.pyx | 5 ++++- tests/test_async_query.py | 6 ++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/edgedb/protocol/codecs/array.pyx b/edgedb/protocol/codecs/array.pyx index 2f709531..34cc567b 100644 --- a/edgedb/protocol/codecs/array.pyx +++ b/edgedb/protocol/codecs/array.pyx @@ -60,7 +60,10 @@ cdef class BaseArrayCodec(BaseCodec): for i in range(objlen): item = obj[i] if item is None: - elem_data.write_int32(-1) + raise ValueError( + "invalid array element at index {}: " + "None is not allowed".format(i) + ) else: try: self.sub_codec.encode(elem_data, item) diff --git a/tests/test_async_query.py b/tests/test_async_query.py index af334216..e51ac197 100644 --- a/tests/test_async_query.py +++ b/tests/test_async_query.py @@ -380,6 +380,12 @@ async def test_async_args_03(self): 'combine positional and named parameters'): await self.client.query('select $0 + $bar;') + with self.assertRaisesRegex(edgedb.InvalidArgumentError, + "None is not allowed"): + await self.client.query( + "select >$0", [1, None, 3] + ) + async def test_async_args_04(self): aware_datetime = datetime.datetime.now(datetime.timezone.utc) naive_datetime = datetime.datetime.now() From 6bce57e25ed9b97c6666c9437072a84689efbfbf Mon Sep 17 00:00:00 2001 From: Fantix King Date: Mon, 21 Nov 2022 14:06:42 -0500 Subject: [PATCH 06/40] Codegen: allow providing a path after --file (#400) This is not a breaking change; `--file` itself without a path works the same way as it was before. * Also use color in codegen output --- edgedb/codegen/cli.py | 17 ++++++++++--- edgedb/codegen/generator.py | 49 +++++++++++++++++++++++-------------- 2 files changed, 45 insertions(+), 21 deletions(-) diff --git a/edgedb/codegen/cli.py b/edgedb/codegen/cli.py index 1a229bae..19966ca8 100644 --- a/edgedb/codegen/cli.py +++ b/edgedb/codegen/cli.py @@ -23,11 +23,21 @@ from . import generator -parser = argparse.ArgumentParser( +class ColoredArgumentParser(argparse.ArgumentParser): + def error(self, message): + c = generator.C + self.exit( + 2, + f"{c.BOLD}{c.FAIL}error:{c.ENDC} " + f"{c.BOLD}{message:s}{c.ENDC}\n", + ) + + +parser = ColoredArgumentParser( description="Generate Python code for .edgeql files." ) parser.add_argument("--dsn") -parser.add_argument("--credentials_file", metavar="PATH") +parser.add_argument("--credentials-file", metavar="PATH") parser.add_argument("-I", "--instance", metavar="NAME") parser.add_argument("-H", "--host") parser.add_argument("-P", "--port") @@ -42,7 +52,8 @@ ) parser.add_argument( "--file", - action="store_true", + action="append", + nargs="?", help="Generate a single file instead of one per .edgeql file.", ) parser.add_argument( diff --git a/edgedb/codegen/generator.py b/edgedb/codegen/generator.py index f357d50a..ff49b82e 100644 --- a/edgedb/codegen/generator.py +++ b/edgedb/codegen/generator.py @@ -29,8 +29,10 @@ from edgedb import abstract from edgedb import describe from edgedb.con_utils import find_edgedb_project_dir +from edgedb.color import get_color +C = get_color() SYS_VERSION_INFO = os.getenv("EDGEDB_PYTHON_CODEGEN_PY_VER") if SYS_VERSION_INFO: SYS_VERSION_INFO = tuple(map(int, SYS_VERSION_INFO.split(".")))[:2] @@ -88,13 +90,20 @@ def __get_validators__(cls): """ +def print_msg(msg): + print(msg, file=sys.stderr) + + +def print_error(msg): + print_msg(f"{C.BOLD}{C.FAIL}error: {C.ENDC}{C.BOLD}{msg}{C.ENDC}") + + def _get_conn_args(args: argparse.Namespace): if args.password_from_stdin: if args.password: - print( + print_error( "--password and --password-from-stdin are " "mutually exclusive", - file=sys.stderr, ) sys.exit(22) if sys.stdin.isatty(): @@ -104,7 +113,7 @@ def _get_conn_args(args: argparse.Namespace): else: password = args.password if args.dsn and args.instance: - print("--dsn and --instance are mutually exclusive", file=sys.stderr) + print_error("--dsn and --instance are mutually exclusive") sys.exit(22) return dict( dsn=args.dsn or args.instance, @@ -133,9 +142,9 @@ def __init__(self, args: argparse.Namespace): "codegen must be run under an EdgeDB project dir" ) sys.exit(2) - print(f"Found EdgeDB project: {self._project_dir}", file=sys.stderr) + print_msg(f"Found EdgeDB project: {C.BOLD}{self._project_dir}{C.ENDC}") self._client = edgedb.create_client(**_get_conn_args(args)) - self._file_mode = args.file + self._single_mode_files = args.file self._method_names = set() self._describe_results = [] @@ -165,11 +174,12 @@ def run(self): for target, suffix, is_async in SUFFIXES: if target in self._targets: self._async = is_async - if self._file_mode: + if self._single_mode_files: self._generate_single_file(suffix) else: self._generate_files(suffix) self._new_file() + print_msg(f"{C.GREEN}{C.BOLD}Done.{C.ENDC}") def _process_dir(self, dir_: pathlib.Path): for file_or_dir in dir_.iterdir(): @@ -184,13 +194,13 @@ def _process_dir(self, dir_: pathlib.Path): self._process_file(file_or_dir) def _process_file(self, source: pathlib.Path): - print(f"Processing {source}", file=sys.stderr) + print_msg(f"{C.BOLD}Processing{C.ENDC} {C.BLUE}{source}{C.ENDC}") with source.open() as f: query = f.read() name = source.stem - if self._file_mode: + if self._single_mode_files: if name in self._method_names: - print(f"Conflict method names: {name}", file=sys.stderr) + print_error(f"Conflict method names: {name}") sys.exit(17) self._method_names.add(name) dr = self._client._describe_query(query, inject_type_names=True) @@ -199,7 +209,7 @@ def _process_file(self, source: pathlib.Path): def _generate_files(self, suffix: str): for name, source, query, dr in self._describe_results: target = source.parent / f"{name}{suffix}" - print(f"Generating {target}", file=sys.stderr) + print_msg(f"{C.BOLD}Generating{C.ENDC} {C.BLUE}{target}{C.ENDC}") self._new_file() content = self._generate(name, query, dr) buf = io.StringIO() @@ -210,8 +220,7 @@ def _generate_files(self, suffix: str): f.write(buf.getvalue()) def _generate_single_file(self, suffix: str): - target = self._project_dir / f"{FILE_MODE_OUTPUT_FILE}{suffix}" - print(f"Generating {target}", file=sys.stderr) + print_msg(f"{C.BOLD}Generating single file output...{C.ENDC}") buf = io.StringIO() output = [] sources = [] @@ -225,8 +234,15 @@ def _generate_single_file(self, suffix: str): if i < len(output) - 1: print(file=buf) print(file=buf) - with target.open("w") as f: - f.write(buf.getvalue()) + + for target in self._single_mode_files: + if target: + target = pathlib.Path(target).absolute() + else: + target = self._project_dir / f"{FILE_MODE_OUTPUT_FILE}{suffix}" + print_msg(f"{C.BOLD}Writing{C.ENDC} {C.BLUE}{target}{C.ENDC}") + with target.open("w") as f: + f.write(buf.getvalue()) def _write_comments( self, f: io.TextIOBase, src: typing.List[pathlib.Path] @@ -510,10 +526,7 @@ def _find_name(self, name: str) -> str: name = new break else: - print( - f"Failed to find a unique name for: {name}", - file=sys.stderr, - ) + print_error(f"Failed to find a unique name for: {name}") sys.exit(17) self._names.add(name) return name From c4413a60ec836874c12d0aa33344c2974389a13a Mon Sep 17 00:00:00 2001 From: Fantix King Date: Wed, 23 Nov 2022 11:56:27 -0500 Subject: [PATCH 07/40] edgedb-python 1.2.0 Changes ======= * Output pretty error if possible (#399) (by @fantix in a2bec180 for #399) * Codegen: allow providing a path after --file (#400) (by @fantix in 6bce57e2 for #400) Fixes ===== * Handle ErrorResponse in ping (#393) (by @fantix in 8b289473 for #393) * Disallow None in elements of array argument (by @fantix in 26fb6d8e) Docs ==== * Remove references to unix-domain sockets (by @quinchs in 4b8bec67) --- edgedb/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edgedb/_version.py b/edgedb/_version.py index fbf380d1..4da184ee 100644 --- a/edgedb/_version.py +++ b/edgedb/_version.py @@ -28,4 +28,4 @@ # supported platforms, publish the packages on PyPI, merge the PR # to the target branch, create a Git tag pointing to the commit. -__version__ = '1.1.0' +__version__ = '1.2.0' From 33a2f6a6ee8cc4a7b59fa123b25136c7b582b6a3 Mon Sep 17 00:00:00 2001 From: Fantix King Date: Wed, 23 Nov 2022 12:07:11 -0500 Subject: [PATCH 08/40] CRF: unicode width Co-authored-by: Paul Colomiets --- edgedb/errors/_base.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/edgedb/errors/_base.py b/edgedb/errors/_base.py index 5756f264..675ef567 100644 --- a/edgedb/errors/_base.py +++ b/edgedb/errors/_base.py @@ -282,10 +282,9 @@ def _format_error(msg, query, start, offset, line, col, hint): def _unicode_width(text): - return sum( - 2 if unicodedata.east_asian_width(c) == "W" else 1 - for c in unicodedata.normalize("NFC", text) - ) + return sum(0 if unicodedata.category(c) in ('Mn', 'Cf') else + 2 if unicodedata.east_asian_width(c) == "W" else 1 + for c in text) FIELD_HINT = 0x_00_01 From df1f6fb34ddfa3b057124cddd59e269ff629a01a Mon Sep 17 00:00:00 2001 From: Fantix King Date: Wed, 28 Dec 2022 16:25:59 -0500 Subject: [PATCH 09/40] Add support for secret key (#405) Secret keys are JWTs prefixed with nbwt_ or edbt_ for authentication, default to secret_key in $config_dir/$cloud_profile.json for cloud instances. --- .github/workflows/tests.yml | 15 +++- docs/api/asyncio_client.rst | 9 +++ docs/api/blocking_client.rst | 9 +++ edgedb/asyncio_client.py | 2 + edgedb/base_client.py | 2 + edgedb/blocking_client.py | 2 + edgedb/con_utils.py | 146 +++++++++++++++++++++++++++++----- edgedb/protocol/protocol.pyx | 2 + tests/shared-client-testcases | 2 +- tests/test_con_utils.py | 18 ++++- 10 files changed, 183 insertions(+), 24 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5a66c55d..ece1f886 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -26,12 +26,23 @@ jobs: matrix: python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] edgedb-version: [stable , nightly] - os: [ubuntu-latest, macos-latest, windows-2019] + os: [ubuntu-20.04, ubuntu-latest, macos-latest, windows-2019] loop: [asyncio, uvloop] exclude: # uvloop does not support windows - loop: uvloop os: windows-2019 + # Python 3.7 on ubuntu-22.04 has a broken OpenSSL 3.0 + - python-version: 3.7 + os: ubuntu-latest + - python-version: 3.8 + os: ubuntu-20.04 + - python-version: 3.9 + os: ubuntu-20.04 + - python-version: 3.10 + os: ubuntu-20.04 + - python-version: 3.11 + os: ubuntu-20.04 steps: - uses: actions/checkout@v2 @@ -70,7 +81,7 @@ jobs: server-version: ${{ matrix.edgedb-version }} - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 if: steps.release.outputs.version == 0 with: python-version: ${{ matrix.python-version }} diff --git a/docs/api/asyncio_client.rst b/docs/api/asyncio_client.rst index 97bf7623..18f93b31 100644 --- a/docs/api/asyncio_client.rst +++ b/docs/api/asyncio_client.rst @@ -15,6 +15,7 @@ Client .. py:function:: create_async_client(dsn=None, *, \ host=None, port=None, \ user=None, password=None, \ + secret_key=None, \ database=None, \ timeout=60, \ concurrency=None) @@ -85,6 +86,14 @@ Client other users and applications may be able to read it without needing specific privileges. + :param secret_key: + Secret key to be used for authentication, if the server requires one. + If not specified, the value parsed from the *dsn* argument is used, + or the value of the ``EDGEDB_SECRET_KEY`` environment variable. + Note that the use of the environment variable is discouraged as + other users and applications may be able to read it without needing + specific privileges. + :param float timeout: Connection timeout in seconds. diff --git a/docs/api/blocking_client.rst b/docs/api/blocking_client.rst index 7ab26507..eca92240 100644 --- a/docs/api/blocking_client.rst +++ b/docs/api/blocking_client.rst @@ -15,6 +15,7 @@ Client .. py:function:: create_client(dsn=None, *, \ host=None, port=None, \ user=None, password=None, \ + secret_key=None, \ database=None, \ timeout=60, \ concurrency=None) @@ -85,6 +86,14 @@ Client other users and applications may be able to read it without needing specific privileges. + :param secret_key: + Secret key to be used for authentication, if the server requires one. + If not specified, the value parsed from the *dsn* argument is used, + or the value of the ``EDGEDB_SECRET_KEY`` environment variable. + Note that the use of the environment variable is discouraged as + other users and applications may be able to read it without needing + specific privileges. + :param float timeout: Connection timeout in seconds. diff --git a/edgedb/asyncio_client.py b/edgedb/asyncio_client.py index b30171e0..fd4ac438 100644 --- a/edgedb/asyncio_client.py +++ b/edgedb/asyncio_client.py @@ -378,6 +378,7 @@ def create_async_client( credentials_file: str = None, user: str = None, password: str = None, + secret_key: str = None, database: str = None, tls_ca: str = None, tls_ca_file: str = None, @@ -397,6 +398,7 @@ def create_async_client( credentials_file=credentials_file, user=user, password=password, + secret_key=secret_key, database=database, tls_ca=tls_ca, tls_ca_file=tls_ca_file, diff --git a/edgedb/base_client.py b/edgedb/base_client.py index 94d85972..c0262069 100644 --- a/edgedb/base_client.py +++ b/edgedb/base_client.py @@ -670,6 +670,7 @@ def __init__( credentials_file: str = None, user: str = None, password: str = None, + secret_key: str = None, database: str = None, tls_ca: str = None, tls_ca_file: str = None, @@ -687,6 +688,7 @@ def __init__( "credentials_file": credentials_file, "user": user, "password": password, + "secret_key": secret_key, "database": database, "timeout": timeout, "tls_ca": tls_ca, diff --git a/edgedb/blocking_client.py b/edgedb/blocking_client.py index 05e8b95c..e293a534 100644 --- a/edgedb/blocking_client.py +++ b/edgedb/blocking_client.py @@ -401,6 +401,7 @@ def create_client( credentials_file: str = None, user: str = None, password: str = None, + secret_key: str = None, database: str = None, tls_ca: str = None, tls_ca_file: str = None, @@ -420,6 +421,7 @@ def create_client( credentials_file=credentials_file, user=user, password=password, + secret_key=secret_key, database=database, tls_ca=tls_ca, tls_ca_file=tls_ca_file, diff --git a/edgedb/con_utils.py b/edgedb/con_utils.py index d17cf32a..85adec46 100644 --- a/edgedb/con_utils.py +++ b/edgedb/con_utils.py @@ -17,6 +17,8 @@ # +import base64 +import binascii import errno import json import os @@ -71,6 +73,9 @@ HUMAN_US_RE = re.compile( r'((?:(?:\s|^)-\s*)?\d*\.?\d*)\s*(?i:us(\s|\d|\.|$)|microseconds?(\s|$))', ) +INSTANCE_NAME_RE = re.compile( + r'^([A-Za-z_]\w*)(?:/([A-Za-z_]\w*))?$', +) class ClientConfiguration(typing.NamedTuple): @@ -175,6 +180,9 @@ class ResolvedConnectConfig: _password = None _password_source = None + _secret_key = None + _secret_key_source = None + _tls_ca_data = None _tls_ca_data_source = None @@ -183,6 +191,9 @@ class ResolvedConnectConfig: _wait_until_available = None + _cloud_profile = None + _cloud_profile_source = None + server_settings = {} def _set_param(self, param, value, source, validator=None): @@ -211,6 +222,9 @@ def set_user(self, user, source): def set_password(self, password, source): self._set_param('password', password, source) + def set_secret_key(self, secret_key, source): + self._set_param('secret_key', secret_key, source) + def set_tls_ca_data(self, ca_data, source): self._set_param('tls_ca_data', ca_data, source) @@ -256,6 +270,10 @@ def user(self): def password(self): return self._password + @property + def secret_key(self): + return self._secret_key + @property def tls_security(self): tls_security = self._tls_security or 'default' @@ -491,6 +509,7 @@ def _parse_connect_dsn_and_args( credentials_file, user, password, + secret_key, database, tls_ca, tls_ca_file, @@ -505,6 +524,7 @@ def _parse_connect_dsn_and_args( if dsn is not None and re.match('(?i)^[a-z]+://', dsn) else (None, dsn) ) + cloud_profile = os.getenv('EDGEDB_CLOUD_PROFILE') has_compound_options = _resolve_config_options( resolved_config, @@ -534,6 +554,10 @@ def _parse_connect_dsn_and_args( (password, '"password" option') if password is not None else None ), + secret_key=( + (secret_key, '"secret_key" option') + if secret_key is not None else None + ), tls_ca=( (tls_ca, '"tls_ca" option') if tls_ca is not None else None @@ -553,7 +577,12 @@ def _parse_connect_dsn_and_args( wait_until_available=( (wait_until_available, '"wait_until_available" option') if wait_until_available is not None else None - ) + ), + cloud_profile=( + (cloud_profile, + '"EDGEDB_CLOUD_PROFILE" environment variable') + if cloud_profile is not None else None + ), ) if has_compound_options is False: @@ -574,6 +603,7 @@ def _parse_connect_dsn_and_args( env_database = os.getenv('EDGEDB_DATABASE') env_user = os.getenv('EDGEDB_USER') env_password = os.getenv('EDGEDB_PASSWORD') + env_secret_key = os.getenv('EDGEDB_SECRET_KEY') env_tls_ca = os.getenv('EDGEDB_TLS_CA') env_tls_ca_file = os.getenv('EDGEDB_TLS_CA_FILE') env_tls_security = os.getenv('EDGEDB_CLIENT_TLS_SECURITY') @@ -617,6 +647,10 @@ def _parse_connect_dsn_and_args( (env_password, '"EDGEDB_PASSWORD" environment variable') if env_password is not None else None ), + secret_key=( + (env_secret_key, '"EDGEDB_SECRET_KEY" environment variable') + if env_secret_key is not None else None + ), tls_ca=( (env_tls_ca, '"EDGEDB_TLS_CA" environment variable') if env_tls_ca is not None else None @@ -635,7 +669,7 @@ def _parse_connect_dsn_and_args( env_wait_until_available, '"EDGEDB_WAIT_UNTIL_AVAILABLE" environment variable' ) if env_wait_until_available is not None else None - ) + ), ) if not has_compound_options: @@ -644,15 +678,25 @@ def _parse_connect_dsn_and_args( if os.path.exists(stash_dir): with open(os.path.join(stash_dir, 'instance-name'), 'rt') as f: instance_name = f.read().strip() - - _resolve_config_options( - resolved_config, - '', - instance_name=( - instance_name, - f'project linked instance ("{instance_name}")' - ) - ) + cloud_profile_file = os.path.join(stash_dir, 'cloud-profile') + if os.path.exists(cloud_profile_file): + with open(cloud_profile_file, 'rt') as f: + cloud_profile = f.read().strip() + else: + cloud_profile = None + + _resolve_config_options( + resolved_config, + '', + instance_name=( + instance_name, + f'project linked instance ("{instance_name}")' + ), + cloud_profile=( + cloud_profile, + f'project defined cloud profile ("{cloud_profile}")' + ), + ) else: raise errors.ClientConnectionError( f'Found `edgedb.toml` but the project is not initialized. ' @@ -774,6 +818,11 @@ def strip_leading_slash(str): resolved_config._password, resolved_config.set_password ) + handle_dsn_part( + 'secret_key', None, + resolved_config._secret_key, resolved_config.set_secret_key + ) + handle_dsn_part( 'tls_ca_file', None, resolved_config._tls_ca_data, resolved_config.set_tls_ca_file @@ -794,6 +843,56 @@ def strip_leading_slash(str): resolved_config.add_server_settings(query) +def _jwt_base64_decode(payload): + remainder = len(payload) % 4 + if remainder == 2: + payload += '==' + elif remainder == 3: + payload += '=' + elif remainder != 0: + raise errors.ClientConnectionError("Invalid secret key") + payload = base64.urlsafe_b64decode(payload.encode("utf-8")) + return json.loads(payload.decode("utf-8")) + + +def _parse_cloud_instance_name_into_config( + resolved_config: ResolvedConnectConfig, + source: str, + org_slug: str, + instance_name: str, +): + secret_key = resolved_config.secret_key + if secret_key is None: + try: + config_dir = platform.config_dir() + if resolved_config._cloud_profile is None: + profile = profile_src = "default" + else: + profile = resolved_config._cloud_profile + profile_src = resolved_config._cloud_profile_source + path = config_dir / "cloud-credentials" / f"{profile}.json" + with open(path, "rt") as f: + secret_key = json.load(f)["secret_key"] + except Exception: + raise errors.ClientConnectionError( + "Cannot connect to cloud instances without secret key." + ) + resolved_config.set_secret_key( + secret_key, + f"cloud-credentials/{profile}.json specified by {profile_src}", + ) + try: + dns_zone = _jwt_base64_decode(secret_key.split(".", 2)[1])["iss"] + except errors.EdgeDBError: + raise + except Exception: + raise errors.ClientConnectionError("Invalid secret key") + payload = f"{org_slug}/{instance_name}".encode("utf-8") + dns_bucket = binascii.crc_hqx(payload, 0) % 9900 + host = f"{instance_name}.{org_slug}.c-{dns_bucket:x}.i.{dns_zone}" + resolved_config.set_host(host, source) + + def _resolve_config_options( resolved_config: ResolvedConnectConfig, compound_error: str, @@ -807,11 +906,13 @@ def _resolve_config_options( database=None, user=None, password=None, + secret_key=None, tls_ca=None, tls_ca_file=None, tls_security=None, server_settings=None, wait_until_available=None, + cloud_profile=None, ): if database is not None: resolved_config.set_database(*database) @@ -819,6 +920,8 @@ def _resolve_config_options( resolved_config.set_user(*user) if password is not None: resolved_config.set_password(*password) + if secret_key is not None: + resolved_config.set_secret_key(*secret_key) if tls_ca_file is not None: if tls_ca is not None: raise errors.ClientConnectionError( @@ -832,6 +935,8 @@ def _resolve_config_options( resolved_config.add_server_settings(server_settings[0]) if wait_until_available is not None: resolved_config.set_wait_until_available(*wait_until_available) + if cloud_profile is not None: + resolved_config._set_param('cloud_profile', *cloud_profile) compound_params = [ dsn, @@ -869,22 +974,23 @@ def _resolve_config_options( creds = cred_utils.validate_credentials(cred_data) source = "credentials" else: - if ( - re.match( - '^[A-Za-z_][A-Za-z_0-9]*$', - instance_name[0] - ) is None - ): + name_match = INSTANCE_NAME_RE.match(instance_name[0]) + if name_match is None: raise ValueError( f'invalid DSN or instance name: "{instance_name[0]}"' ) + source = instance_name[1] + org, inst = name_match.groups() + if inst is not None: + _parse_cloud_instance_name_into_config( + resolved_config, source, org, inst + ) + return True creds = cred_utils.read_credentials( cred_utils.get_credentials_path(instance_name[0]), ) - source = instance_name[1] - resolved_config.set_host(creds.get('host'), source) resolved_config.set_port(creds.get('port'), source) resolved_config.set_database(creds.get('database'), source) @@ -939,6 +1045,7 @@ def parse_connect_arguments( database, user, password, + secret_key, tls_ca, tls_ca_file, tls_security, @@ -970,6 +1077,7 @@ def parse_connect_arguments( database=database, user=user, password=password, + secret_key=secret_key, tls_ca=tls_ca, tls_ca_file=tls_ca_file, tls_security=tls_security, diff --git a/edgedb/protocol/protocol.pyx b/edgedb/protocol/protocol.pyx index b6b86295..a33f815d 100644 --- a/edgedb/protocol/protocol.pyx +++ b/edgedb/protocol/protocol.pyx @@ -849,6 +849,8 @@ cdef class SansIOProtocol: 'user': self.con_params.user, 'database': self.con_params.database, } + if self.con_params.secret_key: + params['token'] = self.con_params.secret_key handshake_buf.write_int16(len(params)) for k, v in params.items(): handshake_buf.write_len_prefixed_utf8(k) diff --git a/tests/shared-client-testcases b/tests/shared-client-testcases index 70433a6d..ca5a082e 160000 --- a/tests/shared-client-testcases +++ b/tests/shared-client-testcases @@ -1 +1 @@ -Subproject commit 70433a6da0f3f1c9e991fdac7bb7f7ccab5ad878 +Subproject commit ca5a082e823befce73e80ddb37618ccf071cf231 diff --git a/tests/test_con_utils.py b/tests/test_con_utils.py index 0582541a..5854a76b 100644 --- a/tests/test_con_utils.py +++ b/tests/test_con_utils.py @@ -68,7 +68,12 @@ class TestConUtils(unittest.TestCase): 'file_not_found': (FileNotFoundError, 'No such file or directory'), 'invalid_tls_security': ( ValueError, 'tls_security can only be one of `insecure`, ' - '|tls_security must be set to strict') + '|tls_security must be set to strict'), + 'invalid_secret_key': ( + errors.ClientConnectionError, "Invalid secret key"), + 'secret_key_not_found': ( + errors.ClientConnectionError, + "Cannot connect to cloud instances without secret key"), } @contextlib.contextmanager @@ -98,6 +103,7 @@ def run_testcase(self, testcase): env = testcase.get('env', {}) test_env = {'EDGEDB_HOST': None, 'EDGEDB_PORT': None, 'EDGEDB_USER': None, 'EDGEDB_PASSWORD': None, + 'EDGEDB_SECRET_KEY': None, 'EDGEDB_DATABASE': None, 'PGSSLMODE': None, 'XDG_CONFIG_HOME': None} test_env.update(env) @@ -105,7 +111,7 @@ def run_testcase(self, testcase): fs = testcase.get('fs') opts = testcase.get('opts', {}) - dsn = opts.get('dsn') + dsn = opts['instance'] if 'instance' in opts else opts.get('dsn') credentials = opts.get('credentials') credentials_file = opts.get('credentialsFile') host = opts.get('host') @@ -113,6 +119,7 @@ def run_testcase(self, testcase): database = opts.get('database') user = opts.get('user') password = opts.get('password') + secret_key = opts.get('secretKey') tls_ca = opts.get('tlsCA') tls_ca_file = opts.get('tlsCAFile') tls_security = opts.get('tlsSecurity') @@ -172,6 +179,9 @@ def run_testcase(self, testcase): files[instance] = v['instance-name'] project = os.path.join(dir, 'project-path') files[project] = v['project-path'] + if 'cloud-profile' in v: + profile = os.path.join(dir, 'cloud-profile') + files[profile] = v['cloud-profile'] del files[f] es.enter_context( @@ -219,6 +229,7 @@ def mocked_open(filepath, *args, **kwargs): database=database, user=user, password=password, + secret_key=secret_key, tls_ca=tls_ca, tls_ca_file=tls_ca_file, tls_security=tls_security, @@ -235,6 +246,7 @@ def mocked_open(filepath, *args, **kwargs): 'database': connect_config.database, 'user': connect_config.user, 'password': connect_config.password, + 'secretKey': connect_config.secret_key, 'tlsCAData': connect_config._tls_ca_data, 'tlsSecurity': connect_config.tls_security, 'serverSettings': connect_config.server_settings, @@ -285,6 +297,7 @@ def test_test_connect_params_run_testcase(self): 'database': 'edgedb', 'user': '__test__', 'password': None, + 'secretKey': None, 'tlsCAData': None, 'tlsSecurity': 'strict', 'serverSettings': {}, @@ -378,6 +391,7 @@ def test_project_config(self): credentials_file=None, user=None, password=None, + secret_key=None, database=None, tls_ca=None, tls_ca_file=None, From 076e8c86059f2fe77d9e657e6fbd340cb920ae35 Mon Sep 17 00:00:00 2001 From: Fantix King Date: Tue, 10 Jan 2023 15:00:13 -0500 Subject: [PATCH 10/40] Fix EDGEDB_CLOUD_PROFILE options layer Refs edgedb/shared-client-testcases#22 --- edgedb/con_utils.py | 12 ++++++------ tests/shared-client-testcases | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/edgedb/con_utils.py b/edgedb/con_utils.py index 85adec46..45b68b1d 100644 --- a/edgedb/con_utils.py +++ b/edgedb/con_utils.py @@ -524,7 +524,6 @@ def _parse_connect_dsn_and_args( if dsn is not None and re.match('(?i)^[a-z]+://', dsn) else (None, dsn) ) - cloud_profile = os.getenv('EDGEDB_CLOUD_PROFILE') has_compound_options = _resolve_config_options( resolved_config, @@ -578,11 +577,6 @@ def _parse_connect_dsn_and_args( (wait_until_available, '"wait_until_available" option') if wait_until_available is not None else None ), - cloud_profile=( - (cloud_profile, - '"EDGEDB_CLOUD_PROFILE" environment variable') - if cloud_profile is not None else None - ), ) if has_compound_options is False: @@ -608,6 +602,7 @@ def _parse_connect_dsn_and_args( env_tls_ca_file = os.getenv('EDGEDB_TLS_CA_FILE') env_tls_security = os.getenv('EDGEDB_CLIENT_TLS_SECURITY') env_wait_until_available = os.getenv('EDGEDB_WAIT_UNTIL_AVAILABLE') + cloud_profile = os.getenv('EDGEDB_CLOUD_PROFILE') has_compound_options = _resolve_config_options( resolved_config, @@ -670,6 +665,11 @@ def _parse_connect_dsn_and_args( '"EDGEDB_WAIT_UNTIL_AVAILABLE" environment variable' ) if env_wait_until_available is not None else None ), + cloud_profile=( + (cloud_profile, + '"EDGEDB_CLOUD_PROFILE" environment variable') + if cloud_profile is not None else None + ), ) if not has_compound_options: diff --git a/tests/shared-client-testcases b/tests/shared-client-testcases index ca5a082e..5f2453c3 160000 --- a/tests/shared-client-testcases +++ b/tests/shared-client-testcases @@ -1 +1 @@ -Subproject commit ca5a082e823befce73e80ddb37618ccf071cf231 +Subproject commit 5f2453c30521cfb027ac56e517289da359e90097 From 65c9c3791da39aade7017c947b693b97462776f0 Mon Sep 17 00:00:00 2001 From: Elvis Pranskevichus Date: Wed, 25 Jan 2023 20:40:45 -0800 Subject: [PATCH 11/40] Adjust cloud instance hostname derivation (#412) --- edgedb/con_utils.py | 4 ++-- tests/shared-client-testcases | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/edgedb/con_utils.py b/edgedb/con_utils.py index 45b68b1d..eb79f6e0 100644 --- a/edgedb/con_utils.py +++ b/edgedb/con_utils.py @@ -888,8 +888,8 @@ def _parse_cloud_instance_name_into_config( except Exception: raise errors.ClientConnectionError("Invalid secret key") payload = f"{org_slug}/{instance_name}".encode("utf-8") - dns_bucket = binascii.crc_hqx(payload, 0) % 9900 - host = f"{instance_name}.{org_slug}.c-{dns_bucket:x}.i.{dns_zone}" + dns_bucket = binascii.crc_hqx(payload, 0) % 100 + host = f"{instance_name}--{org_slug}.c-{dns_bucket:02d}.i.{dns_zone}" resolved_config.set_host(host, source) diff --git a/tests/shared-client-testcases b/tests/shared-client-testcases index 5f2453c3..72675edf 160000 --- a/tests/shared-client-testcases +++ b/tests/shared-client-testcases @@ -1 +1 @@ -Subproject commit 5f2453c30521cfb027ac56e517289da359e90097 +Subproject commit 72675edfd43cd39bbe39b706e0847453676aac35 From 14363fba351cb277c572986ad353866b521b4fc3 Mon Sep 17 00:00:00 2001 From: 0xsirsaif Date: Tue, 31 Jan 2023 20:41:06 +0200 Subject: [PATCH 12/40] Minor changes (#413) * usage.rst & asyncio_client.rst * Update docs/api/asyncio_client.rst Co-authored-by: Devon Campbell --------- Co-authored-by: Devon Campbell --- docs/api/asyncio_client.rst | 4 ++-- docs/usage.rst | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/api/asyncio_client.rst b/docs/api/asyncio_client.rst index 18f93b31..7435dafa 100644 --- a/docs/api/asyncio_client.rst +++ b/docs/api/asyncio_client.rst @@ -557,8 +557,8 @@ transaction), so we have to redo all the work done. Generally it's recommended to not execute any long running code within the transaction unless absolutely necessary. -Transactions allocate expensive server resources and having -too many concurrently running long-running transactions will +Transactions allocate expensive server resources, and having +too many concurrent long-running transactions will negatively impact the performance of the DB server. To rollback a transaction that is in progress raise an exception. diff --git a/docs/usage.rst b/docs/usage.rst index a531a61b..575fdfe3 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -78,8 +78,8 @@ types and vice versa. See :ref:`edgedb-python-datatypes` for details. Client connection pools ----------------------- -For server-type type applications that handle frequent requests and need -the database connection for a short period time while handling a request, +For server-type applications that handle frequent requests and need +the database connection for a short period of time while handling a request, the use of a connection pool is recommended. Both :py:class:`edgedb.Client` and :py:class:`edgedb.AsyncIOClient` come with such a pool. From 27e7a4facf2b1280d999a267e07f7d91af977beb Mon Sep 17 00:00:00 2001 From: Fantix King Date: Thu, 9 Feb 2023 15:18:41 -0500 Subject: [PATCH 13/40] Fix for cloud: auth key rename, and send SNI even when check_hostname=False --- edgedb/blocking_client.py | 6 +----- edgedb/protocol/protocol.pyx | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/edgedb/blocking_client.py b/edgedb/blocking_client.py index e293a534..e185b501 100644 --- a/edgedb/blocking_client.py +++ b/edgedb/blocking_client.py @@ -61,14 +61,10 @@ async def connect_addr(self, addr, timeout): raise TimeoutError # Upgrade to TLS - if self._params.ssl_ctx.check_hostname: - server_hostname = addr[0] - else: - server_hostname = None sock.settimeout(time_left) try: sock = self._params.ssl_ctx.wrap_socket( - sock, server_hostname=server_hostname + sock, server_hostname=addr[0] ) except ssl.CertificateError as e: raise con_utils.wrap_error(e) from e diff --git a/edgedb/protocol/protocol.pyx b/edgedb/protocol/protocol.pyx index a33f815d..b973b885 100644 --- a/edgedb/protocol/protocol.pyx +++ b/edgedb/protocol/protocol.pyx @@ -850,7 +850,7 @@ cdef class SansIOProtocol: 'database': self.con_params.database, } if self.con_params.secret_key: - params['token'] = self.con_params.secret_key + params['secret_key'] = self.con_params.secret_key handshake_buf.write_int16(len(params)) for k, v in params.items(): handshake_buf.write_len_prefixed_utf8(k) From 1b98324ba1a65676f49789ff2da376e78f2cd3b8 Mon Sep 17 00:00:00 2001 From: Fantix King Date: Thu, 9 Feb 2023 17:03:15 -0500 Subject: [PATCH 14/40] edgedb-python 1.3.0 Changes ======= * Add support for secret key authentication, and Cloud instance (#405) (by @fantix in df1f6fb3, 076e8c86, 27e7a4fa for #405, @elprans in 65c9c379 for #412) Docs ==== * Minor changes (#413) (by @0xsirsaif in 14363fba for #413) --- edgedb/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edgedb/_version.py b/edgedb/_version.py index 4da184ee..27b4c5b7 100644 --- a/edgedb/_version.py +++ b/edgedb/_version.py @@ -28,4 +28,4 @@ # supported platforms, publish the packages on PyPI, merge the PR # to the target branch, create a Git tag pointing to the commit. -__version__ = '1.2.0' +__version__ = '1.3.0' From ffe74a174951ab9c69c0b36a97ea603b82da7ec5 Mon Sep 17 00:00:00 2001 From: Andreas Brodersen Date: Tue, 14 Mar 2023 12:20:06 +0100 Subject: [PATCH 15/40] docs: add Code Generation to table of contents (#421) --- docs/index.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index 1b04d26e..1a647fb6 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -32,6 +32,10 @@ and :ref:`asyncio ` implementations. EdgeDB Python types documentation. +* :ref:`edgedb-python-codegen` + + Python code generation command-line tool documentation. + * :ref:`edgedb-python-advanced` Advanced usages of the state and optional customization. From 5bc56992a3804fae821a7b43b070a09a6cd19cf3 Mon Sep 17 00:00:00 2001 From: Fantix King Date: Wed, 22 Mar 2023 14:09:49 -0400 Subject: [PATCH 16/40] Update for rules of instance names (#423) * Instance names allow leading digits * Cloud instance name max length: 62 * Dashes are allowed, except for consecutive ones like -- * Ban underscores for the cloud --- edgedb/con_utils.py | 48 +++++++++++++++++++++++------------ tests/shared-client-testcases | 2 +- tests/test_con_utils.py | 2 ++ 3 files changed, 35 insertions(+), 17 deletions(-) diff --git a/edgedb/con_utils.py b/edgedb/con_utils.py index eb79f6e0..d8e29881 100644 --- a/edgedb/con_utils.py +++ b/edgedb/con_utils.py @@ -74,8 +74,18 @@ r'((?:(?:\s|^)-\s*)?\d*\.?\d*)\s*(?i:us(\s|\d|\.|$)|microseconds?(\s|$))', ) INSTANCE_NAME_RE = re.compile( - r'^([A-Za-z_]\w*)(?:/([A-Za-z_]\w*))?$', + r'^(\w(?:-?\w)*)$', + re.ASCII, ) +CLOUD_INSTANCE_NAME_RE = re.compile( + r'^([A-Za-z0-9](?:-?[A-Za-z0-9])*)/([A-Za-z0-9](?:-?[A-Za-z0-9])*)$', + re.ASCII, +) +DSN_RE = re.compile( + r'^[a-z]+://', + re.IGNORECASE, +) +DOMAIN_LABEL_MAX_LENGTH = 63 class ClientConfiguration(typing.NamedTuple): @@ -519,11 +529,10 @@ def _parse_connect_dsn_and_args( ): resolved_config = ResolvedConnectConfig() - dsn, instance_name = ( - (dsn, None) - if dsn is not None and re.match('(?i)^[a-z]+://', dsn) - else (None, dsn) - ) + if dsn and DSN_RE.match(dsn): + instance_name = None + else: + instance_name, dsn = dsn, None has_compound_options = _resolve_config_options( resolved_config, @@ -861,6 +870,13 @@ def _parse_cloud_instance_name_into_config( org_slug: str, instance_name: str, ): + label = f"{instance_name}--{org_slug}" + if len(label) > DOMAIN_LABEL_MAX_LENGTH: + raise ValueError( + f"invalid instance name: cloud instance name length cannot exceed " + f"{DOMAIN_LABEL_MAX_LENGTH - 1} characters: " + f"{org_slug}/{instance_name}" + ) secret_key = resolved_config.secret_key if secret_key is None: try: @@ -889,7 +905,7 @@ def _parse_cloud_instance_name_into_config( raise errors.ClientConnectionError("Invalid secret key") payload = f"{org_slug}/{instance_name}".encode("utf-8") dns_bucket = binascii.crc_hqx(payload, 0) % 100 - host = f"{instance_name}--{org_slug}.c-{dns_bucket:02d}.i.{dns_zone}" + host = f"{label}.c-{dns_bucket:02d}.i.{dns_zone}" resolved_config.set_host(host, source) @@ -973,23 +989,23 @@ def _resolve_config_options( else: creds = cred_utils.validate_credentials(cred_data) source = "credentials" + elif INSTANCE_NAME_RE.match(instance_name[0]): + source = instance_name[1] + creds = cred_utils.read_credentials( + cred_utils.get_credentials_path(instance_name[0]), + ) else: - name_match = INSTANCE_NAME_RE.match(instance_name[0]) + name_match = CLOUD_INSTANCE_NAME_RE.match(instance_name[0]) if name_match is None: raise ValueError( f'invalid DSN or instance name: "{instance_name[0]}"' ) source = instance_name[1] org, inst = name_match.groups() - if inst is not None: - _parse_cloud_instance_name_into_config( - resolved_config, source, org, inst - ) - return True - - creds = cred_utils.read_credentials( - cred_utils.get_credentials_path(instance_name[0]), + _parse_cloud_instance_name_into_config( + resolved_config, source, org, inst ) + return True resolved_config.set_host(creds.get('host'), source) resolved_config.set_port(creds.get('port'), source) diff --git a/tests/shared-client-testcases b/tests/shared-client-testcases index 72675edf..32cb54ca 160000 --- a/tests/shared-client-testcases +++ b/tests/shared-client-testcases @@ -1 +1 @@ -Subproject commit 72675edfd43cd39bbe39b706e0847453676aac35 +Subproject commit 32cb54cad0d414874962c0d31f2380b3198c07ae diff --git a/tests/test_con_utils.py b/tests/test_con_utils.py index 5854a76b..742391ba 100644 --- a/tests/test_con_utils.py +++ b/tests/test_con_utils.py @@ -46,6 +46,8 @@ class TestConUtils(unittest.TestCase): RuntimeError, 'cannot read credentials'), 'invalid_dsn_or_instance_name': ( ValueError, 'invalid DSN or instance name'), + 'invalid_instance_name': ( + ValueError, 'invalid instance name'), 'invalid_dsn': (ValueError, 'invalid DSN'), 'unix_socket_unsupported': ( ValueError, 'unix socket paths not supported'), From 03e40121d6fcd63d7782ff8a5e5a8a6849f84dc0 Mon Sep 17 00:00:00 2001 From: Fantix King Date: Thu, 25 May 2023 11:26:35 -0400 Subject: [PATCH 17/40] Sync errors --- edgedb/errors/__init__.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/edgedb/errors/__init__.py b/edgedb/errors/__init__.py index c5ea6daa..cfe55e1d 100644 --- a/edgedb/errors/__init__.py +++ b/edgedb/errors/__init__.py @@ -68,6 +68,7 @@ 'DuplicateFunctionDefinitionError', 'DuplicateConstraintDefinitionError', 'DuplicateCastDefinitionError', + 'DuplicateMigrationError', 'SessionTimeoutError', 'IdleSessionTimeoutError', 'QueryTimeoutError', @@ -78,6 +79,7 @@ 'DivisionByZeroError', 'NumericOutOfRangeError', 'AccessPolicyError', + 'QueryAssertionError', 'IntegrityError', 'ConstraintViolationError', 'CardinalityViolationError', @@ -86,6 +88,7 @@ 'TransactionConflictError', 'TransactionSerializationError', 'TransactionDeadlockError', + 'WatchError', 'ConfigurationError', 'AccessError', 'AuthenticationError', @@ -328,12 +331,17 @@ class DuplicateCastDefinitionError(DuplicateDefinitionError): _code = 0x_04_05_02_0A +class DuplicateMigrationError(DuplicateDefinitionError): + _code = 0x_04_05_02_0B + + class SessionTimeoutError(QueryError): _code = 0x_04_06_00_00 class IdleSessionTimeoutError(SessionTimeoutError): _code = 0x_04_06_01_00 + tags = frozenset({SHOULD_RETRY}) class QueryTimeoutError(SessionTimeoutError): @@ -368,6 +376,10 @@ class AccessPolicyError(InvalidValueError): _code = 0x_05_01_00_03 +class QueryAssertionError(InvalidValueError): + _code = 0x_05_01_00_04 + + class IntegrityError(ExecutionError): _code = 0x_05_02_00_00 @@ -403,6 +415,10 @@ class TransactionDeadlockError(TransactionConflictError): tags = frozenset({SHOULD_RETRY}) +class WatchError(ExecutionError): + _code = 0x_05_04_00_00 + + class ConfigurationError(EdgeDBError): _code = 0x_06_00_00_00 From 2de7e3fb2e8e1d55bf0e2d8b772465c147a1710a Mon Sep 17 00:00:00 2001 From: Fantix King Date: Thu, 25 May 2023 15:54:26 -0400 Subject: [PATCH 18/40] Allow enums in array codec --- edgedb/protocol/codecs/array.pyx | 2 +- tests/test_enum.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/edgedb/protocol/codecs/array.pyx b/edgedb/protocol/codecs/array.pyx index 34cc567b..fb0f872b 100644 --- a/edgedb/protocol/codecs/array.pyx +++ b/edgedb/protocol/codecs/array.pyx @@ -39,7 +39,7 @@ cdef class BaseArrayCodec(BaseCodec): if not isinstance( self.sub_codec, - (ScalarCodec, TupleCodec, NamedTupleCodec, RangeCodec) + (ScalarCodec, TupleCodec, NamedTupleCodec, RangeCodec, EnumCodec) ): raise TypeError( 'only arrays of scalars are supported (got type {!r})'.format( diff --git a/tests/test_enum.py b/tests/test_enum.py index a5e99b15..86bf40b6 100644 --- a/tests/test_enum.py +++ b/tests/test_enum.py @@ -99,3 +99,12 @@ async def test_enum_03(self): c_red = await self.client.query_single('SELECT "red"') c_red2 = await self.client.query_single('SELECT $0', c_red) self.assertIs(c_red, c_red2) + + async def test_enum_04(self): + enums = await self.client.query_single( + 'SELECT >$0', ['red', 'white'] + ) + enums2 = await self.client.query_single( + 'SELECT >$0', enums + ) + self.assertEqual(enums, enums2) From f1fa612b9d7eab978e469b203ea22f6dd9bfdf28 Mon Sep 17 00:00:00 2001 From: Fantix King Date: Thu, 25 May 2023 17:15:59 -0400 Subject: [PATCH 19/40] Prohibit concurrent operations on the same transaction object (#430) Co-authored-by: Elvis Pranskevichus --- edgedb/asyncio_client.py | 30 +++++++++++++++++++++++++++--- edgedb/blocking_client.py | 38 ++++++++++++++++++++++++++++---------- tests/test_async_tx.py | 15 +++++++++++++++ tests/test_sync_tx.py | 16 ++++++++++++++++ 4 files changed, 86 insertions(+), 13 deletions(-) diff --git a/edgedb/asyncio_client.py b/edgedb/asyncio_client.py index fd4ac438..c03c6423 100644 --- a/edgedb/asyncio_client.py +++ b/edgedb/asyncio_client.py @@ -18,6 +18,7 @@ import asyncio +import contextlib import logging import socket import ssl @@ -273,11 +274,12 @@ def _warn_on_long_close(self): class AsyncIOIteration(transaction.BaseTransaction, abstract.AsyncIOExecutor): - __slots__ = ("_managed",) + __slots__ = ("_managed", "_locked") def __init__(self, retry, client, iteration): super().__init__(retry, client, iteration) self._managed = False + self._locked = False async def __aenter__(self): if self._managed: @@ -287,8 +289,9 @@ async def __aenter__(self): return self async def __aexit__(self, extype, ex, tb): - self._managed = False - return await self._exit(extype, ex) + with self._exclusive(): + self._managed = False + return await self._exit(extype, ex) async def _ensure_transaction(self): if not self._managed: @@ -298,6 +301,27 @@ async def _ensure_transaction(self): ) await super()._ensure_transaction() + async def _query(self, query_context: abstract.QueryContext): + with self._exclusive(): + return await super()._query(query_context) + + async def _execute(self, execute_context: abstract.ExecuteContext) -> None: + with self._exclusive(): + await super()._execute(execute_context) + + @contextlib.contextmanager + def _exclusive(self): + if self._locked: + raise errors.InterfaceError( + "concurrent queries within the same transaction " + "are not allowed" + ) + self._locked = True + try: + yield + finally: + self._locked = False + class AsyncIORetry(transaction.BaseRetry): diff --git a/edgedb/blocking_client.py b/edgedb/blocking_client.py index e185b501..7eb761b9 100644 --- a/edgedb/blocking_client.py +++ b/edgedb/blocking_client.py @@ -17,6 +17,7 @@ # +import contextlib import datetime import queue import socket @@ -271,22 +272,25 @@ async def close(self, timeout=None): class Iteration(transaction.BaseTransaction, abstract.Executor): - __slots__ = ("_managed",) + __slots__ = ("_managed", "_lock") def __init__(self, retry, client, iteration): super().__init__(retry, client, iteration) self._managed = False + self._lock = threading.Lock() def __enter__(self): - if self._managed: - raise errors.InterfaceError( - 'cannot enter context: already in a `with` block') - self._managed = True - return self + with self._exclusive(): + if self._managed: + raise errors.InterfaceError( + 'cannot enter context: already in a `with` block') + self._managed = True + return self def __exit__(self, extype, ex, tb): - self._managed = False - return self._client._iter_coroutine(self._exit(extype, ex)) + with self._exclusive(): + self._managed = False + return self._client._iter_coroutine(self._exit(extype, ex)) async def _ensure_transaction(self): if not self._managed: @@ -297,10 +301,24 @@ async def _ensure_transaction(self): await super()._ensure_transaction() def _query(self, query_context: abstract.QueryContext): - return self._client._iter_coroutine(super()._query(query_context)) + with self._exclusive(): + return self._client._iter_coroutine(super()._query(query_context)) def _execute(self, execute_context: abstract.ExecuteContext) -> None: - self._client._iter_coroutine(super()._execute(execute_context)) + with self._exclusive(): + self._client._iter_coroutine(super()._execute(execute_context)) + + @contextlib.contextmanager + def _exclusive(self): + if not self._lock.acquire(blocking=False): + raise errors.InterfaceError( + "concurrent queries within the same transaction " + "are not allowed" + ) + try: + yield + finally: + self._lock.release() class Retry(transaction.BaseRetry): diff --git a/tests/test_async_tx.py b/tests/test_async_tx.py index 71a10287..8ceeb239 100644 --- a/tests/test_async_tx.py +++ b/tests/test_async_tx.py @@ -16,6 +16,7 @@ # limitations under the License. # +import asyncio import itertools import edgedb @@ -89,3 +90,17 @@ async def test_async_transaction_commit_failure(self): async with tx: await tx.execute("start migration to {};") self.assertEqual(await self.client.query_single("select 42"), 42) + + async def test_async_transaction_exclusive(self): + async for tx in self.client.transaction(): + async with tx: + query = "select sys::_sleep(0.01)" + f1 = self.loop.create_task(tx.execute(query)) + f2 = self.loop.create_task(tx.execute(query)) + with self.assertRaisesRegex( + edgedb.InterfaceError, + "concurrent queries within the same transaction " + "are not allowed" + ): + await asyncio.wait_for(f1, timeout=5) + await asyncio.wait_for(f2, timeout=5) diff --git a/tests/test_sync_tx.py b/tests/test_sync_tx.py index eb1abc0e..3ed2fc55 100644 --- a/tests/test_sync_tx.py +++ b/tests/test_sync_tx.py @@ -17,6 +17,7 @@ # import itertools +from concurrent.futures import ThreadPoolExecutor import edgedb @@ -97,3 +98,18 @@ def test_sync_transaction_commit_failure(self): with tx: tx.execute("start migration to {};") self.assertEqual(self.client.query_single("select 42"), 42) + + def test_sync_transaction_exclusive(self): + for tx in self.client.transaction(): + with tx: + query = "select sys::_sleep(0.01)" + with ThreadPoolExecutor(max_workers=2) as executor: + f1 = executor.submit(tx.execute, query) + f2 = executor.submit(tx.execute, query) + with self.assertRaisesRegex( + edgedb.InterfaceError, + "concurrent queries within the same transaction " + "are not allowed" + ): + f1.result(timeout=5) + f2.result(timeout=5) From 297de7228b3a7cc0518011874394ebc530fd76c9 Mon Sep 17 00:00:00 2001 From: Fantix King Date: Thu, 20 Apr 2023 19:20:07 -0400 Subject: [PATCH 20/40] Fix state of transaction start This affects the case when compilation config is set on the client, while a new command is compiled within a transaction. In this case, the compiler will only use the state issued in the transaction start. Before this fix, we didn't send state on transaction commands, so those config was not in effect within transactions, even though each query did carry the right config. --- edgedb/base_client.py | 4 ++++ tests/test_async_query.py | 15 +++++++++++++++ tests/test_sync_query.py | 15 +++++++++++++++ 3 files changed, 34 insertions(+) diff --git a/edgedb/base_client.py b/edgedb/base_client.py index c0262069..331562e6 100644 --- a/edgedb/base_client.py +++ b/edgedb/base_client.py @@ -186,6 +186,10 @@ async def privileged_execute( qc=execute_context.cache.query_cache, output_format=protocol.OutputFormat.NONE, allow_capabilities=enums.Capability.ALL, + state=( + execute_context.state.as_dict() + if execute_context.state else None + ), ) def is_in_transaction(self) -> bool: diff --git a/tests/test_async_query.py b/tests/test_async_query.py index e51ac197..bd9fd3a9 100644 --- a/tests/test_async_query.py +++ b/tests/test_async_query.py @@ -1030,3 +1030,18 @@ async def test_dup_link_prop_name(self): DROP TYPE test::dup_link_prop_name_p; DROP TYPE test::dup_link_prop_name; ''') + + async def test_transaction_state(self): + with self.assertRaisesRegex(edgedb.QueryError, "cannot assign to id"): + async for tx in self.client.transaction(): + async with tx: + await tx.execute(''' + INSERT test::Tmp { id := $0, tmp := '' } + ''', uuid.uuid4()) + + client = self.client.with_config(allow_user_specified_id=True) + async for tx in client.transaction(): + async with tx: + await tx.execute(''' + INSERT test::Tmp { id := $0, tmp := '' } + ''', uuid.uuid4()) diff --git a/tests/test_sync_query.py b/tests/test_sync_query.py index 0d793616..8dd35c3c 100644 --- a/tests/test_sync_query.py +++ b/tests/test_sync_query.py @@ -868,3 +868,18 @@ def test_sync_banned_transaction(self): r'cannot execute transaction control commands', ): self.client.execute('start transaction') + + def test_transaction_state(self): + with self.assertRaisesRegex(edgedb.QueryError, "cannot assign to id"): + for tx in self.client.transaction(): + with tx: + tx.execute(''' + INSERT test::Tmp { id := $0, tmp := '' } + ''', uuid.uuid4()) + + client = self.client.with_config(allow_user_specified_id=True) + for tx in client.transaction(): + with tx: + tx.execute(''' + INSERT test::Tmp { id := $0, tmp := '' } + ''', uuid.uuid4()) From e1ec16dee318449cb4d30bc6ffb02f3370d259c3 Mon Sep 17 00:00:00 2001 From: Fantix King Date: Thu, 25 May 2023 17:23:24 -0400 Subject: [PATCH 21/40] codegen: Handle non-identifier characters in enum values --- edgedb/codegen/generator.py | 38 ++++++++++++++++++- .../generated_async_edgeql.py.assert | 5 ++- .../parpkg/subpkg/my_query.edgeql | 2 +- .../subpkg/my_query_async_edgeql.py.assert | 5 ++- .../parpkg/subpkg/my_query_edgeql.py.assert | 5 ++- 5 files changed, 49 insertions(+), 6 deletions(-) diff --git a/edgedb/codegen/generator.py b/edgedb/codegen/generator.py index ff49b82e..529a3fe4 100644 --- a/edgedb/codegen/generator.py +++ b/edgedb/codegen/generator.py @@ -483,8 +483,8 @@ def _generate_code( buf = io.StringIO() self._imports.add("enum") print(f"class {rv}(enum.Enum):", file=buf) - for member in type_.members: - print(f'{INDENT}{member.upper()} = "{member}"', file=buf) + for member, member_id in self._to_unique_idents(type_.members): + print(f'{INDENT}{member_id.upper()} = "{member}"', file=buf) self._defs[rv] = buf.getvalue().strip() elif isinstance(type_, describe.RangeType): @@ -537,3 +537,37 @@ def _snake_to_camel(self, name: str) -> str: return "".join(map(str.title, parts)) else: return name + + def _to_unique_idents( + self, names: typing.Iterable[typing.Tuple[str, str]] + ) -> typing.Iterator[str]: + dedup = set() + for name in names: + if name.isidentifier(): + name_id = name + sep = name.endswith("_") + else: + sep = True + result = [] + for i, c in enumerate(name): + if c.isdigit(): + if i == 0: + result.append("e_") + result.append(c) + sep = False + elif c.isidentifier(): + result.append(c) + sep = c == "_" + elif not sep: + result.append("_") + sep = True + name_id = "".join(result) + rv = name_id + if not sep: + name_id = name_id + "_" + i = 1 + while rv in dedup: + rv = f"{name_id}{i}" + i += 1 + dedup.add(rv) + yield name, rv diff --git a/tests/codegen/test-project2/generated_async_edgeql.py.assert b/tests/codegen/test-project2/generated_async_edgeql.py.assert index 40e256ba..75f91b5b 100644 --- a/tests/codegen/test-project2/generated_async_edgeql.py.assert +++ b/tests/codegen/test-project2/generated_async_edgeql.py.assert @@ -51,6 +51,9 @@ class LinkPropResultFriendsItem: class MyEnum(enum.Enum): THIS = "This" THAT = "That" + E_1 = "1" + F_B = "f. b" + F_B_1 = "f-b" @dataclasses.dataclass @@ -218,7 +221,7 @@ async def my_query( return await executor.query_single( """\ create scalar type MyScalar extending int64; - create scalar type MyEnum extending enum; + create scalar type MyEnum extending enum<'This', 'That', '1', 'f. b', 'f-b'>; select { a := $a, diff --git a/tests/codegen/test-project2/parpkg/subpkg/my_query.edgeql b/tests/codegen/test-project2/parpkg/subpkg/my_query.edgeql index 2a9b2e49..a00f8964 100644 --- a/tests/codegen/test-project2/parpkg/subpkg/my_query.edgeql +++ b/tests/codegen/test-project2/parpkg/subpkg/my_query.edgeql @@ -1,5 +1,5 @@ create scalar type MyScalar extending int64; -create scalar type MyEnum extending enum; +create scalar type MyEnum extending enum<'This', 'That', '1', 'f. b', 'f-b'>; select { a := $a, diff --git a/tests/codegen/test-project2/parpkg/subpkg/my_query_async_edgeql.py.assert b/tests/codegen/test-project2/parpkg/subpkg/my_query_async_edgeql.py.assert index ba841cc8..6225eb88 100644 --- a/tests/codegen/test-project2/parpkg/subpkg/my_query_async_edgeql.py.assert +++ b/tests/codegen/test-project2/parpkg/subpkg/my_query_async_edgeql.py.assert @@ -26,6 +26,9 @@ class NoPydanticValidation: class MyEnum(enum.Enum): THIS = "This" THAT = "That" + E_1 = "1" + F_B = "f. b" + F_B_1 = "f-b" @dataclasses.dataclass @@ -144,7 +147,7 @@ async def my_query( return await executor.query_single( """\ create scalar type MyScalar extending int64; - create scalar type MyEnum extending enum; + create scalar type MyEnum extending enum<'This', 'That', '1', 'f. b', 'f-b'>; select { a := $a, diff --git a/tests/codegen/test-project2/parpkg/subpkg/my_query_edgeql.py.assert b/tests/codegen/test-project2/parpkg/subpkg/my_query_edgeql.py.assert index 00fd747a..35f89338 100644 --- a/tests/codegen/test-project2/parpkg/subpkg/my_query_edgeql.py.assert +++ b/tests/codegen/test-project2/parpkg/subpkg/my_query_edgeql.py.assert @@ -17,6 +17,9 @@ MyScalar = int class MyEnum(enum.Enum): THIS = "This" THAT = "That" + E_1 = "1" + F_B = "f. b" + F_B_1 = "f-b" @dataclasses.dataclass @@ -135,7 +138,7 @@ def my_query( return executor.query_single( """\ create scalar type MyScalar extending int64; - create scalar type MyEnum extending enum; + create scalar type MyEnum extending enum<'This', 'That', '1', 'f. b', 'f-b'>; select { a := $a, From c40dc46e62bf9ba93ec898bdc1abe0ceeb750354 Mon Sep 17 00:00:00 2001 From: Fantix King Date: Fri, 26 May 2023 17:48:23 -0400 Subject: [PATCH 22/40] Update release CI (#436) --- .github/workflows/install-edgedb.sh | 15 ++++--- .github/workflows/release.yml | 62 +++++++++++++++++++---------- edgedb/_testbase.py | 7 +++- pyproject.toml | 3 ++ setup.py | 3 +- 5 files changed, 63 insertions(+), 27 deletions(-) create mode 100644 pyproject.toml diff --git a/.github/workflows/install-edgedb.sh b/.github/workflows/install-edgedb.sh index 0469c99f..f6b9e734 100755 --- a/.github/workflows/install-edgedb.sh +++ b/.github/workflows/install-edgedb.sh @@ -5,15 +5,20 @@ shopt -s nullglob srv="https://packages.edgedb.com" -curl -fL "${srv}/dist/x86_64-unknown-linux-musl/edgedb-cli" \ - > "/usr/local/bin/edgedb" +curl -fL "${srv}/dist/$(uname -m)-unknown-linux-musl/edgedb-cli" \ + > "/usr/bin/edgedb" -chmod +x "/usr/local/bin/edgedb" +chmod +x "/usr/bin/edgedb" -useradd --shell /bin/bash edgedb +if command -v useradd >/dev/null 2>&1; then + useradd --shell /bin/bash edgedb +else + # musllinux/alpine doesn't have useradd + adduser -s /bin/bash -D edgedb +fi su -l edgedb -c "edgedb server install" ln -s $(su -l edgedb -c "edgedb server info --latest --bin-path") \ - "/usr/local/bin/edgedb-server" + "/usr/bin/edgedb-server" edgedb-server --version diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c9643910..3e019fb6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -39,7 +39,7 @@ jobs: mkdir -p dist/ echo "${VERSION}" > dist/VERSION - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 with: name: dist path: dist/ @@ -52,7 +52,7 @@ jobs: PIP_DISABLE_PIP_VERSION_CHECK: 1 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 50 submodules: true @@ -65,19 +65,41 @@ jobs: pip install -U setuptools wheel pip python setup.py sdist - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 with: name: dist path: dist/*.tar.* - build-wheels: + build-wheels-matrix: needs: validate-release-request + runs-on: ubuntu-latest + outputs: + include: ${{ steps.set-matrix.outputs.include }} + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.x" + - run: pip install cibuildwheel==2.12.3 + - id: set-matrix + # Cannot test on Musl distros yet. + run: | + MATRIX_INCLUDE=$( + { + cibuildwheel --print-build-identifiers --platform linux --arch x86_64,aarch64 | grep cp | grep many | jq -nRc '{"only": inputs, "os": "ubuntu-latest"}' \ + && cibuildwheel --print-build-identifiers --platform macos --arch x86_64,arm64 | grep cp | jq -nRc '{"only": inputs, "os": "macos-latest"}' \ + && cibuildwheel --print-build-identifiers --platform windows --arch AMD64 | grep cp | jq -nRc '{"only": inputs, "os": "windows-2019"}' + } | jq -sc + ) + echo "include=$MATRIX_INCLUDE" >> $GITHUB_OUTPUT + build-wheels: + needs: build-wheels-matrix runs-on: ${{ matrix.os }} + name: Build ${{ matrix.only }} strategy: + fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-2019] - cibw_python: ["cp37-*", "cp38-*", "cp39-*", "cp310-*"] - cibw_arch: ["auto64"] + include: ${{ fromJson(needs.build-wheels-matrix.outputs.include) }} defaults: run: @@ -87,32 +109,32 @@ jobs: PIP_DISABLE_PIP_VERSION_CHECK: 1 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 50 submodules: true - name: Setup WSL - if: ${{ steps.release.outputs.version == 0 && matrix.os == 'windows-2019' }} - uses: vampire/setup-wsl@v1 + if: ${{ matrix.os == 'windows-2019' }} + uses: vampire/setup-wsl@v2 with: wsl-shell-user: edgedb additional-packages: ca-certificates curl + - name: Set up QEMU + if: runner.os == 'Linux' + uses: docker/setup-qemu-action@v2 + - name: Install EdgeDB uses: edgedb/setup-edgedb@v1 - - uses: pypa/cibuildwheel@v2.3.1 + - uses: pypa/cibuildwheel@v2.12.3 + with: + only: ${{ matrix.only }} env: CIBW_BUILD_VERBOSITY: 1 - CIBW_BUILD: ${{ matrix.cibw_python }} - # Cannot test on Musl distros yet. - CIBW_SKIP: "*-musllinux*" - CIBW_ARCHS: ${{ matrix.cibw_arch }} - # EdgeDB doesn't run on CentOS 6, so use 2014 as baseline - CIBW_MANYLINUX_X86_64_IMAGE: "quay.io/pypa/manylinux2014_x86_64" CIBW_BEFORE_ALL_LINUX: > .github/workflows/install-edgedb.sh CIBW_TEST_EXTRAS: "test" @@ -126,7 +148,7 @@ jobs: && chmod -R go+rX "$(dirname $(dirname $(dirname $PY)))" && su -l edgedb -c "EDGEDB_PYTHON_TEST_CODEGEN_CMD=$CODEGEN $PY {project}/tests/__init__.py" - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 with: name: dist path: wheelhouse/*.whl @@ -136,12 +158,12 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 5 submodules: false - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v3 with: name: dist path: dist/ diff --git a/edgedb/_testbase.py b/edgedb/_testbase.py index a228615f..47667add 100644 --- a/edgedb/_testbase.py +++ b/edgedb/_testbase.py @@ -135,7 +135,7 @@ def _start_cluster(*, cleanup_atexit=True): stderr=subprocess.STDOUT, ) - for _ in range(250): + for _ in range(600): try: with open(status_file, 'rb') as f: for line in f: @@ -171,6 +171,11 @@ def _start_cluster(*, cleanup_atexit=True): client = edgedb.create_client(password='test', **con_args) client.ensure_connected() + client.execute(""" + # Set session_idle_transaction_timeout to 5 minutes. + CONFIGURE INSTANCE SET session_idle_transaction_timeout := + '5 minutes'; + """) _default_cluster = { 'proc': p, 'client': client, diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..fed528d4 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" diff --git a/setup.py b/setup.py index 784c3d3e..27f3aa53 100644 --- a/setup.py +++ b/setup.py @@ -287,7 +287,7 @@ def finalize_options(self): author_email='hello@magic.io', url='https://github.com/edgedb/edgedb-python', license='Apache License, Version 2.0', - packages=['edgedb'], + packages=setuptools.find_packages(), provides=['edgedb'], zip_safe=False, include_package_data=True, @@ -337,6 +337,7 @@ def finalize_options(self): ], cmdclass={'build_ext': build_ext}, test_suite='tests.suite', + python_requires=">=3.7", install_requires=[ 'typing-extensions>=3.10.0; python_version < "3.8.0"', 'certifi>=2021.5.30; platform_system == "Windows"', From 045f127b238aeaff2c388c1f80925d9ef6664984 Mon Sep 17 00:00:00 2001 From: Fantix King Date: Fri, 26 May 2023 18:01:15 -0400 Subject: [PATCH 23/40] edgedb-python 1.4.0 Changes ======= * Update for rules of instance names (#423) (by @fantix in 5bc56992 for #420) * Synchronize error types (#429) (by @fantix in 03e40121) * Allow enums in array codec (#431) (by @fantix in 2de7e3fb for #408) * Prohibit concurrent operations on the same transaction object (#430) (by @fantix in f1fa612b for #130) * Fix state of transaction start (#424) (by @fantix in 297de722) * codegen: Handle non-identifier characters in enum values (#432) (by @fantix in e1ec16de for #428) Docs ==== * docs: add Code Generation to table of contents (#421) (by @AndreasPB in ffe74a17 for #421) --- edgedb/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edgedb/_version.py b/edgedb/_version.py index 27b4c5b7..762ed3f9 100644 --- a/edgedb/_version.py +++ b/edgedb/_version.py @@ -28,4 +28,4 @@ # supported platforms, publish the packages on PyPI, merge the PR # to the target branch, create a Git tag pointing to the commit. -__version__ = '1.3.0' +__version__ = '1.4.0' From 272dc5f98931b55f135fce1338c838dc8574fcd9 Mon Sep 17 00:00:00 2001 From: Fantix King Date: Thu, 25 May 2023 19:02:14 -0400 Subject: [PATCH 24/40] Add --dir option to codegen for searching .edgeql files --- edgedb/codegen/cli.py | 5 +++++ edgedb/codegen/generator.py | 20 +++++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/edgedb/codegen/cli.py b/edgedb/codegen/cli.py index 19966ca8..e8149ae6 100644 --- a/edgedb/codegen/cli.py +++ b/edgedb/codegen/cli.py @@ -56,6 +56,11 @@ def error(self, message): nargs="?", help="Generate a single file instead of one per .edgeql file.", ) +parser.add_argument( + "--dir", + action="append", + help="Only search .edgeql files under specified directories.", +) parser.add_argument( "--target", choices=["blocking", "async"], diff --git a/edgedb/codegen/generator.py b/edgedb/codegen/generator.py index 529a3fe4..d2c42699 100644 --- a/edgedb/codegen/generator.py +++ b/edgedb/codegen/generator.py @@ -145,6 +145,20 @@ def __init__(self, args: argparse.Namespace): print_msg(f"Found EdgeDB project: {C.BOLD}{self._project_dir}{C.ENDC}") self._client = edgedb.create_client(**_get_conn_args(args)) self._single_mode_files = args.file + self._search_dirs = [] + for search_dir in args.dir or []: + search_dir = pathlib.Path(search_dir).absolute() + if ( + search_dir == self._project_dir + or self._project_dir in search_dir.parents + ): + self._search_dirs.append(search_dir) + else: + print( + f"--dir '{search_dir}' is not under " + f"the project directory: {self._project_dir}" + ) + sys.exit(1) self._method_names = set() self._describe_results = [] @@ -170,7 +184,11 @@ def run(self): print(f"Failed to connect to EdgeDB instance: {e}") sys.exit(61) with self._client: - self._process_dir(self._project_dir) + if self._search_dirs: + for search_dir in self._search_dirs: + self._process_dir(search_dir) + else: + self._process_dir(self._project_dir) for target, suffix, is_async in SUFFIXES: if target in self._targets: self._async = is_async From 9cca5ef647392e033f841e75ee506d29a6777bce Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Thu, 8 Jun 2023 17:24:26 -0700 Subject: [PATCH 25/40] Implement support for vector type (#439) Vectors get decoded into array.array. Encoding supports any list-like array of numbers, but has an optimized fast path for things like array and ndarray that avoids needing to box integers. --- edgedb/protocol/codecs/codecs.pyx | 107 ++++++++++++++++++++++- tests/test_vector.py | 141 ++++++++++++++++++++++++++++++ 2 files changed, 245 insertions(+), 3 deletions(-) create mode 100644 tests/test_vector.py diff --git a/edgedb/protocol/codecs/codecs.pyx b/edgedb/protocol/codecs/codecs.pyx index 315d7abe..99685371 100644 --- a/edgedb/protocol/codecs/codecs.pyx +++ b/edgedb/protocol/codecs/codecs.pyx @@ -17,6 +17,7 @@ # +import array import decimal import uuid import datetime @@ -24,6 +25,8 @@ from edgedb import describe from edgedb import enums from edgedb.datatypes import datatypes +from libc.string cimport memcpy + include "./edb_types.pxi" @@ -347,14 +350,16 @@ cdef dict BASE_SCALAR_CODECS = {} cdef register_base_scalar_codec( str name, pgproto.encode_func encoder, - pgproto.decode_func decoder): + pgproto.decode_func decoder, + object tid = None): cdef: BaseCodec codec - tid = TYPE_IDS.get(name) if tid is None: - raise RuntimeError(f'cannot find known ID for type {name!r}') + tid = TYPE_IDS.get(name) + if tid is None: + raise RuntimeError(f'cannot find known ID for type {name!r}') tid = tid.bytes if tid in BASE_SCALAR_CODECS: @@ -510,6 +515,94 @@ cdef config_memory_decode(pgproto.CodecContext settings, FRBuffer *buf): return datatypes.ConfigMemory(bytes=bytes) +DEF PGVECTOR_MAX_DIM = (1 << 16) - 1 + + +cdef pgvector_encode_memview(pgproto.CodecContext settings, WriteBuffer buf, + float[:] obj): + cdef: + float item + Py_ssize_t objlen + Py_ssize_t i + + objlen = len(obj) + if objlen > PGVECTOR_MAX_DIM: + raise ValueError('too many elements in vector value') + + buf.write_int32(4 + objlen*4) + buf.write_int16(objlen) + buf.write_int16(0) + for i in range(objlen): + buf.write_float(obj[i]) + + +cdef pgvector_encode(pgproto.CodecContext settings, WriteBuffer buf, + object obj): + cdef: + float item + Py_ssize_t objlen + float[:] memview + Py_ssize_t i + + # If we can take a typed memview of the object, we use that. + # That is good, because it means we can consume array.array and + # numpy.ndarray without needing to unbox. + # Otherwise we take the slow path, indexing into the array using + # the normal protocol. + try: + memview = obj + except (ValueError, TypeError) as e: + pass + else: + pgvector_encode_memview(settings, buf, memview) + return + + if not _is_array_iterable(obj): + raise TypeError( + 'a sized iterable container expected (got type {!r})'.format( + type(obj).__name__)) + + # Annoyingly, this is literally identical code to the fast path... + # but the types are different in critical ways. + objlen = len(obj) + if objlen > PGVECTOR_MAX_DIM: + raise ValueError('too many elements in vector value') + + buf.write_int32(4 + objlen*4) + buf.write_int16(objlen) + buf.write_int16(0) + for i in range(objlen): + buf.write_float(obj[i]) + + +cdef object ONE_EL_ARRAY = array.array('f', [0.0]) + + +cdef pgvector_decode(pgproto.CodecContext settings, FRBuffer *buf): + cdef: + int32_t dim + Py_ssize_t size + Py_buffer view + char *p + float[:] array_view + + dim = hton.unpack_uint16(frb_read(buf, 2)) + frb_read(buf, 2) + + size = dim * 4 + p = frb_read(buf, size) + + # Create a float array with size dim + val = ONE_EL_ARRAY * dim + + # And fill it with the buffer contents + array_view = val + memcpy(&array_view[0], p, size) + val.byteswap() + + return val + + cdef checked_decimal_encode( pgproto.CodecContext settings, WriteBuffer buf, obj ): @@ -708,4 +801,12 @@ cdef register_base_scalar_codecs(): config_memory_decode) + register_base_scalar_codec( + 'ext::pgvector::vector', + pgvector_encode, + pgvector_decode, + uuid.UUID('9565dd88-04f5-11ee-a691-0b6ebe179825'), + ) + + register_base_scalar_codecs() diff --git a/tests/test_vector.py b/tests/test_vector.py new file mode 100644 index 00000000..a96119d3 --- /dev/null +++ b/tests/test_vector.py @@ -0,0 +1,141 @@ +# +# This source file is part of the EdgeDB open source project. +# +# Copyright 2019-present MagicStack Inc. and the EdgeDB authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from edgedb import _testbase as tb +import edgedb + +import array + + +# An array.array subtype where indexing doesn't work. +# We use this to verify that the non-boxing memoryview based +# fast path works, since the slow path won't work on this object. +class brokenarray(array.array): + def __getitem__(self, i): + raise AssertionError("the fast path wasn't used!") + + +class TestVector(tb.SyncQueryTestCase): + def setUp(self): + super().setUp() + + if not self.client.query_required_single(''' + select exists ( + select sys::ExtensionPackage filter .name = 'vector' + ) + '''): + self.skipTest("feature not implemented") + + self.client.execute(''' + create extension vector version '1.0' + ''') + + def tearDown(self): + try: + self.client.execute(''' + drop extension vector version '1.0' + ''') + finally: + super().tearDown() + + async def test_vector_01(self): + # if not self.client.query_required_single(''' + # select exists ( + # select sys::ExtensionPackage filter .name = 'vector' + # ) + # '''): + # self.skipTest("feature not implemented") + + # self.client.execute(''' + # create extension vector version '1.0' + # ''') + + val = self.client.query_single(''' + select '[1.5,2.0,3.8]' + ''') + self.assertTrue(isinstance(val, array.array)) + self.assertEqual(val, array.array('f', [1.5, 2.0, 3.8])) + + val = self.client.query_single( + ''' + select $0 + ''', + [3.0, 9.0, -42.5], + ) + self.assertEqual(val, '[3,9,-42.5]') + + val = self.client.query_single( + ''' + select $0 + ''', + array.array('f', [3.0, 9.0, -42.5]) + ) + self.assertEqual(val, '[3,9,-42.5]') + + val = self.client.query_single( + ''' + select $0 + ''', + array.array('i', [1, 2, 3]), + ) + self.assertEqual(val, '[1,2,3]') + + # Test that the fast-path works: if the encoder tries to + # call __getitem__ on this brokenarray, it will fail. + val = self.client.query_single( + ''' + select $0 + ''', + brokenarray('f', [3.0, 9.0, -42.5]) + ) + self.assertEqual(val, '[3,9,-42.5]') + + # I don't think it's worth adding a dependency to test this, + # but this works too: + # import numpy as np + # val = self.client.query_single( + # ''' + # select $0 + # ''', + # np.asarray([3.0, 9.0, -42.5], dtype=np.float32), + # ) + + # Some sad path tests + with self.assertRaises(edgedb.InvalidArgumentError): + self.client.query_single( + ''' + select $0 + ''', + [3.0, None, -42.5], + ) + + with self.assertRaises(edgedb.InvalidArgumentError): + self.client.query_single( + ''' + select $0 + ''', + [3.0, 'x', -42.5], + ) + + with self.assertRaises(edgedb.InvalidArgumentError): + self.client.query_single( + ''' + select $0 + ''', + 'foo', + ) From be886ecbd1b57d5583c686278e28bce46b102395 Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Fri, 9 Jun 2023 15:41:48 -0700 Subject: [PATCH 26/40] Fix the vector tests to match the final PR (#440) --- tests/test_vector.py | 44 +++++++++++++++++--------------------------- 1 file changed, 17 insertions(+), 27 deletions(-) diff --git a/tests/test_vector.py b/tests/test_vector.py index a96119d3..ede4a3d0 100644 --- a/tests/test_vector.py +++ b/tests/test_vector.py @@ -36,90 +36,80 @@ def setUp(self): if not self.client.query_required_single(''' select exists ( - select sys::ExtensionPackage filter .name = 'vector' + select sys::ExtensionPackage filter .name = 'pgvector' ) '''): self.skipTest("feature not implemented") self.client.execute(''' - create extension vector version '1.0' + create extension pgvector; ''') def tearDown(self): try: self.client.execute(''' - drop extension vector version '1.0' + drop extension pgvector; ''') finally: super().tearDown() async def test_vector_01(self): - # if not self.client.query_required_single(''' - # select exists ( - # select sys::ExtensionPackage filter .name = 'vector' - # ) - # '''): - # self.skipTest("feature not implemented") - - # self.client.execute(''' - # create extension vector version '1.0' - # ''') - val = self.client.query_single(''' - select '[1.5,2.0,3.8]' + select [1.5,2.0,3.8] ''') self.assertTrue(isinstance(val, array.array)) self.assertEqual(val, array.array('f', [1.5, 2.0, 3.8])) val = self.client.query_single( ''' - select $0 + select $0 ''', [3.0, 9.0, -42.5], ) - self.assertEqual(val, '[3,9,-42.5]') + self.assertEqual(val, '[3, 9, -42.5]') val = self.client.query_single( ''' - select $0 + select $0 ''', array.array('f', [3.0, 9.0, -42.5]) ) - self.assertEqual(val, '[3,9,-42.5]') + self.assertEqual(val, '[3, 9, -42.5]') val = self.client.query_single( ''' - select $0 + select $0 ''', array.array('i', [1, 2, 3]), ) - self.assertEqual(val, '[1,2,3]') + self.assertEqual(val, '[1, 2, 3]') # Test that the fast-path works: if the encoder tries to # call __getitem__ on this brokenarray, it will fail. val = self.client.query_single( ''' - select $0 + select $0 ''', brokenarray('f', [3.0, 9.0, -42.5]) ) - self.assertEqual(val, '[3,9,-42.5]') + self.assertEqual(val, '[3, 9, -42.5]') # I don't think it's worth adding a dependency to test this, # but this works too: # import numpy as np # val = self.client.query_single( # ''' - # select $0 + # select $0 # ''', # np.asarray([3.0, 9.0, -42.5], dtype=np.float32), # ) + # self.assertEqual(val, '[3,9,-42.5]') # Some sad path tests with self.assertRaises(edgedb.InvalidArgumentError): self.client.query_single( ''' - select $0 + select $0 ''', [3.0, None, -42.5], ) @@ -127,7 +117,7 @@ async def test_vector_01(self): with self.assertRaises(edgedb.InvalidArgumentError): self.client.query_single( ''' - select $0 + select $0 ''', [3.0, 'x', -42.5], ) @@ -135,7 +125,7 @@ async def test_vector_01(self): with self.assertRaises(edgedb.InvalidArgumentError): self.client.query_single( ''' - select $0 + select $0 ''', 'foo', ) From 662a9e31fa8b292f53e23d76f5e5c6833b743481 Mon Sep 17 00:00:00 2001 From: Elvis Pranskevichus Date: Fri, 16 Jun 2023 04:24:39 -0700 Subject: [PATCH 27/40] Lowercase org/instance name when computing Cloud instance DNS (#441) --- edgedb/con_utils.py | 3 +++ tests/shared-client-testcases | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/edgedb/con_utils.py b/edgedb/con_utils.py index d8e29881..78a450be 100644 --- a/edgedb/con_utils.py +++ b/edgedb/con_utils.py @@ -870,6 +870,9 @@ def _parse_cloud_instance_name_into_config( org_slug: str, instance_name: str, ): + org_slug = org_slug.lower() + instance_name = instance_name.lower() + label = f"{instance_name}--{org_slug}" if len(label) > DOMAIN_LABEL_MAX_LENGTH: raise ValueError( diff --git a/tests/shared-client-testcases b/tests/shared-client-testcases index 32cb54ca..f16d2c17 160000 --- a/tests/shared-client-testcases +++ b/tests/shared-client-testcases @@ -1 +1 @@ -Subproject commit 32cb54cad0d414874962c0d31f2380b3198c07ae +Subproject commit f16d2c17f502ad5bd5c35d8872cd5dc962d0fb7e From e6e6a56aff563b082561b6fd6556ae22536d63dc Mon Sep 17 00:00:00 2001 From: Paul Colomiets Date: Fri, 16 Jun 2023 19:24:18 +0300 Subject: [PATCH 28/40] Implement `database` config in project dir (#442) Complements edgedb/edgedb-cli#1086 --- edgedb/con_utils.py | 6 ++++++ tests/shared-client-testcases | 2 +- tests/test_con_utils.py | 4 ++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/edgedb/con_utils.py b/edgedb/con_utils.py index 78a450be..285790fc 100644 --- a/edgedb/con_utils.py +++ b/edgedb/con_utils.py @@ -706,6 +706,12 @@ def _parse_connect_dsn_and_args( f'project defined cloud profile ("{cloud_profile}")' ), ) + + opt_database_file = os.path.join(stash_dir, 'database') + if os.path.exists(opt_database_file): + with open(opt_database_file, 'rt') as f: + database = f.read().strip() + resolved_config.set_database(database, "project") else: raise errors.ClientConnectionError( f'Found `edgedb.toml` but the project is not initialized. ' diff --git a/tests/shared-client-testcases b/tests/shared-client-testcases index f16d2c17..b8959be8 160000 --- a/tests/shared-client-testcases +++ b/tests/shared-client-testcases @@ -1 +1 @@ -Subproject commit f16d2c17f502ad5bd5c35d8872cd5dc962d0fb7e +Subproject commit b8959be8968aceeeac2af3da7639de02b19d7030 diff --git a/tests/test_con_utils.py b/tests/test_con_utils.py index 742391ba..c2820a6e 100644 --- a/tests/test_con_utils.py +++ b/tests/test_con_utils.py @@ -32,6 +32,7 @@ class TestConUtils(unittest.TestCase): + maxDiff = 1000 error_mapping = { 'credentials_file_not_found': ( @@ -184,6 +185,9 @@ def run_testcase(self, testcase): if 'cloud-profile' in v: profile = os.path.join(dir, 'cloud-profile') files[profile] = v['cloud-profile'] + if 'database' in v: + database_file = os.path.join(dir, 'database') + files[database_file] = v['database'] del files[f] es.enter_context( From 4e435fa40da02b90b61c80e071a61ff9f36836ee Mon Sep 17 00:00:00 2001 From: Fantix King Date: Fri, 23 Jun 2023 00:03:10 +0800 Subject: [PATCH 29/40] Longer timeout to be less flaky as there's more in codegen test --- tests/test_codegen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_codegen.py b/tests/test_codegen.py index ddc6307f..35580303 100644 --- a/tests/test_codegen.py +++ b/tests/test_codegen.py @@ -62,7 +62,7 @@ async def run(*args, extra_env=None): stderr=subprocess.STDOUT, ) try: - await asyncio.wait_for(p.wait(), 30) + await asyncio.wait_for(p.wait(), 120) except asyncio.TimeoutError: p.terminate() await p.wait() From a9af42681a07462794b82e2a85d51f0d917ea1b6 Mon Sep 17 00:00:00 2001 From: Fantix King Date: Thu, 22 Jun 2023 00:15:05 +0800 Subject: [PATCH 30/40] edgedb-python 1.5.0 Changes ======= * Add --dir option to codegen for searching .edgeql files (#434) (by @fantix in ec90e35d for #434) * Implement support for vector type (#439 #440) (by @msullivan in 0bee718f for #439, 50a25ef6 for #440) * Lowercase org/instance name when computing Cloud instance DNS (#441) (by @elprans in 0f30b26e for #441) * Implement `database` config in project dir (#442) (by @tailhook in bee7327c for #442) --- edgedb/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edgedb/_version.py b/edgedb/_version.py index 762ed3f9..400f4395 100644 --- a/edgedb/_version.py +++ b/edgedb/_version.py @@ -28,4 +28,4 @@ # supported platforms, publish the packages on PyPI, merge the PR # to the target branch, create a Git tag pointing to the commit. -__version__ = '1.4.0' +__version__ = '1.5.0' From 9f44104d0afc4437e0dd57ada2b9ef0398b618e2 Mon Sep 17 00:00:00 2001 From: Elvis Pranskevichus Date: Fri, 7 Jul 2023 14:09:08 -0700 Subject: [PATCH 31/40] Implement support for new type descriptor protocol (#427) Co-authored-by: Fantix King --- edgedb/protocol/codecs/array.pyx | 2 +- edgedb/protocol/codecs/codecs.pyx | 195 ++++++++++++++++++++++++-- edgedb/protocol/codecs/namedtuple.pyx | 2 +- edgedb/protocol/codecs/range.pyx | 2 +- edgedb/protocol/codecs/tuple.pyx | 2 +- edgedb/protocol/consts.pxi | 6 +- edgedb/protocol/protocol.pyx | 13 +- 7 files changed, 196 insertions(+), 26 deletions(-) diff --git a/edgedb/protocol/codecs/array.pyx b/edgedb/protocol/codecs/array.pyx index fb0f872b..ef64fadc 100644 --- a/edgedb/protocol/codecs/array.pyx +++ b/edgedb/protocol/codecs/array.pyx @@ -156,7 +156,7 @@ cdef class ArrayCodec(BaseArrayCodec): def make_type(self, describe_context): return describe.ArrayType( desc_id=uuid.UUID(bytes=self.tid), - name=None, + name=self.type_name, element_type=self.sub_codec.make_type(describe_context), ) diff --git a/edgedb/protocol/codecs/codecs.pyx b/edgedb/protocol/codecs/codecs.pyx index 99685371..e95a84f3 100644 --- a/edgedb/protocol/codecs/codecs.pyx +++ b/edgedb/protocol/codecs/codecs.pyx @@ -52,6 +52,8 @@ DEF CTYPE_ARRAY = 6 DEF CTYPE_ENUM = 7 DEF CTYPE_INPUT_SHAPE = 8 DEF CTYPE_RANGE = 9 +DEF CTYPE_OBJECT = 10 +DEF CTYPE_COMPOUND = 11 DEF CTYPE_ANNO_TYPENAME = 255 DEF _CODECS_BUILD_CACHE_SIZE = 200 @@ -94,8 +96,9 @@ cdef class CodecsRegistry: cdef BaseCodec _build_codec(self, FRBuffer *spec, list codecs_list, protocol_version): cdef: - uint8_t t = (frb_read(spec, 1)[0]) - bytes tid = frb_read(spec, 16)[:16] + uint32_t desc_len = 0 + uint8_t t + bytes tid uint16_t els uint16_t i uint32_t str_len @@ -104,12 +107,21 @@ cdef class CodecsRegistry: BaseCodec res BaseCodec sub_codec + if protocol_version >= (2, 0): + desc_len = frb_get_len(spec) - 16 - 1 + + t = (frb_read(spec, 1)[0]) + tid = frb_read(spec, 16)[:16] + res = self.codecs.get(tid, None) if res is None: res = self.codecs_build_cache.get(tid, None) if res is not None: # We have a codec for this "tid"; advance the buffer # so that we can process the next codec. + if desc_len > 0: + frb_read(spec, desc_len) + return res if t == CTYPE_SET: frb_read(spec, 2) @@ -182,7 +194,50 @@ cdef class CodecsRegistry: sub_codec = codecs_list[pos] res = SetCodec.new(tid, sub_codec) - elif t == CTYPE_SHAPE or t == CTYPE_INPUT_SHAPE: + elif t == CTYPE_SHAPE: + if protocol_version >= (2, 0): + ephemeral_free_shape = frb_read(spec, 1)[0] + objtype_pos = hton.unpack_int16(frb_read(spec, 2)) + + els = hton.unpack_int16(frb_read(spec, 2)) + codecs = cpython.PyTuple_New(els) + names = cpython.PyTuple_New(els) + flags = cpython.PyTuple_New(els) + cards = cpython.PyTuple_New(els) + for i in range(els): + flag = hton.unpack_uint32(frb_read(spec, 4)) # flags + cardinality = frb_read(spec, 1)[0] + + str_len = hton.unpack_uint32(frb_read(spec, 4)) + name = cpythonx.PyUnicode_FromStringAndSize( + frb_read(spec, str_len), str_len) + pos = hton.unpack_int16(frb_read(spec, 2)) + + if flag & datatypes._EDGE_POINTER_IS_LINKPROP: + name = "@" + name + cpython.Py_INCREF(name) + cpython.PyTuple_SetItem(names, i, name) + + sub_codec = codecs_list[pos] + cpython.Py_INCREF(sub_codec) + cpython.PyTuple_SetItem(codecs, i, sub_codec) + + cpython.Py_INCREF(flag) + cpython.PyTuple_SetItem(flags, i, flag) + + cpython.Py_INCREF(cardinality) + cpython.PyTuple_SetItem(cards, i, cardinality) + + if protocol_version >= (2, 0): + source_type_pos = hton.unpack_int16( + frb_read(spec, 2)) + source_type = codecs_list[source_type_pos] + + res = ObjectCodec.new( + tid, names, flags, cards, codecs, t == CTYPE_INPUT_SHAPE + ) + + elif t == CTYPE_INPUT_SHAPE: els = hton.unpack_int16(frb_read(spec, 2)) codecs = cpython.PyTuple_New(els) names = cpython.PyTuple_New(els) @@ -223,15 +278,60 @@ cdef class CodecsRegistry: res = BASE_SCALAR_CODECS[tid] elif t == CTYPE_SCALAR: - pos = hton.unpack_int16(frb_read(spec, 2)) - codec = codecs_list[pos] - if type(codec) is not ScalarCodec: - raise RuntimeError( - f'a scalar codec expected for base scalar type, ' - f'got {type(codec).__name__}') - res = (codecs_list[pos]).derive(tid) + if protocol_version >= (2, 0): + str_len = hton.unpack_uint32(frb_read(spec, 4)) + type_name = cpythonx.PyUnicode_FromStringAndSize( + frb_read(spec, str_len), str_len) + schema_defined = frb_read(spec, 1)[0] + + ancestor_count = hton.unpack_int16(frb_read(spec, 2)) + ancestors = [] + for _ in range(ancestor_count): + ancestor_pos = hton.unpack_int16( + frb_read(spec, 2)) + ancestor_codec = codecs_list[ancestor_pos] + if type(ancestor_codec) is not ScalarCodec: + raise RuntimeError( + f'a scalar codec expected for base scalar type, ' + f'got {type(ancestor_codec).__name__}') + ancestors.append(ancestor_codec) + + if ancestor_count == 0: + if tid in self.base_codec_overrides: + res = self.base_codec_overrides[tid] + else: + res = BASE_SCALAR_CODECS[tid] + else: + fundamental_codec = ancestors[-1] + if type(fundamental_codec) is not ScalarCodec: + raise RuntimeError( + f'a scalar codec expected for base scalar type, ' + f'got {type(fundamental_codec).__name__}') + res = (fundamental_codec).derive(tid) + res.type_name = type_name + else: + fundamental_pos = hton.unpack_int16( + frb_read(spec, 2)) + fundamental_codec = codecs_list[fundamental_pos] + if type(fundamental_codec) is not ScalarCodec: + raise RuntimeError( + f'a scalar codec expected for base scalar type, ' + f'got {type(fundamental_codec).__name__}') + res = (fundamental_codec).derive(tid) elif t == CTYPE_TUPLE: + if protocol_version >= (2, 0): + str_len = hton.unpack_uint32(frb_read(spec, 4)) + type_name = cpythonx.PyUnicode_FromStringAndSize( + frb_read(spec, str_len), str_len) + schema_defined = frb_read(spec, 1)[0] + ancestor_count = hton.unpack_int16(frb_read(spec, 2)) + for _ in range(ancestor_count): + ancestor_pos = hton.unpack_int16( + frb_read(spec, 2)) + ancestor_codec = codecs_list[ancestor_pos] + else: + type_name = None els = hton.unpack_int16(frb_read(spec, 2)) codecs = cpython.PyTuple_New(els) for i in range(els): @@ -242,8 +342,21 @@ cdef class CodecsRegistry: cpython.PyTuple_SetItem(codecs, i, sub_codec) res = TupleCodec.new(tid, codecs) + res.type_name = type_name elif t == CTYPE_NAMEDTUPLE: + if protocol_version >= (2, 0): + str_len = hton.unpack_uint32(frb_read(spec, 4)) + type_name = cpythonx.PyUnicode_FromStringAndSize( + frb_read(spec, str_len), str_len) + schema_defined = frb_read(spec, 1)[0] + ancestor_count = hton.unpack_int16(frb_read(spec, 2)) + for _ in range(ancestor_count): + ancestor_pos = hton.unpack_int16( + frb_read(spec, 2)) + ancestor_codec = codecs_list[ancestor_pos] + else: + type_name = None els = hton.unpack_int16(frb_read(spec, 2)) codecs = cpython.PyTuple_New(els) names = cpython.PyTuple_New(els) @@ -261,8 +374,21 @@ cdef class CodecsRegistry: cpython.PyTuple_SetItem(codecs, i, sub_codec) res = NamedTupleCodec.new(tid, names, codecs) + res.type_name = type_name elif t == CTYPE_ENUM: + if protocol_version >= (2, 0): + str_len = hton.unpack_uint32(frb_read(spec, 4)) + type_name = cpythonx.PyUnicode_FromStringAndSize( + frb_read(spec, str_len), str_len) + schema_defined = frb_read(spec, 1)[0] + ancestor_count = hton.unpack_int16(frb_read(spec, 2)) + for _ in range(ancestor_count): + ancestor_pos = hton.unpack_int16( + frb_read(spec, 2)) + ancestor_codec = codecs_list[ancestor_pos] + else: + type_name = None els = hton.unpack_int16(frb_read(spec, 2)) names = cpython.PyTuple_New(els) for i in range(els): @@ -274,8 +400,21 @@ cdef class CodecsRegistry: cpython.PyTuple_SetItem(names, i, name) res = EnumCodec.new(tid, names) + res.type_name = type_name elif t == CTYPE_ARRAY: + if protocol_version >= (2, 0): + str_len = hton.unpack_uint32(frb_read(spec, 4)) + type_name = cpythonx.PyUnicode_FromStringAndSize( + frb_read(spec, str_len), str_len) + schema_defined = frb_read(spec, 1)[0] + ancestor_count = hton.unpack_int16(frb_read(spec, 2)) + for _ in range(ancestor_count): + ancestor_pos = hton.unpack_int16( + frb_read(spec, 2)) + ancestor_codec = codecs_list[ancestor_pos] + else: + type_name = None pos = hton.unpack_int16(frb_read(spec, 2)) els = hton.unpack_int16(frb_read(spec, 2)) if els != 1: @@ -285,11 +424,35 @@ cdef class CodecsRegistry: dim_len = hton.unpack_int32(frb_read(spec, 4)) sub_codec = codecs_list[pos] res = ArrayCodec.new(tid, sub_codec, dim_len) + res.type_name = type_name elif t == CTYPE_RANGE: + if protocol_version >= (2, 0): + str_len = hton.unpack_uint32(frb_read(spec, 4)) + type_name = cpythonx.PyUnicode_FromStringAndSize( + frb_read(spec, str_len), str_len) + schema_defined = frb_read(spec, 1)[0] + ancestor_count = hton.unpack_int16(frb_read(spec, 2)) + for _ in range(ancestor_count): + ancestor_pos = hton.unpack_int16( + frb_read(spec, 2)) + ancestor_codec = codecs_list[ancestor_pos] + else: + type_name = None pos = hton.unpack_int16(frb_read(spec, 2)) sub_codec = codecs_list[pos] res = RangeCodec.new(tid, sub_codec) + res.type_name = type_name + + elif t == CTYPE_OBJECT and protocol_version >= (2, 0): + # Ignore + frb_read(spec, desc_len) + res = NULL_CODEC + + elif t == CTYPE_COMPOUND and protocol_version >= (2, 0): + # Ignore + frb_read(spec, desc_len) + res = NULL_CODEC else: raise NotImplementedError( @@ -321,6 +484,7 @@ cdef class CodecsRegistry: cdef BaseCodec build_codec(self, bytes spec, protocol_version): cdef: FRBuffer buf + FRBuffer elem_buf BaseCodec res list codecs_list @@ -331,7 +495,16 @@ cdef class CodecsRegistry: codecs_list = [] while frb_get_len(&buf): - res = self._build_codec(&buf, codecs_list, protocol_version) + if protocol_version >= (2, 0): + desc_len = hton.unpack_int32(frb_read(&buf, 4)) + frb_slice_from(&elem_buf, &buf, desc_len) + res = self._build_codec( + &elem_buf, codecs_list, protocol_version) + if frb_get_len(&elem_buf): + raise RuntimeError( + f'unexpected trailing data in type descriptor datum') + else: + res = self._build_codec(&buf, codecs_list, protocol_version) if res is None: # An annotation; ignore. continue diff --git a/edgedb/protocol/codecs/namedtuple.pyx b/edgedb/protocol/codecs/namedtuple.pyx index 930ee0ee..6514ef36 100644 --- a/edgedb/protocol/codecs/namedtuple.pyx +++ b/edgedb/protocol/codecs/namedtuple.pyx @@ -78,7 +78,7 @@ cdef class NamedTupleCodec(BaseNamedRecordCodec): def make_type(self, describe_context): return describe.NamedTupleType( desc_id=uuid.UUID(bytes=self.tid), - name=None, + name=self.type_name, element_types={ field: codec.make_type(describe_context) for field, codec in zip( diff --git a/edgedb/protocol/codecs/range.pyx b/edgedb/protocol/codecs/range.pyx index 9555d969..3d608fd0 100644 --- a/edgedb/protocol/codecs/range.pyx +++ b/edgedb/protocol/codecs/range.pyx @@ -143,6 +143,6 @@ cdef class RangeCodec(BaseCodec): def make_type(self, describe_context): return describe.RangeType( desc_id=uuid.UUID(bytes=self.tid), - name=None, + name=self.type_name, value_type=self.sub_codec.make_type(describe_context), ) diff --git a/edgedb/protocol/codecs/tuple.pyx b/edgedb/protocol/codecs/tuple.pyx index 68ed0352..d29415d6 100644 --- a/edgedb/protocol/codecs/tuple.pyx +++ b/edgedb/protocol/codecs/tuple.pyx @@ -81,7 +81,7 @@ cdef class TupleCodec(BaseRecordCodec): def make_type(self, describe_context): return describe.TupleType( desc_id=uuid.UUID(bytes=self.tid), - name=None, + name=self.type_name, element_types=tuple( codec.make_type(describe_context) for codec in self.fields_codecs diff --git a/edgedb/protocol/consts.pxi b/edgedb/protocol/consts.pxi index 6d93c07f..10fe4cf8 100644 --- a/edgedb/protocol/consts.pxi +++ b/edgedb/protocol/consts.pxi @@ -61,8 +61,8 @@ DEF TRANS_STATUS_IDLE = b'I' DEF TRANS_STATUS_INTRANS = b'T' DEF TRANS_STATUS_ERROR = b'E' -DEF PROTO_VER_MAJOR = 1 +DEF PROTO_VER_MAJOR = 2 DEF PROTO_VER_MINOR = 0 -DEF LEGACY_PROTO_VER_MAJOR = 0 -DEF LEGACY_PROTO_VER_MINOR_MIN = 13 +DEF MIN_PROTO_VER_MAJOR = 0 +DEF MIN_PROTO_VER_MINOR = 13 diff --git a/edgedb/protocol/protocol.pyx b/edgedb/protocol/protocol.pyx index b973b885..469d3cb4 100644 --- a/edgedb/protocol/protocol.pyx +++ b/edgedb/protocol/protocol.pyx @@ -148,7 +148,7 @@ cdef class SansIOProtocol: self.internal_reg = CodecsRegistry() self.server_settings = {} self.reset_status() - self.protocol_version = (PROTO_VER_MAJOR, 0) + self.protocol_version = (PROTO_VER_MAJOR, PROTO_VER_MINOR) self.state_type_id = NULL_CODEC_ID self.state_codec = None @@ -873,22 +873,19 @@ cdef class SansIOProtocol: minor = self.buffer.read_int16() # TODO: drop this branch when dropping protocol_v0 - if major == LEGACY_PROTO_VER_MAJOR: + if major == 0: self.is_legacy = True self.ignore_headers() self.buffer.finish_message() - if major != PROTO_VER_MAJOR and not ( - major == LEGACY_PROTO_VER_MAJOR and - minor >= LEGACY_PROTO_VER_MINOR_MIN - ): + if (major, minor) < (MIN_PROTO_VER_MAJOR, MIN_PROTO_VER_MINOR): raise errors.ClientConnectionError( f'the server requested an unsupported version of ' f'the protocol: {major}.{minor}' ) - - self.protocol_version = (major, minor) + else: + self.protocol_version = (major, minor) elif mtype == AUTH_REQUEST_MSG: # Authentication... From 0db7ffbf3be90a874c3b038dfc17acd51d372e93 Mon Sep 17 00:00:00 2001 From: Elvis Pranskevichus Date: Thu, 20 Jul 2023 16:37:37 -0700 Subject: [PATCH 32/40] Sync errors (#449) Also fixes a test broken by parser rework in nightly. --- edgedb/__init__.py | 8 ++++++++ edgedb/errors/__init__.py | 6 ++++++ tests/test_async_query.py | 12 ++++++++---- tests/test_sync_query.py | 12 ++++++++---- 4 files changed, 30 insertions(+), 8 deletions(-) diff --git a/edgedb/__init__.py b/edgedb/__init__.py index bf565921..353bdfcd 100644 --- a/edgedb/__init__.py +++ b/edgedb/__init__.py @@ -137,6 +137,7 @@ DuplicateFunctionDefinitionError, DuplicateConstraintDefinitionError, DuplicateCastDefinitionError, + DuplicateMigrationError, SessionTimeoutError, IdleSessionTimeoutError, QueryTimeoutError, @@ -147,6 +148,7 @@ DivisionByZeroError, NumericOutOfRangeError, AccessPolicyError, + QueryAssertionError, IntegrityError, ConstraintViolationError, CardinalityViolationError, @@ -155,11 +157,13 @@ TransactionConflictError, TransactionSerializationError, TransactionDeadlockError, + WatchError, ConfigurationError, AccessError, AuthenticationError, AvailabilityError, BackendUnavailableError, + ServerOfflineError, BackendError, UnsupportedBackendFeatureError, LogMessage, @@ -234,6 +238,7 @@ "DuplicateFunctionDefinitionError", "DuplicateConstraintDefinitionError", "DuplicateCastDefinitionError", + "DuplicateMigrationError", "SessionTimeoutError", "IdleSessionTimeoutError", "QueryTimeoutError", @@ -244,6 +249,7 @@ "DivisionByZeroError", "NumericOutOfRangeError", "AccessPolicyError", + "QueryAssertionError", "IntegrityError", "ConstraintViolationError", "CardinalityViolationError", @@ -252,11 +258,13 @@ "TransactionConflictError", "TransactionSerializationError", "TransactionDeadlockError", + "WatchError", "ConfigurationError", "AccessError", "AuthenticationError", "AvailabilityError", "BackendUnavailableError", + "ServerOfflineError", "BackendError", "UnsupportedBackendFeatureError", "LogMessage", diff --git a/edgedb/errors/__init__.py b/edgedb/errors/__init__.py index cfe55e1d..424edc91 100644 --- a/edgedb/errors/__init__.py +++ b/edgedb/errors/__init__.py @@ -94,6 +94,7 @@ 'AuthenticationError', 'AvailabilityError', 'BackendUnavailableError', + 'ServerOfflineError', 'BackendError', 'UnsupportedBackendFeatureError', 'LogMessage', @@ -440,6 +441,11 @@ class BackendUnavailableError(AvailabilityError): tags = frozenset({SHOULD_RETRY}) +class ServerOfflineError(AvailabilityError): + _code = 0x_08_00_00_02 + tags = frozenset({SHOULD_RECONNECT, SHOULD_RETRY}) + + class BackendError(EdgeDBError): _code = 0x_09_00_00_00 diff --git a/tests/test_async_query.py b/tests/test_async_query.py index bd9fd3a9..6058a980 100644 --- a/tests/test_async_query.py +++ b/tests/test_async_query.py @@ -61,12 +61,16 @@ async def test_async_parse_error_recover_01(self): with self.assertRaises(edgedb.EdgeQLSyntaxError): await self.client.query('select syntax error') - with self.assertRaisesRegex(edgedb.EdgeQLSyntaxError, - 'Unexpected end of line'): + with self.assertRaisesRegex( + edgedb.EdgeQLSyntaxError, + r"(Unexpected end of line)|(Missing '\)')" + ): await self.client.query('select (') - with self.assertRaisesRegex(edgedb.EdgeQLSyntaxError, - 'Unexpected end of line'): + with self.assertRaisesRegex( + edgedb.EdgeQLSyntaxError, + r"(Unexpected end of line)|(Missing '\)')" + ): await self.client.query_json('select (') for _ in range(10): diff --git a/tests/test_sync_query.py b/tests/test_sync_query.py index 8dd35c3c..399df28b 100644 --- a/tests/test_sync_query.py +++ b/tests/test_sync_query.py @@ -53,12 +53,16 @@ def test_sync_parse_error_recover_01(self): with self.assertRaises(edgedb.EdgeQLSyntaxError): self.client.query('select syntax error') - with self.assertRaisesRegex(edgedb.EdgeQLSyntaxError, - 'Unexpected end of line'): + with self.assertRaisesRegex( + edgedb.EdgeQLSyntaxError, + r"(Unexpected end of line)|(Missing '\)')" + ): self.client.query('select (') - with self.assertRaisesRegex(edgedb.EdgeQLSyntaxError, - 'Unexpected end of line'): + with self.assertRaisesRegex( + edgedb.EdgeQLSyntaxError, + r"(Unexpected end of line)|(Missing '\)')" + ): self.client.query_json('select (') for _ in range(10): From 869f8d525ec06b1a26928898ac5b44529e2503a2 Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Mon, 24 Jul 2023 10:41:03 -0700 Subject: [PATCH 33/40] Don't depend on exact syntax errors in tests (#451) We have tests for syntax errors in the server, we're just testing the binding/protocol flow here. (And other places test that messages come through more generally.) --- .github/workflows/tests.yml | 1 + tests/test_async_query.py | 12 +++--------- tests/test_sync_query.py | 10 ++-------- 3 files changed, 6 insertions(+), 17 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ece1f886..050de749 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -23,6 +23,7 @@ jobs: PIP_DISABLE_PIP_VERSION_CHECK: 1 strategy: + fail-fast: false matrix: python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] edgedb-version: [stable , nightly] diff --git a/tests/test_async_query.py b/tests/test_async_query.py index 6058a980..44a00209 100644 --- a/tests/test_async_query.py +++ b/tests/test_async_query.py @@ -61,16 +61,10 @@ async def test_async_parse_error_recover_01(self): with self.assertRaises(edgedb.EdgeQLSyntaxError): await self.client.query('select syntax error') - with self.assertRaisesRegex( - edgedb.EdgeQLSyntaxError, - r"(Unexpected end of line)|(Missing '\)')" - ): + with self.assertRaises(edgedb.EdgeQLSyntaxError): await self.client.query('select (') - with self.assertRaisesRegex( - edgedb.EdgeQLSyntaxError, - r"(Unexpected end of line)|(Missing '\)')" - ): + with self.assertRaises(edgedb.EdgeQLSyntaxError): await self.client.query_json('select (') for _ in range(10): @@ -854,7 +848,7 @@ async def exec_to_fail(): g.create_task(exec_to_fail()) - await asyncio.wait_for(fut, 1) + await asyncio.wait_for(fut, 5) await asyncio.sleep(0.1) with self.assertRaises(asyncio.TimeoutError): diff --git a/tests/test_sync_query.py b/tests/test_sync_query.py index 399df28b..622fceed 100644 --- a/tests/test_sync_query.py +++ b/tests/test_sync_query.py @@ -53,16 +53,10 @@ def test_sync_parse_error_recover_01(self): with self.assertRaises(edgedb.EdgeQLSyntaxError): self.client.query('select syntax error') - with self.assertRaisesRegex( - edgedb.EdgeQLSyntaxError, - r"(Unexpected end of line)|(Missing '\)')" - ): + with self.assertRaises(edgedb.EdgeQLSyntaxError): self.client.query('select (') - with self.assertRaisesRegex( - edgedb.EdgeQLSyntaxError, - r"(Unexpected end of line)|(Missing '\)')" - ): + with self.assertRaises(edgedb.EdgeQLSyntaxError): self.client.query_json('select (') for _ in range(10): From 3a59bf509c29f109e0c2d466a431720ab91f7c6a Mon Sep 17 00:00:00 2001 From: Victor Petrovykh Date: Mon, 24 Jul 2023 19:36:10 -0400 Subject: [PATCH 34/40] Fix an error in string representation of RelativeDuration. (#453) RelativeDuration was getting the '-' for the seconds component wrong in some cases. Specifically, when fractional seconds were 0, but larger units (hours or seconds) were non zero and negative. Also add test cases for similar situtation with other duration types. --- edgedb/datatypes/relative_duration.pyx | 7 +- tests/test_datetime.py | 120 +++++++++++++++++++++++++ 2 files changed, 126 insertions(+), 1 deletion(-) diff --git a/edgedb/datatypes/relative_duration.pyx b/edgedb/datatypes/relative_duration.pyx index cf2c0ab9..27c0c648 100644 --- a/edgedb/datatypes/relative_duration.pyx +++ b/edgedb/datatypes/relative_duration.pyx @@ -108,7 +108,12 @@ cdef class RelativeDuration: buf.append(f'{min}M') if sec or fsec: - sign = '-' if min < 0 or fsec < 0 else '' + # If the original microseconds are negative we expect '-' in front + # of all non-zero hour/min/second components. The hour/min sign + # can be taken as is, but seconds are constructed out of sec and + # fsec parts, both of which have their own sign and thus we cannot + # just use their string representations directly. + sign = '-' if self.microseconds < 0 else '' buf.append(f'{sign}{abs(sec)}') if fsec: diff --git a/tests/test_datetime.py b/tests/test_datetime.py index 08199077..ff7dfbc0 100644 --- a/tests/test_datetime.py +++ b/tests/test_datetime.py @@ -25,6 +25,11 @@ from edgedb.datatypes.datatypes import RelativeDuration, DateDuration +USECS_PER_HOUR = 3600000000 +USECS_PER_MINUTE = 60000000 +USECS_PER_SEC = 1000000 + + class TestDatetimeTypes(tb.SyncQueryTestCase): async def test_duration_01(self): @@ -60,6 +65,57 @@ async def test_duration_01(self): ''', durs) self.assertEqual(list(durs_from_db), durs) + async def test_duration_02(self): + # Make sure that when we break down the microseconds into the bigger + # components we still get consistent values. + tdn1h = timedelta(microseconds=-USECS_PER_HOUR) + tdn1m = timedelta(microseconds=-USECS_PER_MINUTE) + tdn1s = timedelta(microseconds=-USECS_PER_SEC) + tdn1us = timedelta(microseconds=-1) + durs = [ + ( + tdn1h, tdn1m, + timedelta(microseconds=-USECS_PER_HOUR - USECS_PER_MINUTE), + ), + ( + tdn1h, tdn1s, + timedelta(microseconds=-USECS_PER_HOUR - USECS_PER_SEC), + ), + ( + tdn1m, tdn1s, + timedelta(microseconds=-USECS_PER_MINUTE - USECS_PER_SEC), + ), + ( + tdn1h, tdn1us, + timedelta(microseconds=-USECS_PER_HOUR - 1), + ), + ( + tdn1m, tdn1us, + timedelta(microseconds=-USECS_PER_MINUTE - 1), + ), + ( + tdn1s, tdn1us, + timedelta(microseconds=-USECS_PER_SEC - 1), + ), + ] + + # Test encode + durs_enc = self.client.query(''' + WITH args := array_unpack( + >>$0) + SELECT args.0 + args.1 = args.2; + ''', durs) + + # Test decode + durs_dec = self.client.query(''' + WITH args := array_unpack( + >>$0) + SELECT (args.0 + args.1, args.2); + ''', durs) + + self.assertEqual(durs_enc, [True] * len(durs)) + self.assertEqual(list(durs_dec), [(d[2], d[2]) for d in durs]) + async def test_relative_duration_01(self): try: self.client.query("SELECT '1y'") @@ -124,6 +180,41 @@ async def test_relative_duration_02(self): self.assertEqual(repr(d1), '') + async def test_relative_duration_03(self): + # Make sure that when we break down the microseconds into the bigger + # components we still get the sign correctly in string + # representation. + durs = [ + RelativeDuration(microseconds=-USECS_PER_HOUR), + RelativeDuration(microseconds=-USECS_PER_MINUTE), + RelativeDuration(microseconds=-USECS_PER_SEC), + RelativeDuration(microseconds=-USECS_PER_HOUR - USECS_PER_MINUTE), + RelativeDuration(microseconds=-USECS_PER_HOUR - USECS_PER_SEC), + RelativeDuration(microseconds=-USECS_PER_MINUTE - USECS_PER_SEC), + RelativeDuration(microseconds=-USECS_PER_HOUR - USECS_PER_MINUTE - + USECS_PER_SEC), + RelativeDuration(microseconds=-USECS_PER_HOUR - 1), + RelativeDuration(microseconds=-USECS_PER_MINUTE - 1), + RelativeDuration(microseconds=-USECS_PER_SEC - 1), + RelativeDuration(microseconds=-1), + ] + + # Test that RelativeDuration.__str__ formats the + # same as + durs_as_text = self.client.query(''' + WITH args := array_unpack(>$0) + SELECT args; + ''', durs) + + # Test encode/decode roundtrip + durs_from_db = self.client.query(''' + WITH args := array_unpack(>$0) + SELECT args; + ''', durs) + + self.assertEqual(durs_as_text, [str(d) for d in durs]) + self.assertEqual(list(durs_from_db), durs) + async def test_date_duration_01(self): try: self.client.query("SELECT '1y'") @@ -168,3 +259,32 @@ async def test_date_duration_01(self): self.assertEqual(db_dur, str(client_dur)) self.assertEqual(list(durs_from_db), durs) + + async def test_date_duration_02(self): + # Make sure that when we break down the microseconds into the bigger + # components we still get the sign correctly in string + # representation. + durs = [ + DateDuration(months=11), + DateDuration(months=12), + DateDuration(months=13), + DateDuration(months=-11), + DateDuration(months=-12), + DateDuration(months=-13), + ] + + # Test that DateDuration.__str__ formats the + # same as + durs_as_text = self.client.query(''' + WITH args := array_unpack(>$0) + SELECT args; + ''', durs) + + # Test encode/decode roundtrip + durs_from_db = self.client.query(''' + WITH args := array_unpack(>$0) + SELECT args; + ''', durs) + + self.assertEqual(durs_as_text, [str(d) for d in durs]) + self.assertEqual(list(durs_from_db), durs) From 91608098153ae43ba83aaef1fdcba7e7e70dc329 Mon Sep 17 00:00:00 2001 From: Victor Petrovykh Date: Mon, 24 Jul 2023 23:14:35 -0400 Subject: [PATCH 35/40] Add multirange support. (#452) Add multirange codec. Adjust edgedb.Range and create edgedb.MultiRange class as Python representation of ranges and multiranges. --- edgedb/__init__.py | 2 +- edgedb/datatypes/range.py | 58 +++++++++-- edgedb/describe.py | 5 + edgedb/protocol/codecs/array.pyx | 3 +- edgedb/protocol/codecs/base.pyx | 2 +- edgedb/protocol/codecs/codecs.pyx | 22 ++++ edgedb/protocol/codecs/range.pxd | 16 +++ edgedb/protocol/codecs/range.pyx | 122 ++++++++++++++++++++-- tests/datatypes/test_datatypes.py | 162 ++++++++++++++++++++++++++++++ tests/test_async_query.py | 77 ++++++++++++++ 10 files changed, 449 insertions(+), 20 deletions(-) diff --git a/edgedb/__init__.py b/edgedb/__init__.py index 353bdfcd..5ba24795 100644 --- a/edgedb/__init__.py +++ b/edgedb/__init__.py @@ -25,7 +25,7 @@ Tuple, NamedTuple, EnumValue, RelativeDuration, DateDuration, ConfigMemory ) from edgedb.datatypes.datatypes import Set, Object, Array, Link, LinkSet -from edgedb.datatypes.range import Range +from edgedb.datatypes.range import Range, MultiRange from .abstract import ( Executor, AsyncIOExecutor, ReadOnlyExecutor, AsyncIOReadOnlyExecutor, diff --git a/edgedb/datatypes/range.py b/edgedb/datatypes/range.py index eaeb4bcb..e3fd3d1e 100644 --- a/edgedb/datatypes/range.py +++ b/edgedb/datatypes/range.py @@ -16,8 +16,8 @@ # limitations under the License. # -from typing import Generic, Optional, TypeVar - +from typing import (TypeVar, Any, Generic, Optional, Iterable, Iterator, + Sequence) T = TypeVar("T") @@ -78,8 +78,10 @@ def is_empty(self) -> bool: def __bool__(self): return not self.is_empty() - def __eq__(self, other): - if not isinstance(other, Range): + def __eq__(self, other) -> bool: + if isinstance(other, Range): + o = other + else: return NotImplemented return ( @@ -87,13 +89,13 @@ def __eq__(self, other): self._upper, self._inc_lower, self._inc_upper, - self._empty - ) == ( - other._lower, - other._upper, - other._inc_lower, - other._inc_upper, self._empty, + ) == ( + o._lower, + o._upper, + o._inc_lower, + o._inc_upper, + o._empty, ) def __hash__(self) -> int: @@ -125,3 +127,39 @@ def __str__(self) -> str: return f"" __repr__ = __str__ + + +# TODO: maybe we should implement range and multirange operations as well as +# normalization of the sub-ranges? +class MultiRange(Iterable[T]): + + _ranges: Sequence[T] + + def __init__(self, iterable: Optional[Iterable[T]] = None) -> None: + if iterable is not None: + self._ranges = tuple(iterable) + else: + self._ranges = tuple() + + def __len__(self) -> int: + return len(self._ranges) + + def __iter__(self) -> Iterator[T]: + return iter(self._ranges) + + def __reversed__(self) -> Iterator[T]: + return reversed(self._ranges) + + def __str__(self) -> str: + return f'' + + __repr__ = __str__ + + def __eq__(self, other: Any) -> bool: + if isinstance(other, MultiRange): + return set(self._ranges) == set(other._ranges) + else: + return NotImplemented + + def __hash__(self) -> int: + return hash(self._ranges) diff --git a/edgedb/describe.py b/edgedb/describe.py index 05c16398..92e38854 100644 --- a/edgedb/describe.py +++ b/edgedb/describe.py @@ -91,3 +91,8 @@ class SparseObjectType(ObjectType): @dataclasses.dataclass(frozen=True) class RangeType(AnyType): value_type: AnyType + + +@dataclasses.dataclass(frozen=True) +class MultiRangeType(AnyType): + value_type: AnyType diff --git a/edgedb/protocol/codecs/array.pyx b/edgedb/protocol/codecs/array.pyx index ef64fadc..2906f1e8 100644 --- a/edgedb/protocol/codecs/array.pyx +++ b/edgedb/protocol/codecs/array.pyx @@ -39,7 +39,8 @@ cdef class BaseArrayCodec(BaseCodec): if not isinstance( self.sub_codec, - (ScalarCodec, TupleCodec, NamedTupleCodec, RangeCodec, EnumCodec) + (ScalarCodec, TupleCodec, NamedTupleCodec, EnumCodec, + RangeCodec, MultiRangeCodec) ): raise TypeError( 'only arrays of scalars are supported (got type {!r})'.format( diff --git a/edgedb/protocol/codecs/base.pyx b/edgedb/protocol/codecs/base.pyx index 3bd52bb0..a40f6e57 100644 --- a/edgedb/protocol/codecs/base.pyx +++ b/edgedb/protocol/codecs/base.pyx @@ -149,7 +149,7 @@ cdef class BaseRecordCodec(BaseCodec): if not isinstance( codec, (ScalarCodec, ArrayCodec, TupleCodec, NamedTupleCodec, - EnumCodec, RangeCodec), + EnumCodec, RangeCodec, MultiRangeCodec), ): self.encoder_flags |= RECORD_ENCODER_INVALID break diff --git a/edgedb/protocol/codecs/codecs.pyx b/edgedb/protocol/codecs/codecs.pyx index e95a84f3..ab97e498 100644 --- a/edgedb/protocol/codecs/codecs.pyx +++ b/edgedb/protocol/codecs/codecs.pyx @@ -54,6 +54,7 @@ DEF CTYPE_INPUT_SHAPE = 8 DEF CTYPE_RANGE = 9 DEF CTYPE_OBJECT = 10 DEF CTYPE_COMPOUND = 11 +DEF CTYPE_MULTIRANGE = 12 DEF CTYPE_ANNO_TYPENAME = 255 DEF _CODECS_BUILD_CACHE_SIZE = 200 @@ -165,6 +166,9 @@ cdef class CodecsRegistry: elif t == CTYPE_RANGE: frb_read(spec, 2) + elif t == CTYPE_MULTIRANGE: + frb_read(spec, 2) + elif t == CTYPE_ENUM: els = hton.unpack_int16(frb_read(spec, 2)) for i in range(els): @@ -444,6 +448,24 @@ cdef class CodecsRegistry: res = RangeCodec.new(tid, sub_codec) res.type_name = type_name + elif t == CTYPE_MULTIRANGE: + if protocol_version >= (2, 0): + str_len = hton.unpack_uint32(frb_read(spec, 4)) + type_name = cpythonx.PyUnicode_FromStringAndSize( + frb_read(spec, str_len), str_len) + schema_defined = frb_read(spec, 1)[0] + ancestor_count = hton.unpack_int16(frb_read(spec, 2)) + for _ in range(ancestor_count): + ancestor_pos = hton.unpack_int16( + frb_read(spec, 2)) + ancestor_codec = codecs_list[ancestor_pos] + else: + type_name = None + pos = hton.unpack_int16(frb_read(spec, 2)) + sub_codec = codecs_list[pos] + res = MultiRangeCodec.new(tid, sub_codec) + res.type_name = type_name + elif t == CTYPE_OBJECT and protocol_version >= (2, 0): # Ignore frb_read(spec, desc_len) diff --git a/edgedb/protocol/codecs/range.pxd b/edgedb/protocol/codecs/range.pxd index 13d642f2..9b232b10 100644 --- a/edgedb/protocol/codecs/range.pxd +++ b/edgedb/protocol/codecs/range.pxd @@ -25,3 +25,19 @@ cdef class RangeCodec(BaseCodec): @staticmethod cdef BaseCodec new(bytes tid, BaseCodec sub_codec) + + @staticmethod + cdef encode_range(WriteBuffer buf, object obj, BaseCodec sub_codec) + + @staticmethod + cdef decode_range(FRBuffer *buf, BaseCodec sub_codec) + + +@cython.final +cdef class MultiRangeCodec(BaseCodec): + + cdef: + BaseCodec sub_codec + + @staticmethod + cdef BaseCodec new(bytes tid, BaseCodec sub_codec) diff --git a/edgedb/protocol/codecs/range.pyx b/edgedb/protocol/codecs/range.pyx index 3d608fd0..ea573b89 100644 --- a/edgedb/protocol/codecs/range.pyx +++ b/edgedb/protocol/codecs/range.pyx @@ -46,7 +46,8 @@ cdef class RangeCodec(BaseCodec): return codec - cdef encode(self, WriteBuffer buf, object obj): + @staticmethod + cdef encode_range(WriteBuffer buf, object obj, BaseCodec sub_codec): cdef: uint8_t flags = 0 WriteBuffer sub_data @@ -56,10 +57,10 @@ cdef class RangeCodec(BaseCodec): bint inc_upper = obj.inc_upper bint empty = obj.is_empty() - if not isinstance(self.sub_codec, ScalarCodec): + if not isinstance(sub_codec, ScalarCodec): raise TypeError( 'only scalar ranges are supported (got type {!r})'.format( - type(self.sub_codec).__name__ + type(sub_codec).__name__ ) ) @@ -78,14 +79,14 @@ cdef class RangeCodec(BaseCodec): sub_data = WriteBuffer.new() if lower is not None: try: - self.sub_codec.encode(sub_data, lower) + sub_codec.encode(sub_data, lower) except TypeError as e: raise ValueError( 'invalid range lower bound: {}'.format( e.args[0])) from None if upper is not None: try: - self.sub_codec.encode(sub_data, upper) + sub_codec.encode(sub_data, upper) except TypeError as e: raise ValueError( 'invalid range upper bound: {}'.format( @@ -95,7 +96,8 @@ cdef class RangeCodec(BaseCodec): buf.write_byte(flags) buf.write_buffer(sub_data) - cdef decode(self, FRBuffer *buf): + @staticmethod + cdef decode_range(FRBuffer *buf, BaseCodec sub_codec): cdef: uint8_t flags = frb_read(buf, 1)[0] bint empty = (flags & RANGE_EMPTY) != 0 @@ -107,7 +109,6 @@ cdef class RangeCodec(BaseCodec): object upper = None int32_t sub_len FRBuffer sub_buf - BaseCodec sub_codec = self.sub_codec if has_lower: sub_len = hton.unpack_int32(frb_read(buf, 4)) @@ -137,6 +138,12 @@ cdef class RangeCodec(BaseCodec): empty=empty, ) + cdef encode(self, WriteBuffer buf, object obj): + RangeCodec.encode_range(buf, obj, self.sub_codec) + + cdef decode(self, FRBuffer *buf): + return RangeCodec.decode_range(buf, self.sub_codec) + cdef dump(self, int level = 0): return f'{level * " "}{self.name}\n{self.sub_codec.dump(level + 1)}' @@ -146,3 +153,104 @@ cdef class RangeCodec(BaseCodec): name=self.type_name, value_type=self.sub_codec.make_type(describe_context), ) + + +@cython.final +cdef class MultiRangeCodec(BaseCodec): + + def __cinit__(self): + self.sub_codec = None + + @staticmethod + cdef BaseCodec new(bytes tid, BaseCodec sub_codec): + cdef: + MultiRangeCodec codec + + codec = MultiRangeCodec.__new__(MultiRangeCodec) + + codec.tid = tid + codec.name = 'MultiRange' + codec.sub_codec = sub_codec + + return codec + + cdef encode(self, WriteBuffer buf, object obj): + cdef: + WriteBuffer elem_data + Py_ssize_t objlen + Py_ssize_t elem_data_len + + if not isinstance(self.sub_codec, ScalarCodec): + raise TypeError( + f'only scalar multiranges are supported (got type ' + f'{type(self.sub_codec).__name__!r})' + ) + + if not _is_array_iterable(obj): + raise TypeError( + f'a sized iterable container expected (got type ' + f'{type(obj).__name__!r})' + ) + + objlen = len(obj) + if objlen > _MAXINT32: + raise ValueError('too many elements in multirange value') + + elem_data = WriteBuffer.new() + for item in obj: + try: + RangeCodec.encode_range(elem_data, item, self.sub_codec) + except TypeError as e: + raise ValueError( + f'invalid multirange element: {e.args[0]}') from None + + elem_data_len = elem_data.len() + if elem_data_len > _MAXINT32 - 4: + raise OverflowError( + f'size of encoded multirange datum exceeds the maximum ' + f'allowed {_MAXINT32 - 4} bytes') + + # Datum length + buf.write_int32(4 + elem_data_len) + # Number of elements in multirange + buf.write_int32(objlen) + buf.write_buffer(elem_data) + + cdef decode(self, FRBuffer *buf): + cdef: + Py_ssize_t elem_count = hton.unpack_int32( + frb_read(buf, 4)) + object result + Py_ssize_t i + int32_t elem_len + FRBuffer elem_buf + + result = cpython.PyList_New(elem_count) + for i in range(elem_count): + elem_len = hton.unpack_int32(frb_read(buf, 4)) + if elem_len == -1: + raise RuntimeError( + 'unexpected NULL element in multirange value') + else: + frb_slice_from(&elem_buf, buf, elem_len) + elem = RangeCodec.decode_range(&elem_buf, self.sub_codec) + if frb_get_len(&elem_buf): + raise RuntimeError( + f'unexpected trailing data in buffer after ' + f'multirange element decoding: ' + f'{frb_get_len(&elem_buf)}') + + cpython.Py_INCREF(elem) + cpython.PyList_SET_ITEM(result, i, elem) + + return range_mod.MultiRange(result) + + cdef dump(self, int level = 0): + return f'{level * " "}{self.name}\n{self.sub_codec.dump(level + 1)}' + + def make_type(self, describe_context): + return describe.MultiRangeType( + desc_id=uuid.UUID(bytes=self.tid), + name=self.type_name, + value_type=self.sub_codec.make_type(describe_context), + ) \ No newline at end of file diff --git a/tests/datatypes/test_datatypes.py b/tests/datatypes/test_datatypes.py index eaff8aff..741489d4 100644 --- a/tests/datatypes/test_datatypes.py +++ b/tests/datatypes/test_datatypes.py @@ -1003,3 +1003,165 @@ def test_array_6(self): self.assertNotEqual( edgedb.Array([1, 2, 3]), False) + + +class TestRange(unittest.TestCase): + + def test_range_empty_1(self): + t = edgedb.Range(empty=True) + self.assertEqual(t.lower, None) + self.assertEqual(t.upper, None) + self.assertFalse(t.inc_lower) + self.assertFalse(t.inc_upper) + self.assertTrue(t.is_empty()) + self.assertFalse(t) + + self.assertEqual(t, edgedb.Range(1, 1, empty=True)) + + with self.assertRaisesRegex(ValueError, 'conflicting arguments'): + edgedb.Range(1, 2, empty=True) + + def test_range_2(self): + t = edgedb.Range(1, 2) + self.assertEqual(repr(t), "") + self.assertEqual(str(t), "") + + self.assertEqual(t.lower, 1) + self.assertEqual(t.upper, 2) + self.assertTrue(t.inc_lower) + self.assertFalse(t.inc_upper) + self.assertFalse(t.is_empty()) + self.assertTrue(t) + + def test_range_3(self): + t = edgedb.Range(1) + self.assertEqual(t.lower, 1) + self.assertEqual(t.upper, None) + self.assertTrue(t.inc_lower) + self.assertFalse(t.inc_upper) + self.assertFalse(t.is_empty()) + + t = edgedb.Range(None, 1) + self.assertEqual(t.lower, None) + self.assertEqual(t.upper, 1) + self.assertFalse(t.inc_lower) + self.assertFalse(t.inc_upper) + self.assertFalse(t.is_empty()) + + t = edgedb.Range(None, None) + self.assertEqual(t.lower, None) + self.assertEqual(t.upper, None) + self.assertFalse(t.inc_lower) + self.assertFalse(t.inc_upper) + self.assertFalse(t.is_empty()) + + def test_range_4(self): + for il in (False, True): + for iu in (False, True): + t = edgedb.Range(1, 2, inc_lower=il, inc_upper=iu) + self.assertEqual(t.lower, 1) + self.assertEqual(t.upper, 2) + self.assertEqual(t.inc_lower, il) + self.assertEqual(t.inc_upper, iu) + self.assertFalse(t.is_empty()) + + def test_range_5(self): + # test hash + self.assertEqual( + { + edgedb.Range(None, 2, inc_upper=True), + edgedb.Range(1, 2), + edgedb.Range(1, 2), + edgedb.Range(1, 2), + edgedb.Range(None, 2, inc_upper=True), + }, + { + edgedb.Range(1, 2), + edgedb.Range(None, 2, inc_upper=True), + } + ) + + +class TestMultiRange(unittest.TestCase): + + def test_multirange_empty_1(self): + t = edgedb.MultiRange() + self.assertEqual(len(t), 0) + self.assertEqual(t, edgedb.MultiRange([])) + + def test_multirange_2(self): + t = edgedb.MultiRange([ + edgedb.Range(1, 2), + edgedb.Range(4), + ]) + self.assertEqual( + repr(t), ", ]>") + self.assertEqual( + str(t), ", ]>") + + self.assertEqual( + t, + edgedb.MultiRange([ + edgedb.Range(1, 2), + edgedb.Range(4), + ]) + ) + + def test_multirange_3(self): + ranges = [ + edgedb.Range(None, 0), + edgedb.Range(1, 2), + edgedb.Range(4), + ] + t = edgedb.MultiRange([ + edgedb.Range(None, 0), + edgedb.Range(1, 2), + edgedb.Range(4), + ]) + + for el, r in zip(t, ranges): + self.assertEqual(el, r) + + def test_multirange_4(self): + # test hash + self.assertEqual( + { + edgedb.MultiRange([ + edgedb.Range(1, 2), + edgedb.Range(4), + ]), + edgedb.MultiRange([edgedb.Range(None, 2, inc_upper=True)]), + edgedb.MultiRange([ + edgedb.Range(1, 2), + edgedb.Range(4), + ]), + edgedb.MultiRange([ + edgedb.Range(1, 2), + edgedb.Range(4), + ]), + edgedb.MultiRange([edgedb.Range(None, 2, inc_upper=True)]), + }, + { + edgedb.MultiRange([edgedb.Range(None, 2, inc_upper=True)]), + edgedb.MultiRange([ + edgedb.Range(1, 2), + edgedb.Range(4), + ]), + } + ) + + def test_multirange_5(self): + # test hash + self.assertEqual( + edgedb.MultiRange([ + edgedb.Range(None, 2, inc_upper=True), + edgedb.Range(5, 9), + edgedb.Range(5, 9), + edgedb.Range(5, 9), + edgedb.Range(None, 2, inc_upper=True), + ]), + edgedb.MultiRange([ + edgedb.Range(5, 9), + edgedb.Range(None, 2, inc_upper=True), + ]), + ) diff --git a/tests/test_async_query.py b/tests/test_async_query.py index 44a00209..7fade678 100644 --- a/tests/test_async_query.py +++ b/tests/test_async_query.py @@ -798,6 +798,83 @@ async def test_async_range_02(self): ) self.assertEqual([edgedb.Range(1, 2)], result) + async def test_async_multirange_01(self): + has_range = await self.client.query( + "select schema::ObjectType filter .name = 'schema::MultiRange'") + if not has_range: + raise unittest.SkipTest( + "server has no support for std::multirange") + + samples = [ + ('multirange', [ + edgedb.MultiRange(), + dict( + input=edgedb.MultiRange([edgedb.Range(empty=True)]), + output=edgedb.MultiRange(), + ), + edgedb.MultiRange([ + edgedb.Range(None, 0), + edgedb.Range(1, 2), + edgedb.Range(4), + ]), + dict( + input=edgedb.MultiRange([ + edgedb.Range(None, 2, inc_upper=True), + edgedb.Range(5, 9), + edgedb.Range(5, 9), + edgedb.Range(5, 9), + edgedb.Range(None, 2, inc_upper=True), + ]), + output=edgedb.MultiRange([ + edgedb.Range(5, 9), + edgedb.Range(None, 3), + ]), + ), + dict( + input=edgedb.MultiRange([ + edgedb.Range(None, 2), + edgedb.Range(-5, 9), + edgedb.Range(13), + ]), + output=edgedb.MultiRange([ + edgedb.Range(None, 9), + edgedb.Range(13), + ]), + ), + ]), + ] + + for typename, sample_data in samples: + for sample in sample_data: + with self.subTest(sample=sample, typname=typename): + stmt = f"SELECT <{typename}>$0" + if isinstance(sample, dict): + inputval = sample['input'] + outputval = sample['output'] + else: + inputval = outputval = sample + + result = await self.client.query_single(stmt, inputval) + err_msg = ( + "unexpected result for {} when passing {!r}: " + "received {!r}, expected {!r}".format( + typename, inputval, result, outputval)) + + self.assertEqual(result, outputval, err_msg) + + async def test_async_multirange_02(self): + has_range = await self.client.query( + "select schema::ObjectType filter .name = 'schema::MultiRange'") + if not has_range: + raise unittest.SkipTest( + "server has no support for std::multirange") + + result = await self.client.query_single( + "SELECT >>$0", + [edgedb.MultiRange([edgedb.Range(1, 2)])] + ) + self.assertEqual([edgedb.MultiRange([edgedb.Range(1, 2)])], result) + async def test_async_wait_cancel_01(self): underscored_lock = await self.client.query_single(""" SELECT EXISTS( From 8a4ed307b2d5245fbd05e96d90532be4f598d245 Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Tue, 8 Aug 2023 20:15:24 -0700 Subject: [PATCH 36/40] edgedb-python 1.6.0 Changes ======= * Implement support for new type descriptor protocol (by @elprans in 47eec190 for #427) * Sync errors (by @elprans in 3bfb574f for #449) * Don't depend on exact syntax errors in tests (by @msullivan in b3ce0c61 for #451) * Fix an error in string representation of RelativeDuration. (by @vpetrovykh in 667da723 for #453) * Add multirange support. (by @vpetrovykh in 15e280ec for #452) --- edgedb/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edgedb/_version.py b/edgedb/_version.py index 400f4395..620a9a28 100644 --- a/edgedb/_version.py +++ b/edgedb/_version.py @@ -28,4 +28,4 @@ # supported platforms, publish the packages on PyPI, merge the PR # to the target branch, create a Git tag pointing to the commit. -__version__ = '1.5.0' +__version__ = '1.6.0' From ac3f1302c1b6d1d3c880529c03d4f45d955d2e92 Mon Sep 17 00:00:00 2001 From: Elvis Pranskevichus Date: Fri, 2 Aug 2024 16:09:07 -0700 Subject: [PATCH 37/40] Remove deprecated `test_suite` kwarg from `setuptools.setup()` --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 27f3aa53..73873448 100644 --- a/setup.py +++ b/setup.py @@ -336,7 +336,6 @@ def finalize_options(self): include_dirs=INCLUDE_DIRS), ], cmdclass={'build_ext': build_ext}, - test_suite='tests.suite', python_requires=">=3.7", install_requires=[ 'typing-extensions>=3.10.0; python_version < "3.8.0"', From 7be5eb31e04ac0134a8b7554ecd9eac7ccd255a2 Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Thu, 5 Oct 2023 21:07:47 -0700 Subject: [PATCH 38/40] Fix test that broke due to error message change (#465) Some error messages were changed in #6209. When I first saw this failing, I was very worried that it was a state bug introduced by --- tests/test_async_query.py | 2 +- tests/test_sync_query.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_async_query.py b/tests/test_async_query.py index 7fade678..796bf87f 100644 --- a/tests/test_async_query.py +++ b/tests/test_async_query.py @@ -1107,7 +1107,7 @@ async def test_dup_link_prop_name(self): ''') async def test_transaction_state(self): - with self.assertRaisesRegex(edgedb.QueryError, "cannot assign to id"): + with self.assertRaisesRegex(edgedb.QueryError, "cannot assign to.*id"): async for tx in self.client.transaction(): async with tx: await tx.execute(''' diff --git a/tests/test_sync_query.py b/tests/test_sync_query.py index 622fceed..79dae829 100644 --- a/tests/test_sync_query.py +++ b/tests/test_sync_query.py @@ -868,7 +868,7 @@ def test_sync_banned_transaction(self): self.client.execute('start transaction') def test_transaction_state(self): - with self.assertRaisesRegex(edgedb.QueryError, "cannot assign to id"): + with self.assertRaisesRegex(edgedb.QueryError, "cannot assign to.*id"): for tx in self.client.transaction(): with tx: tx.execute(''' From 368ed7ace36981e93c3237ceb6ae794ab453a5a3 Mon Sep 17 00:00:00 2001 From: Fantix King Date: Wed, 19 Jun 2024 13:59:59 -0400 Subject: [PATCH 39/40] ci: give repo write permission to the publish job (#505) --- .github/workflows/release.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3e019fb6..bf51d1bf 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -156,6 +156,8 @@ jobs: publish: needs: [build-sdist, build-wheels] runs-on: ubuntu-latest + permissions: + contents: write steps: - uses: actions/checkout@v3 From 1127555b524b2c18684de69176b572925e327bf7 Mon Sep 17 00:00:00 2001 From: Elvis Pranskevichus Date: Fri, 2 Aug 2024 16:10:11 -0700 Subject: [PATCH 40/40] edgedb-python v1.6.1 --- .github/workflows/release.yml | 9 ++++++--- edgedb/_version.py | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bf51d1bf..e9e125ce 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,7 +5,8 @@ on: branches: - "master" - "ci" - - "[0-9]+.[0-9x]+*" + - "release/[0-9]+.x" + - "release/[0-9]+.[0-9]+.x" paths: - "edgedb/_version.py" @@ -80,9 +81,11 @@ jobs: - uses: actions/setup-python@v4 with: python-version: "3.x" - - run: pip install cibuildwheel==2.12.3 + - run: pip install cibuildwheel==2.19.2 - id: set-matrix # Cannot test on Musl distros yet. + env: + CIBW_SKIP: "cp312-*" run: | MATRIX_INCLUDE=$( { @@ -130,7 +133,7 @@ jobs: - name: Install EdgeDB uses: edgedb/setup-edgedb@v1 - - uses: pypa/cibuildwheel@v2.12.3 + - uses: pypa/cibuildwheel@v2.19.2 with: only: ${{ matrix.only }} env: diff --git a/edgedb/_version.py b/edgedb/_version.py index 620a9a28..101d8e88 100644 --- a/edgedb/_version.py +++ b/edgedb/_version.py @@ -28,4 +28,4 @@ # supported platforms, publish the packages on PyPI, merge the PR # to the target branch, create a Git tag pointing to the commit. -__version__ = '1.6.0' +__version__ = '1.6.1'