From c5a4fd966b8179c81fd10549f32d96269e06f50e Mon Sep 17 00:00:00 2001 From: Gianni Tedesco Date: Sat, 19 Sep 2020 17:58:02 +0900 Subject: [PATCH] Initial commit --- .gitignore | 19 + LICENSE.txt | 674 +++++++++++++++++++++++++++++++++++ MANIFEST.in | 3 + Makefile | 14 + README.md | 102 ++++++ benchmark/__init__.py | 0 benchmark/__main__.py | 50 +++ examples/example1.xpdt | 24 ++ include/xpdt/x1b.h | 43 +++ include/xpdt/xbuf.h | 60 ++++ include/xpdt/xbuf_iter.h | 85 +++++ include/xpdt/xfilemap.h | 88 +++++ include/xpdt/xu128.h | 56 +++ pre-commit.sh | 21 ++ setup.cfg | 2 + setup.py | 53 +++ test/__init__.py | 0 test/test_nested.py | 33 ++ test/test_roundtrip.py | 40 +++ xpdt/__init__.py | 41 +++ xpdt/__main__.py | 95 +++++ xpdt/__version__.py | 8 + xpdt/basetypes.py | 43 +++ xpdt/buftype.py | 33 ++ xpdt/c.py | 29 ++ xpdt/decl.py | 16 + xpdt/dupes.py | 14 + xpdt/integraltype.py | 53 +++ xpdt/jinja.py | 73 ++++ xpdt/lex.py | 58 +++ xpdt/member.py | 34 ++ xpdt/namespace.py | 128 +++++++ xpdt/parse.py | 194 ++++++++++ xpdt/py.typed | 0 xpdt/shiftreduce.py | 85 +++++ xpdt/struct.py | 190 ++++++++++ xpdt/templates/__init__.py | 0 xpdt/templates/func_decls.h | 1 + xpdt/templates/x1b.c | 33 ++ xpdt/templates/x1b.pyt | 65 ++++ xpdt/templates/x1b_defns.c | 341 ++++++++++++++++++ xpdt/templates/x1b_defns.pyt | 37 ++ xpdt/templates/x1v_defns.pyt | 69 ++++ xpdt/type.py | 97 +++++ xpdt/typedef.py | 12 + xpdt/uuidtype.py | 27 ++ 46 files changed, 3143 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE.txt create mode 100644 MANIFEST.in create mode 100644 Makefile create mode 100644 README.md create mode 100644 benchmark/__init__.py create mode 100644 benchmark/__main__.py create mode 100644 examples/example1.xpdt create mode 100644 include/xpdt/x1b.h create mode 100644 include/xpdt/xbuf.h create mode 100644 include/xpdt/xbuf_iter.h create mode 100644 include/xpdt/xfilemap.h create mode 100644 include/xpdt/xu128.h create mode 100755 pre-commit.sh create mode 100644 setup.cfg create mode 100755 setup.py create mode 100644 test/__init__.py create mode 100644 test/test_nested.py create mode 100644 test/test_roundtrip.py create mode 100644 xpdt/__init__.py create mode 100644 xpdt/__main__.py create mode 100644 xpdt/__version__.py create mode 100644 xpdt/basetypes.py create mode 100644 xpdt/buftype.py create mode 100644 xpdt/c.py create mode 100644 xpdt/decl.py create mode 100644 xpdt/dupes.py create mode 100644 xpdt/integraltype.py create mode 100644 xpdt/jinja.py create mode 100644 xpdt/lex.py create mode 100644 xpdt/member.py create mode 100644 xpdt/namespace.py create mode 100644 xpdt/parse.py create mode 100644 xpdt/py.typed create mode 100644 xpdt/shiftreduce.py create mode 100644 xpdt/struct.py create mode 100644 xpdt/templates/__init__.py create mode 100644 xpdt/templates/func_decls.h create mode 100644 xpdt/templates/x1b.c create mode 100644 xpdt/templates/x1b.pyt create mode 100644 xpdt/templates/x1b_defns.c create mode 100644 xpdt/templates/x1b_defns.pyt create mode 100644 xpdt/templates/x1v_defns.pyt create mode 100644 xpdt/type.py create mode 100644 xpdt/typedef.py create mode 100644 xpdt/uuidtype.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4b1e92b --- /dev/null +++ b/.gitignore @@ -0,0 +1,19 @@ +# perf +/perf.data +/perf.data.old + +# python +__pycache__ +*.pyc +/.mypy_cache +/build +/dist +/MANIFEST + +# Text editors +.*.swp + +/*.egg-info + +/bench/bin +/bench/obj diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..94a9ed0 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..75c54b9 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,3 @@ +include README.md COPYING +recursive-include examples * +recursive-include xpdt/templates * diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..ada580b --- /dev/null +++ b/Makefile @@ -0,0 +1,14 @@ +MAKEFLAGS += --no-builtin-rules +.SUFFIXES: + +TARGET: all + +.PHONY: all +all: + ./setup.py build + +.PHONY: clean +clean: + ./setup.py clean + rm -rf build dist *.egg-info + find . -regex '^.*\(__pycache__\|\.py[co]\)$$' -delete diff --git a/README.md b/README.md new file mode 100644 index 0000000..0959c77 --- /dev/null +++ b/README.md @@ -0,0 +1,102 @@ +# xpdt: eXPeditious Data Transfer + +
+ PyPI version +
+ +## About +xpdt is (yet another) language for defining data-types and generating code for +serializing and deserializing them. It aims to produce code with little or no +overhead and is based on fixed-length representations which allows for +zero-copy deserialization and (at-most-)one-copy writes (source to buffer). + +The generated C code, in particular, is highly optimized and often permits the +elimination of data-copying for writes and enables optimizations such as +loop-unrolling for fixed-length objects. This can lead to read speeds in +excess of 500 million objects per second (~1.8 nsec per object). + +## Examples +The xpdt source language looks similar to C struct definitions: + +``` +type timestamp { + u32 tv_sec; + u32 tv_nsec; +}; + +type point { + s32 x; + s32 y; + s32 z; +}; + +type line { + timestamp time; + point line_start; + point line_end; + blob comment; +}; +``` + +Fixed width integer types from 8 to 128 bit are supported, along with the +`blob` type, which is a variable-length sequence of bytes. + +## Target Languages +The following target languages are currently supported: +- C +- Python + +The C code is very highly optimized, and the Python code is fairly well +optimized, it uses typed `NamedTuple` for objects and uses `struct.Struct` for +packing/unpacking. Performance of the pure Python code is comparable to a JSON +library implemented in C or Rust. + +For better performance in Python, it may be desirable to develop a Cython +target. + +Target languages are implemented purely as `jinja2` templates. + +## Serialization format +The serialization format for fixed-length objects is simly a packed C struct. + +For any object which contains `blob` type fields: +- a 32bit unsigned record length is prepended to the struct +- all `blob` type fields are converted to `u32` and contain the length of the blob +- all blob contents are appended after the struct in the order in which they appear + +For example, following the example above, the serialization would be: + +``` +u32 tot_len # = 41 +u32 time.tv_sec +u32 time.tv_usec +s32 line_start.x +s32 line_start.y +s32 line_start.z +s32 line_end.x +s32 line_end.y +s32 line_end.z +u32 comment # = 5 +u8 'H' +u8 'e' +u8 'l' +u8 'l' +u8 'o' +``` + +## Features +The feature-set is, as of now, pretty slim. + +There are no array / sequence / map types, and no keyed unions. + +Support for such things may be added in future provided that suitable +implementations exist. An implementation is suitable if: +- It admits a zero (or close to zero) overhead implementation +- it causes no overhead when the feature isn't being used + +# License +The compiler is released under the GPLv3. + +The C support code/headers are released under the MIT license. + +The generated code is yours. diff --git a/benchmark/__init__.py b/benchmark/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/benchmark/__main__.py b/benchmark/__main__.py new file mode 100644 index 0000000..6e3217b --- /dev/null +++ b/benchmark/__main__.py @@ -0,0 +1,50 @@ +from timeit import default_timer as timer +from pathlib import Path + +from xpdt import NameSpace, StructDef, MemberDef, BaseType + + +def main(): + cls = NameSpace( + structs=( + StructDef( + name='SomeStruct', + members=( + MemberDef('a', BaseType.u64), + MemberDef('b', BaseType.u64), + MemberDef('c', BaseType.u64), + MemberDef('d', BaseType.u64), + MemberDef('e', BaseType.u64), + MemberDef('f', BaseType.u64), + MemberDef('g', BaseType.u64), + MemberDef('h', BaseType.u64), + MemberDef('i', BaseType.u64), + MemberDef('j', BaseType.u64), + MemberDef('k', BaseType.u64), + MemberDef('l', BaseType.u64), + MemberDef('m', BaseType.u64), + MemberDef('n', BaseType.u64), + MemberDef('o', BaseType.u64), + MemberDef('p', BaseType.u64), + ), + ), + ), + ).gen_dynamic_python().SomeStruct + + sz = cls._bin_size + + with Path('/dev/zero').open('rb') as f: + iters = 1000000 + start = timer() + for i in range(iters): + buf = f.read(sz) + obj = cls._frombytes(buf) + end = timer() + elapsed = end - start + per_iter = elapsed / iters + print(f'{elapsed:.2f} sec elapsed') + print(f'{per_iter*1e9:.0f} nsec per record') + + +if __name__ == '__main__': + main() diff --git a/examples/example1.xpdt b/examples/example1.xpdt new file mode 100644 index 0000000..455678d --- /dev/null +++ b/examples/example1.xpdt @@ -0,0 +1,24 @@ +## This is a comment + +type timestamp { + u32 tv_sec; + u32 tv_nsec; +}; + +type point { + s32 x; + s32 y; + s32 z; +}; + +type timespan { + timestamp begin; + timestamp end; +}; + +type item { + u32 id; # User id + blob first_name; # given name + blob surname; # surname + timestamp last_login; +}; diff --git a/include/xpdt/x1b.h b/include/xpdt/x1b.h new file mode 100644 index 0000000..8875ebc --- /dev/null +++ b/include/xpdt/x1b.h @@ -0,0 +1,43 @@ +#pragma once + +/* Copyright (c) 2020 Gianni Tedesco + * https://www.scaramanga.co.uk + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. +*/ + +#include +#include +#include +#include + +#define x1b_unlikely(x) __builtin_expect((bool)(x), false) + +#include "xu128.h" +#include "xbuf.h" +#include "xbuf_iter.h" +#include "xfilemap.h" + +typedef uint32_t x1b_size_t; +typedef uint32_t x1b_strlen_t; + +typedef struct xostream *xostream_t; +uint8_t *xostream_prepare(xostream_t, size_t); +bool xostream_commit(xostream_t, size_t); diff --git a/include/xpdt/xbuf.h b/include/xpdt/xbuf.h new file mode 100644 index 0000000..ce2e583 --- /dev/null +++ b/include/xpdt/xbuf.h @@ -0,0 +1,60 @@ +#pragma once + +/* Copyright (c) 2020 Gianni Tedesco + * https://www.scaramanga.co.uk + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. +*/ + +#include + +#include + +struct xbuf { + const uint8_t *ptr; + size_t len; +}; + +#define XBUF_INIT(_len, _ptr) \ + (struct xbuf){.len = _len, .ptr = _ptr} + +#define XBUF_NIL \ + XBUF_INIT(0, NULL) + +static inline struct xbuf xbuf(size_t len, + const uint8_t buf[static len]) +{ + return XBUF_INIT(len, buf); +} + +static inline struct xbuf xbuf_nil(void) +{ + return XBUF_NIL; +} + +static inline struct xbuf xbuf_cstring(const char *str) +{ + return XBUF_INIT(strlen(str), (uint8_t *)str); +} + +static inline bool xbuf_isset(const struct xbuf *str) +{ + return str->len; +} diff --git a/include/xpdt/xbuf_iter.h b/include/xpdt/xbuf_iter.h new file mode 100644 index 0000000..015899f --- /dev/null +++ b/include/xpdt/xbuf_iter.h @@ -0,0 +1,85 @@ +#pragma once + +/* Copyright (c) 2020 Gianni Tedesco + * https://www.scaramanga.co.uk + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. +*/ + +#include + +#include +#include + +struct xbuf_iter { + const uint8_t *it_ptr; + const uint8_t *it_end; +}; + +#define XBUF_ITER_INIT(_ptr, _end) \ + (struct xbuf_iter){.it_ptr = _ptr, .it_end = _end} + +#define XBUF_ITER_NIL \ + XBUF_ITER_INIT(NULL, NULL) + +static inline struct xbuf_iter xbuf_iter_new(const size_t len, + const uint8_t buf[static len]) +{ + return XBUF_ITER_INIT(buf, buf + len); +} + +static inline struct xbuf_iter xbuf_iter_nil(void) +{ + return XBUF_ITER_NIL; +} + +static inline const uint8_t *xbuf_iter_read(struct xbuf_iter *it, const size_t len) +{ + const uint8_t *ret = it->it_ptr; + const uint8_t *end = it->it_ptr + len; + + if (__builtin_expect(end > it->it_end, false)) + return NULL; + + it->it_ptr = end; + return ret; +} + +static inline size_t xbuf_iter_count_items(const struct xbuf_iter it, const size_t len) +{ + size_t sz; + + if (it.it_ptr >= it.it_end) + return 0; + + sz = it.it_end - it.it_ptr; + return sz / len; +} + +static inline size_t xbuf_iter_bytes_remaining(const struct xbuf_iter it) +{ + ptrdiff_t ret; + + ret = it.it_end - it.it_ptr; + if (ret <= 0) + return 0; + + return ret; +} diff --git a/include/xpdt/xfilemap.h b/include/xpdt/xfilemap.h new file mode 100644 index 0000000..d6aa2ba --- /dev/null +++ b/include/xpdt/xfilemap.h @@ -0,0 +1,88 @@ +#pragma once + +/* Copyright (c) 2020 Gianni Tedesco + * https://www.scaramanga.co.uk + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. +*/ + +#include + +#include +#include +#include +#include + +#include + +struct xfilemap { + size_t m_len; + const uint8_t *m_map; +}; + +#define MAPFILE_INIT(_len, _map) \ + (struct xfilemap){.m_len = _len, .m_map = _map} + +#define MAPFILE_NIL \ + MAPFILE_INIT(0, NULL) + +static inline struct xfilemap xfilemap_new(size_t len, + const uint8_t map[static len]) +{ + return MAPFILE_INIT(len, map); +} + +static inline size_t xfilemap_count_items(const struct xfilemap *xf, size_t len) +{ + return xf->m_len / len; +} + +bool xfilemap_open(struct xfilemap *xf, const char *fn); +void xfilemap_close(struct xfilemap *xf); + +static inline struct xbuf_iter xfilemap_iter(const struct xfilemap *xf) +{ + return xbuf_iter_new(xf->m_len, xf->m_map); +} + +static inline struct xbuf_iter xfilemap_iter_slice(const struct xfilemap *xf, + size_t from, + size_t len) +{ + assert(xf->m_len <= (from + len)); + return xbuf_iter_new(len, xf->m_map + from); +} + +static inline struct xbuf_iter xfilemap_iter_items(const struct xfilemap *xf, + size_t item_size) +{ + size_t trailer = xf->m_len % item_size; + return xbuf_iter_new(xf->m_len - trailer, xf->m_map); +} + +static inline struct xbuf_iter xfilemap_iter_unrolled(const struct xfilemap *xf, + unsigned int burst_size, + size_t item_size, + unsigned int *trailer) +{ + size_t nr_items = xf->m_len / item_size; + *trailer = nr_items % burst_size; + return xfilemap_iter_items(xf, burst_size * item_size); +} diff --git a/include/xpdt/xu128.h b/include/xpdt/xu128.h new file mode 100644 index 0000000..ef76b29 --- /dev/null +++ b/include/xpdt/xu128.h @@ -0,0 +1,56 @@ +#pragma once + +/* Copyright (c) 2020 Gianni Tedesco + * https://www.scaramanga.co.uk + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. +*/ + +#include + +#include + +struct _xu128_s { + union { +#ifdef __SIZEOF_INT128__ + unsigned __int128 u128; +#endif + struct { + uint64_t lo; + uint64_t hi; + }; + uint8_t bytes[16]; + }; +} __attribute__((packed)); + +typedef struct _xu128_s xu128_t; + +#define XU128_INIT(_hi, _lo) \ + (struct _xu128_s){ \ + .lo = _lo, \ + .hi = _hi, \ + } + +#define XU128_NIL XU128_INIT(0, 0) + +static inline xu128_t xu128(uint64_t hi, uint64_t lo) +{ + return XU128_INIT(hi, lo); +} diff --git a/pre-commit.sh b/pre-commit.sh new file mode 100755 index 0000000..79401fe --- /dev/null +++ b/pre-commit.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +set -euo pipefail + +exec 1>&2 + +pkgname='xpdt' +scripts= + +mypy \ + --strict \ + ${pkgname} ${scripts} + +flake8 \ + ${pkgname} ${scripts} + + +pycodestyle-3 \ + ${pkgname} + +python -m unittest diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..b88034e --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[metadata] +description-file = README.md diff --git a/setup.py b/setup.py new file mode 100755 index 0000000..1f57bd5 --- /dev/null +++ b/setup.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 + +from typing import Dict, Any +from pathlib import Path +import setuptools + +version_file = Path('xpdt/__version__.py') +v: Dict[str, Any] = {} +exec(version_file.read_text(), v) + +setuptools.setup( + name=v['__title__'], + version=v['__version__'], + description=v['__description__'], + author=v['__author__'], + author_email=v['__author_email__'], + package_data={ + 'xpdt': ['py.typed'], + }, + install_requires=[ + 'jinja2', + ], + long_description=Path('README.md').read_text(), + long_description_content_type="text/markdown", + include_package_data=True, + license=v['__license__'], + platforms='any', + packages=[ + 'xpdt', + ], + url=v['__url__'], + classifiers=[ + 'Development Status :: 4 - Beta', + 'Intended Audience :: Developers', + 'Natural Language :: English', + 'Operating System :: POSIX', + 'Operating System :: POSIX :: Linux', + 'Operating System :: OS Independent', + 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: Implementation :: CPython', + ], + entry_points={ + 'console_scripts': [ + 'xpdt = xpdt.__main__:main', + ], + }, +) diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/test_nested.py b/test/test_nested.py new file mode 100644 index 0000000..eb82301 --- /dev/null +++ b/test/test_nested.py @@ -0,0 +1,33 @@ +import unittest + +from xpdt import NameSpace, StructDecl, MemberDecl, BaseType + +class Test_Nested(unittest.TestCase): + def setUp(self): + self.code = NameSpace.from_decls( + ( + StructDecl( + struct_name='Point', + members=[ + MemberDecl('x', 's32'), + MemberDecl('y', 's32'), + MemberDecl('z', 's32'), + ], + ), + StructDecl( + struct_name='Entity', + members=[ + MemberDecl('id', 'u32'), + MemberDecl('pos', 'Point'), + ], + ), + ), + name=None, + ).gen_dynamic_python() + + def test_point(self): + point = self.code.Point(1, 2, 3) + orig = self.code.Entity(0xdeadbeef, point) + b = bytes(orig) + clone = orig._frombytes(b) + self.assertEqual(orig, clone) diff --git a/test/test_roundtrip.py b/test/test_roundtrip.py new file mode 100644 index 0000000..b14ac21 --- /dev/null +++ b/test/test_roundtrip.py @@ -0,0 +1,40 @@ +import unittest + +from xpdt import NameSpace, StructDef, MemberDef, BaseType + +class Test_RoundTrips(unittest.TestCase): + def setUp(self): + self.code = NameSpace( + structs=( + StructDef( + name='Point', + members=( + MemberDef('x', BaseType.s32), + MemberDef('y', BaseType.s32), + MemberDef('z', BaseType.s32), + ), + ), + StructDef( + name='Item', + members=( + MemberDef('id', BaseType.u32), + MemberDef('first_name', BaseType.blob), + MemberDef('surname', BaseType.blob), + ), + ), + ), + name=None, + ).gen_dynamic_python() + + def test_point(self): + orig = self.code.Point(1, -2, 3) + b = bytes(orig) + clone = orig._frombytes(b) + self.assertEqual(orig, clone) + + + def test_item(self): + orig = self.code.Item(0xdeadbeef, b'Malkovich', b'') + b = bytes(orig) + clone = orig._frombytes(b) + self.assertEqual(orig, clone) diff --git a/xpdt/__init__.py b/xpdt/__init__.py new file mode 100644 index 0000000..51dfef1 --- /dev/null +++ b/xpdt/__init__.py @@ -0,0 +1,41 @@ +""" +xpdt: eXPeditious Data Transfer +""" + +from .__version__ import \ + __title__, \ + __description__, \ + __url__, \ + __author__, \ + __author_email__, \ + __copyright__, \ + __license__, \ + __version__ # noqa: F401 + +from .type import XpdtType +from .typedef import TypeDef +from .basetypes import BaseType, base_types +from .member import MemberDef +from .struct import StructDef +from .decl import MemberDecl, StructDecl +from .namespace import NameSpace +from .parse import parse, parse_file + +__all__ = ( + 'XpdtType', + + 'TypeDef', + 'BaseType', + 'base_types', + + 'MemberDecl', + 'StructDecl', + + 'MemberDef', + 'StructDef', + + 'NameSpace', + + 'parse', + 'parse_file', +) diff --git a/xpdt/__main__.py b/xpdt/__main__.py new file mode 100644 index 0000000..a922f5f --- /dev/null +++ b/xpdt/__main__.py @@ -0,0 +1,95 @@ +from argparse import ArgumentParser +from pathlib import Path +from sys import stdout +from itertools import chain +import logging + +from xpdt import NameSpace +from .parse import parse_file +from .shiftreduce import ParseError + +_fmt = logging.Formatter('%(message)s') +_stdio_handler = logging.StreamHandler(stream=stdout) +_stdio_handler.setFormatter(_fmt) +_log = logging.getLogger('xpdt') +_log.addHandler(_stdio_handler) + + +def gen_c(ns: NameSpace, base: Path, module_name: str) -> None: + p = base / f'{module_name}.h' + _log.info('Writing code: %s', p) + with p.open('w') as f: + ns.gen_c(f) + + +def gen_py(ns: NameSpace, base: Path, module_name: str) -> None: + p = base / f'{module_name}.py' + _log.info('Writing code: %s', p) + with p.open('w') as f: + ns.gen_python(f) + + +def main() -> None: + opts = ArgumentParser(description='xpdt: eXPedited Data Transfer') + opts.add_argument('--verbose', '-v', + action='count', + default=0, + help='Be more talkative') + opts.add_argument('--module-name', '-n', + default=None, + type=str, + help='Name of module (output file)') + opts.add_argument('--language', '-l', + default='c', + type=str, + help='Output language') + opts.add_argument('--out', '-o', + default=Path(), + type=Path, + help='Output dir') + opts.add_argument('file', + type=Path, + nargs='+', + help='xpdt schema files') + + args = opts.parse_args() + if args.verbose: + _log.setLevel(logging.DEBUG) + else: + _log.setLevel(logging.INFO) + + if args.module_name is None: + if len(args.file) == 1: + in_path, = args.file + module_name = in_path.stem + else: + module_name = 'xpdt' + else: + module_name = args.module_name + + try: + decls = tuple(chain(*map(parse_file, args.file))) + except ParseError as e: + print(e) + raise SystemExit(1) + + ns = NameSpace.from_decls(decls, name=None) + + backends = { + 'c': gen_c, + 'py': gen_py, + 'python': gen_py, + } + + try: + fn = backends[args.language] + except KeyError: + print(f'Unknown language: "{args.language}"') + raise SystemExit(1) + + args.out.mkdir(exist_ok=True) + fn(ns, args.out, module_name) + + +if __name__ == '__main__': + main() diff --git a/xpdt/__version__.py b/xpdt/__version__.py new file mode 100644 index 0000000..2e3b660 --- /dev/null +++ b/xpdt/__version__.py @@ -0,0 +1,8 @@ +__title__ = 'xpdt' +__description__ = 'eXPeditious Data Transfer' +__url__ = 'https://github.com/giannitedesco/xpdt' +__author__ = 'Gianni Tedesco' +__author_email__ = 'gianni@scaramanga.co.uk' +__copyright__ = 'Copyright 2020 Gianni Tedesco' +__license__ = 'GPLv3' +__version__ = '0.0.1' diff --git a/xpdt/basetypes.py b/xpdt/basetypes.py new file mode 100644 index 0000000..d4155bc --- /dev/null +++ b/xpdt/basetypes.py @@ -0,0 +1,43 @@ +from typing import Dict +from enum import Enum + +from .integraltype import IntegralType +from .buftype import BufType +from .uuidtype import UuidType +from .typedef import TypeDef + +__all__ = ( + 'BaseType', + 'base_types', +) + +Blob = BufType() +xu128 = UuidType() +UnsignedInt8 = IntegralType(8, False) +SignedInt8 = IntegralType(8, True) +UnsignedInt16 = IntegralType(16, False) +SignedInt16 = IntegralType(16, True) +UnsignedInt32 = IntegralType(32, False) +SignedInt32 = IntegralType(32, True) +UnsignedInt64 = IntegralType(64, False) +SignedInt64 = IntegralType(64, True) + + +class BaseType(TypeDef, Enum): + __slots__ = () + + blob = 'blob', Blob + u8 = 'u8', UnsignedInt8 + s8 = 's8', SignedInt8 + u16 = 'u16', UnsignedInt16 + s16 = 's16', SignedInt16 + u32 = 'u32', UnsignedInt32 + s32 = 's32', SignedInt32 + u64 = 'u64', UnsignedInt64 + s64 = 's64', SignedInt64 + u128 = 'u128', xu128 + uuid = 'uuid', xu128 + + +def base_types() -> Dict[str, TypeDef]: + return {t.name: t.value for t in BaseType} diff --git a/xpdt/buftype.py b/xpdt/buftype.py new file mode 100644 index 0000000..0072088 --- /dev/null +++ b/xpdt/buftype.py @@ -0,0 +1,33 @@ +from .type import XpdtType +from .integraltype import IntegralType + +__all__ = ( + 'BufType', +) + + +class BufType(XpdtType): + __slots__ = ( + '_off_type', + ) + + pytype = 'bytes' + + def __init__(self) -> None: + self._off_type = IntegralType(32, False) + + @property + def struct_fmt(self) -> str: + return self._off_type.struct_fmt + + @property + def ctype(self) -> str: + return 'struct xbuf' + + @property + def size(self) -> int: + return self._off_type.size + + @property + def needs_vbuf(self) -> bool: + return True diff --git a/xpdt/c.py b/xpdt/c.py new file mode 100644 index 0000000..b7f62a6 --- /dev/null +++ b/xpdt/c.py @@ -0,0 +1,29 @@ +from pathlib import PurePosixPath + +__all__ = ( + 'Include', + 'SysInclude', + 'RelInclude', +) + + +class Include(PurePosixPath): + @property + def path_spec(self) -> str: + raise NotImplementedError + + @property + def include(self) -> str: + return f'#include {self.path_spec}' + + +class SysInclude(Include): + @property + def path_spec(self) -> str: + return f'<{self}>' + + +class RelInclude(Include): + @property + def path_spec(self) -> str: + return f'"{self}"' diff --git a/xpdt/decl.py b/xpdt/decl.py new file mode 100644 index 0000000..c295e3b --- /dev/null +++ b/xpdt/decl.py @@ -0,0 +1,16 @@ +from typing import NamedTuple, List + +__all__ = ( + 'MemberDecl', + 'StructDecl', +) + + +class MemberDecl(NamedTuple): + member_name: str + type_name: str + + +class StructDecl(NamedTuple): + struct_name: str + members: List[MemberDecl] diff --git a/xpdt/dupes.py b/xpdt/dupes.py new file mode 100644 index 0000000..5a1b9c2 --- /dev/null +++ b/xpdt/dupes.py @@ -0,0 +1,14 @@ +from typing import Iterable, TypeVar, List +from collections import Counter + + +__all__ = ( + 'dupes', +) + + +_T = TypeVar('_T') + + +def dupes(s: Iterable[_T]) -> List[_T]: + return [x for x, y in Counter(s).items() if y > 1] diff --git a/xpdt/integraltype.py b/xpdt/integraltype.py new file mode 100644 index 0000000..9a81100 --- /dev/null +++ b/xpdt/integraltype.py @@ -0,0 +1,53 @@ +from .type import XpdtType + +__all__ = ( + 'IntegralType', +) + + +class IntegralType(XpdtType): + __slots__ = ( + '_width', + '_signed', + '_fmt', + '_size', + '_ctype', + ) + + pytype = 'int' + + _struct_map = { + 8: 'b', + 16: 'h', + 32: 'i', + 64: 'q', + } + + def __init__(self, width: int, signed: bool): + self._width = width + self._signed = signed + fmt = self._struct_map[self._width] + if signed: + self._ctype = f'int{width}_t' + fmt = fmt.lower() + else: + self._ctype = f'uint{width}_t' + fmt = fmt.upper() + self._fmt = fmt + self._size = super().size + + @property + def ctype(self) -> str: + return self._ctype + + @property + def struct_fmt(self) -> str: + return self._fmt + + @property + def size(self) -> int: + return self._size + + @property + def needs_vbuf(self) -> bool: + return False diff --git a/xpdt/jinja.py b/xpdt/jinja.py new file mode 100644 index 0000000..6897515 --- /dev/null +++ b/xpdt/jinja.py @@ -0,0 +1,73 @@ +from jinja2 import Environment, PackageLoader, StrictUndefined +import builtins + +__all__ = ( + 'ctemplate', + 'pytemplate', +) + + +def _prefix(s: str, pfx: str) -> str: + return pfx + s + + +def _suffix(s: str, sfx: str) -> str: + return s + sfx + + +def _wrap(s: str, pfx: str, sfx: str) -> str: + return pfx + s + sfx + + +def _macro(s: str) -> str: + return s.replace('\n', ' \\\n') + + +def _setup(env: Environment) -> None: + env.undefined = StrictUndefined + env.filters.update({f: getattr(builtins, f) for f in ( + 'len', + 'range', + 'zip', + 'hex', + 'enumerate', + 'sorted', + )}) + env.filters.update({ + 'prefix': _prefix, + 'suffix': _suffix, + 'wrap': _wrap, + 'macro': _macro, + }) + + +ctemplate = Environment( + loader=PackageLoader('xpdt', 'templates'), + line_statement_prefix='//', + variable_start_string='/*{', + variable_end_string='}*/', + block_start_string='/*#', + block_end_string='#*/', + keep_trailing_newline=False, + trim_blocks=True, + lstrip_blocks=True, + auto_reload=False, + autoescape=False, +) +_setup(ctemplate) + + +pytemplate = Environment( + loader=PackageLoader('xpdt', 'templates'), + block_start_string='#%', + block_end_string='%#', + line_statement_prefix='##', + variable_start_string='$$', + variable_end_string='$$', + keep_trailing_newline=False, + trim_blocks=True, + lstrip_blocks=True, + auto_reload=False, + autoescape=False, +) +_setup(pytemplate) diff --git a/xpdt/lex.py b/xpdt/lex.py new file mode 100644 index 0000000..6c98139 --- /dev/null +++ b/xpdt/lex.py @@ -0,0 +1,58 @@ +from typing import NamedTuple, Generator, FrozenSet, Pattern +import re + + +__all__ = ( + 'lex', + 'Lexeme', +) + + +_re = re.compile('|'.join(( + r'(?Ptype)', + r'(?P[a-zA-Z_][a-zA-Z0-9_]*)', + r'(?P[0-9][0-9]*)', + r'(?P\{)', + r'(?P\})', + r'(?P;)', + r'(?P#.*?(?=\n))', # up to but not including newline + r'(?P[^\S\n][^\S\n]*)', # non-newline whitespace + r'(?P\n)', +))) +_ignore_toks = frozenset({'whitespace', 'comment', 'newline'}) + + +class Lexeme(NamedTuple): + tok_type: str + val: str + file: str + line: int + + +def lex(s: str, + file: str = '', + line: int = 1, + eof: bool = True, + _ignored: FrozenSet[str] = _ignore_toks, + _dfa: Pattern[str] = _re, + ) -> Generator[Lexeme, None, None]: + pos = 0 + + while True: + m = _dfa.match(s, pos) + if not m: + break + grp = m.lastgroup + begin, end = m.span() + pos = end + if grp not in _ignored: + assert(grp is not None) + yield Lexeme(grp, s[begin:end], file, line) + elif grp == 'newline': + line += 1 + + if pos < len(s): + raise ValueError + + if eof: + yield Lexeme('eof', '$', file, line) diff --git a/xpdt/member.py b/xpdt/member.py new file mode 100644 index 0000000..d126c33 --- /dev/null +++ b/xpdt/member.py @@ -0,0 +1,34 @@ +from typing import NamedTuple + +from .type import XpdtType +from .typedef import TypeDef + + +__all__ = ( + 'MemberDef', +) + + +class MemberDef(NamedTuple): + name: str + typedef: TypeDef + + @property + def needs_vbuf(self) -> bool: + return self.typedef.type.needs_vbuf + + @property + def cdecl(self) -> str: + return f'{self.typedef.type.ctype} {self.name}' + + @property + def const_cdecl(self) -> str: + return f'const {self.cdecl}' + + @property + def type(self) -> XpdtType: + return self.typedef.type + + @property + def is_scalar(self) -> bool: + return self.typedef.type.is_scalar diff --git a/xpdt/namespace.py b/xpdt/namespace.py new file mode 100644 index 0000000..9574ed9 --- /dev/null +++ b/xpdt/namespace.py @@ -0,0 +1,128 @@ +from typing import ( + Optional, Tuple, Dict, NamedTuple, Type, TextIO, Iterable, Iterator, Any +) +from types import SimpleNamespace + +from .dupes import dupes +from .decl import StructDecl, MemberDecl +from .typedef import TypeDef +from .basetypes import base_types +from .member import MemberDef +from .struct import StructDef, StructType +from .jinja import ctemplate, pytemplate +from .c import SysInclude + +__all__ = ( + 'NameSpace', +) + + +class NameSpace: + __slots__ = ( + '_name', + '_structs', + ) + + _name: Optional[str] + _structs: Dict[str, StructDef] + + x1b_c = ctemplate.get_template('x1b.c') + x1b_python = pytemplate.get_template('x1b.pyt') + + def __init__(self, + structs: Tuple[StructDef, ...], + name: Optional[str] = None): + + d = dupes((s.name for s in structs)) + if d: + raise ValueError(f'{name}: Duplicated struct name: ' + ", ".join(d)) + + self._name = name + + # TODO: Allow sub-structs + self._structs = {s.name: s for s in structs} + + @property + def name(self) -> Optional[str]: + return self._name + + def __iter__(self) -> Iterator[StructDef]: + yield from self._structs.values() + + def __len__(self) -> int: + return len(self._structs) + + def _gen_dynamic_clsname(self, s: StructDef) -> str: + if self._name: + return f'{self._name}.{s.name}' + else: + return s.name + + def _gen_storage_class(self, s: StructDef) -> Type[NamedTuple]: + sig = [(m.name, m.typedef.type.pytype) for m in s] + + # This class is a tuple for storing the data in each instance + # mypy insists on literals, but sorry, no can do... + return NamedTuple(s.name, sig) # type: ignore + + @property + def python_code(self) -> str: + return self.x1b_python.render(namespace=self) + + def gen_dynamic_python(self) -> SimpleNamespace: + names = {s.name for s in self} + g: Dict[str, Any] = {} + exec(self.python_code, g, g) + return SimpleNamespace(**{x: g[x] for x in names}) + + def gen_python(self, f: TextIO) -> None: + f.write(self.python_code) + + def gen_c(self, f: TextIO) -> None: + x1b_hdr = SysInclude('xpdt/x1b.h') + headers = ( + SysInclude('stdint.h'), + SysInclude('stdbool.h'), + SysInclude('string.h'), + ) + + code = self.x1b_c.render( + headers=headers, + namespace=self, + x1b_hdr=x1b_hdr, + ) + f.write(code) + + @classmethod + def from_decls(cls, + decls: Iterable[StructDecl], + name: Optional[str] = None) -> 'NameSpace': + + typemap = base_types() + + def define_member(decl: MemberDecl) -> MemberDef: + type_name = decl.type_name + try: + member_type = typemap[type_name] + except KeyError: + raise KeyError(f'Type "{type_name}" not known') + return MemberDef( + decl.member_name, + member_type, + ) + + def define_struct(decl: StructDecl) -> StructDef: + name = decl.struct_name + struct = StructDef( + name, + tuple((define_member(m) for m in decl.members)), + ) + if name in typemap: + raise ValueError(f'Type {name} already defined') + typemap[name] = TypeDef(name, StructType(struct)) + return struct + + defns = tuple((define_struct(decl) for decl in decls)) + + return cls(defns, name) diff --git a/xpdt/parse.py b/xpdt/parse.py new file mode 100644 index 0000000..e65c06a --- /dev/null +++ b/xpdt/parse.py @@ -0,0 +1,194 @@ +from __future__ import annotations +from typing import NamedTuple, Dict, Generator, List, Optional, Union +from pathlib import Path + +from .decl import StructDecl, MemberDecl +from .shiftreduce import state, State, ParseError +from .lex import lex, Lexeme + +__all__ = ( + 'parse', + 'parse_file', +) + + +class MemberTypeItem(NamedTuple): + member_type: str + + +class MemberNameItem(NamedTuple): + member_name: str + + +class NamespaceItem(NamedTuple): + ns: Dict[str, StructDecl] + + +Item = Union[ + MemberTypeItem, + MemberNameItem, + NamespaceItem, + StructDecl, +] + + +class Parser: + __slots__ = ( + '_tokens', + '_stack', + ) + + _stack: List[Item] + + def __init__(self, toks: Generator[Lexeme, None, None]): + self._tokens = toks + self._stack = list() + + def push(self, item: Item) -> None: + self._stack.append(item) + + def pop(self) -> Item: + return self._stack.pop() + + def stacktop(self) -> Item: + return self._stack[-1] + + @state + def _initial(self) -> None: + pass + + @state + def _type_name(self) -> None: + pass + + @state + def _type_start(self) -> None: + pass + + @state + def _member_type(self) -> None: + pass + + @state + def _member_name(self) -> None: + pass + + @state + def _member_end(self) -> None: + pass + + @_initial.on('semicolon') + def _initial_semi(self, tok: Lexeme) -> Optional[State[Parser]]: + # reduce + return self._initial + + @_initial.on('eof') + def _initial_eof(self, tok: Lexeme) -> Optional[State[Parser]]: + # accept + return None + + @_initial.on('kw_type') + def _initial_type_keyword(self, tok: Lexeme) -> Optional[State[Parser]]: + # shift + return self._type_name + + @_type_start.on('lbrace') + def _type_start_lbrace(self, tok: Lexeme) -> Optional[State[Parser]]: + # shift + return self._member_type + + @_type_name.on('identifier') + def _type_name_identifier(self, tok: Lexeme) -> Optional[State[Parser]]: + # shift + self.push(StructDecl(tok.val, list())) + return self._type_start + + @_member_type.on('identifier') + def _member_type_identifier(self, tok: Lexeme) -> Optional[State[Parser]]: + # shift + self.push(MemberTypeItem(tok.val)) + return self._member_name + + @_member_name.on('identifier') + def _member_name_identifier(self, tok: Lexeme) -> Optional[State[Parser]]: + # shift + self.push(MemberNameItem(tok.val)) + return self._member_end + + @_member_end.on('semicolon') + def _member_end_semicolon(self, tok: Lexeme) -> Optional[State[Parser]]: + # reduce MemberDecl + name_item = self.pop() + type_item = self.pop() + + assert isinstance(name_item, MemberNameItem) + assert isinstance(type_item, MemberTypeItem) + + member_name = name_item.member_name + member_type = type_item.member_type + + t = self.stacktop() + assert isinstance(t, StructDecl) + + t.members.append(MemberDecl(member_name, member_type)) + return self._member_type + + @_member_type.on('rbrace') + def _member_type_rbrace(self, tok: Lexeme) -> Optional[State[Parser]]: + # reduce StructDef + t = self.pop() + ns = self.stacktop() + + assert isinstance(t, StructDecl) + assert isinstance(ns, NamespaceItem) + + ns.ns[t.struct_name] = t + return self._initial + + def __iter__(self) -> Generator[StructDecl, None, None]: + self._stack.clear() + self.push(NamespaceItem({})) + + gen = self._tokens + st: Optional[State[Parser]] = self._initial + + while st is not None: + try: + tok = next(gen) + except StopIteration: + raise ParseError('Unexpected end of token-stream') + + st = st(tok) + + ns_item = self.pop() + assert isinstance(ns_item, NamespaceItem) + + yield from ns_item.ns.values() + + @classmethod + def fromstring(cls, s: str, file: str = '') -> Parser: + return cls(lex(s, file=file)) + + @classmethod + def fromfile(cls, p: Path) -> Parser: + """ + Loads the file, line at a time. + """ + + def gen() -> Generator[Lexeme, None, None]: + file = str(p) + with p.open() as f: + lno = 1 + for line in f: + yield from lex(line, file, lno, False) + lno += 1 + yield Lexeme('eof', '$', file, lno) + return cls(gen()) + + +def parse(s: str) -> Generator[StructDecl, None, None]: + yield from Parser.fromstring(s) + + +def parse_file(p: Path) -> Generator[StructDecl, None, None]: + yield from Parser.fromfile(p) diff --git a/xpdt/py.typed b/xpdt/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/xpdt/shiftreduce.py b/xpdt/shiftreduce.py new file mode 100644 index 0000000..a75b0fc --- /dev/null +++ b/xpdt/shiftreduce.py @@ -0,0 +1,85 @@ +from __future__ import annotations +from typing import Dict, Optional, Callable, Any, TypeVar, Generic, cast +from typing_extensions import Protocol +from functools import wraps + +""" +Helpers for shift/reduce parsing +""" + +__all__ = ( + 'ParseError', + 'state', +) + + +class Token(Protocol): + @property + def tok_type(self) -> Any: ... + + @property + def val(self) -> str: ... + + @property + def file(self) -> str: ... + + @property + def line(self) -> int: ... + + +class ParseError(Exception): + tok: Optional[Token] + + def __init__(self, + msg: str, + /, + tok: Optional[Token] = None): + super().__init__(msg) + self.tok = tok + + def __str__(self) -> str: + s = super().__str__() + tok = self.tok + if tok is None: + return s + return f'{tok.file}:{tok.line}: at "{tok.val}": {s}' + + +_T = TypeVar('_T') + + +class State(Generic[_T]): + def on(self: _T, + *args: str, + ) -> Callable[..., Callable[[_T, Token], Optional[State[_T]]]]: ... + + def __call__(self: _T, tok: Token) -> Optional[State[_T]]: ... + + +def state(func: Callable[[_T], None]) -> State[_T]: + registry: Dict[Any, Callable[[_T, Token], Optional[State[_T]]]] = {} + + def on(*args: str, + ) -> Callable[..., Callable[[_T, Token], Optional[State[_T]]]]: + def decorator(func: Callable[[_T, Token], Optional[State[_T]]], + ) -> Callable[[_T, Token], Optional[State[_T]]]: + for tok_type in args: + registry[tok_type] = func + return func + return decorator + + @wraps(func) + def wrapper(self: _T, tok: Token) -> Optional[State[_T]]: + tok_type = tok.tok_type + try: + f = registry[tok_type] + except KeyError: + expected = ', '.join(sorted(registry.keys())) + raise ParseError( + f'{tok_type} unexpected, looking for {expected}', + tok=tok, + ) + return f(self, tok) + + wrapper.on = on # type: ignore + return cast(State[_T], wrapper) diff --git a/xpdt/struct.py b/xpdt/struct.py new file mode 100644 index 0000000..e2a4f81 --- /dev/null +++ b/xpdt/struct.py @@ -0,0 +1,190 @@ +from typing import Tuple, Generator, Iterator + +from .dupes import dupes + +from .type import XpdtType, ConstructAction, ConstructElement +from .member import MemberDef + + +__all__ = ( + 'StructDef', + 'StructType', +) + + +class StructDef: + __slots__ = ( + '_name', + '_membs', + '_vmembs', + ) + + def __init__(self, name: str, members: Tuple[MemberDef, ...]): + d = dupes((m.name for m in members)) + if d: + raise ValueError(f'{name}: Duplicated member name: ' + ", ".join(d)) + self._name = name + self._membs = {m.name: m for m in members} + self._vmembs = tuple((m for m in members + if m.needs_vbuf)) + + @property + def name(self) -> str: + return self._name + + @property + def ctype(self) -> str: + return f'struct {self._name}' + + @property + def tag(self) -> str: + return self._name + + @property + def member_types(self) -> Generator[XpdtType, None, None]: + yield from (m.typedef.type for m in self) + + @property + def vbuf_members(self) -> Tuple[MemberDef, ...]: + return self._vmembs + + @property + def needs_vbuf(self) -> bool: + return any((t.needs_vbuf for t in self.member_types)) + + @property + def all_members_scalar(self) -> bool: + return all((t.is_scalar for t in self.member_types)) + + @property + def all_members(self) -> Generator[ConstructElement, None, None]: + return (elem for elem in + self.construct_recursive() + if elem.action == ConstructAction.MEMBER) + + @property + def all_vbuf_members(self) -> Generator[ConstructElement, None, None]: + return (elem for elem in + self.construct_recursive() + if elem.action == ConstructAction.MEMBER + and elem.member.type.needs_vbuf) + + @property + def scalar_members(self) -> Generator[Tuple[str, ...], None, None]: + return (elem.full_path_names for elem in + self.construct_recursive() + if elem.action == ConstructAction.MEMBER) + + @property + def python_var_names(self) -> Generator[str, None, None]: + return (elem.python_var_name for elem in + self.construct_recursive() + if elem.action == ConstructAction.MEMBER) + + @property + def c_named_initializers(self) -> Generator[str, None, None]: + return (elem.c_named_initializer for elem in + self.construct_recursive() + if elem.action == ConstructAction.MEMBER) + + @property + def c_named_initializers_x1v(self) -> Generator[str, None, None]: + return (elem.c_named_initializer for elem in + self.construct_recursive() + if elem.action == ConstructAction.MEMBER + and elem.member.type.needs_vbuf) + + @property + def python_vbuf_names(self) -> Generator[str, None, None]: + return (elem.python_var_name for elem in + self.construct_recursive() + if elem.action == ConstructAction.MEMBER + and elem.member.type.needs_vbuf) + + def construct_recursive(self, + path: Tuple[MemberDef, ...] = (), + ) -> Generator[ConstructElement, None, None]: + for m in self: + if m.is_scalar: + yield ConstructElement(ConstructAction.MEMBER, m, path) + else: + stype = m.type + new_path = path + (m,) + yield ConstructElement(ConstructAction.AGG_PUSH, m, path) + yield from stype.construct_recursive(new_path) + yield ConstructElement(ConstructAction.AGG_POP, m, path) + + @property + def struct_fmt(self) -> str: + return ''.join((t.struct_fmt for t in self.member_types)) + + def __getitem__(self, k: str) -> MemberDef: + return self._membs[k] + + def __iter__(self) -> Iterator[MemberDef]: + return iter(self._membs.values()) + + def __len__(self) -> int: + return len(self._membs) + + +class StructType(XpdtType): + __slots__ = ( + '_struct', + ) + + _struct: StructDef + + def __init__(self, struct: StructDef): + self._struct = struct + + @property + def _members(self) -> Generator[MemberDef, None, None]: + yield from self._struct + + @property + def _member_types(self) -> Generator[XpdtType, None, None]: + yield from (m.typedef.type for m in self._struct) + + @property + def struct_fmt(self) -> str: + return self._struct.struct_fmt + + @property + def struct_tag(self) -> str: + return self._struct.tag + + @property + def struct(self) -> StructDef: + return self._struct + + @property + def ctype(self) -> str: + return self._struct.ctype + + @property + def needs_vbuf(self) -> bool: + return any((t.needs_vbuf for t in self._member_types)) + + @property + def pytype(self) -> str: + return self._struct.name + + @property + def is_scalar(self) -> bool: + return False + + @property + def struct_name(self) -> str: + # Allow the possibility for typedef and struct name to differ + return self._struct.name + + def construct_recursive(self, + path: Tuple[MemberDef, ...] = (), + ) -> Generator[ConstructElement, None, None]: + yield from self._struct.construct_recursive(path) + + @property + def scalar_members(self) -> Generator[Tuple[str, ...], None, None]: + yield from self._struct.scalar_members diff --git a/xpdt/templates/__init__.py b/xpdt/templates/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/xpdt/templates/func_decls.h b/xpdt/templates/func_decls.h new file mode 100644 index 0000000..900264f --- /dev/null +++ b/xpdt/templates/func_decls.h @@ -0,0 +1 @@ +bool x1b_write_/*{s.name}*/(/*{s|map(attribute="cdecl")|join(", ")}*/); diff --git a/xpdt/templates/x1b.c b/xpdt/templates/x1b.c new file mode 100644 index 0000000..19bca85 --- /dev/null +++ b/xpdt/templates/x1b.c @@ -0,0 +1,33 @@ +// import 'x1b_defns.c' as x1b +#pragma once + +/* Generated by xpdt + * xpdt is written by Gianni Tedesco + * https://www.scaramanga.co.uk +*/ + +#include + +#include /*{x1b_hdr.path_spec}*/ + +/* ==== structs ==== */ + +/*#- for struct in namespace #*/ + +/*{x1b.struct_decl(struct)}*/ +/*# if struct.needs_vbuf #*/ + +/*{x1b.serialized_struct_decl(struct)}*/ + +/*{x1b.blob_struct_decl(struct)}*/ +/*# endif #*/ +/*#- endfor -#*/ + +/*# for struct in namespace #*/ + + +/* ==== /*{struct.name}*/ ==== */ +/*{x1b.ctors(struct)}*/ +/*{x1b.writers(struct)}*/ +/*{x1b.readers(struct)}*/ +/*#- endfor -#*/ diff --git a/xpdt/templates/x1b.pyt b/xpdt/templates/x1b.pyt new file mode 100644 index 0000000..bbda8e5 --- /dev/null +++ b/xpdt/templates/x1b.pyt @@ -0,0 +1,65 @@ +# This code is auto-generated by xpdt; do not modify +# xpdt is written by Gianni Tedesco +# https://www.scaramanga.co.uk + +## import 'x1b_defns.pyt' as x1b +## import 'x1v_defns.pyt' as x1v +from struct import Struct as _Struct +from typing import NamedTuple as _NamedTuple, Tuple as _Tuple + +__all__ = ( +#% for struct in namespace %# + '$$struct.name$$', +#% endfor %# +) +#% macro write_class(struct) %# + + +class _$$struct.name$$_storage(_NamedTuple): +#% for name, (type_name, type) in struct %# + $$name$$: $$type.pytype$$ +#% endfor %# + + +class $$struct.name$$(_$$struct.name$$_storage): + __slots__ = () + +## if struct.needs_vbuf + _fmt = _Struct('=I$$struct.struct_fmt$$') +##else + _fmt = _Struct('=$$struct.struct_fmt$$') +## endif + _fmt_size = _fmt.size + _pack = _fmt.pack + _pack_into = _fmt.pack_into + _unpack_from = _fmt.unpack_from + _bin_size = _fmt.size + + @classmethod + def _frombytes(cls, + buf: bytes, + ) -> '$$struct.name$$': + _, obj = cls._frombuf(buf, 0) + return obj +#% if struct.needs_vbuf %# +$$x1v.write_methods(struct)$$ +#% elif not struct.all_members_scalar %# +$$x1b.write_methods(struct)$$ +#% else %# + + def __bytes__(self) -> bytes: + return self._pack(*self) + + @classmethod + def _frombuf(cls, + buf: bytes, + off: int = 0, + ) -> _Tuple[int, '$$struct.name$$']: + fields = cls._unpack_from(buf, off) + return cls._bin_size, cls(*fields) +#% endif %# +#%- endmacro -%# + +#%- for struct in namespace -%# +$$write_class(struct)$$ +#%- endfor -%# diff --git a/xpdt/templates/x1b_defns.c b/xpdt/templates/x1b_defns.c new file mode 100644 index 0000000..d7e3799 --- /dev/null +++ b/xpdt/templates/x1b_defns.c @@ -0,0 +1,341 @@ +/*#- macro init_macro_name(struct) -#*/ +/*{struct.name.upper()}*/_INIT +/*#- endmacro #*/ + + +/*#- macro ctor(struct) -#*/ +/*{struct.name}*/_new +/*#- endmacro -#*/ + + +/*#- macro xrec_ctor(struct) -#*/ +/*{struct.name}*/_xrec +/*#- endmacro -#*/ + + +/*#- macro xptrs_ctor(struct) -#*/ +/*{struct.name}*/_xptrs +/*#- endmacro -#*/ + + +/*#- macro raw_write(struct) -#*/ +/*{struct.name}*/_raw_write +/*#- endmacro -#*/ + + +/*#- macro bufreq(struct) -#*/ +/*{struct.name}*/_bufreq +/*#- endmacro -#*/ + + +/*#- macro construct(struct) -#*/ +/*{struct.name}*/_construct +/*#- endmacro -#*/ + + +/*#- macro cdecls(struct) -#*/ +/*{struct + | map(attribute="const_cdecl") + | join(",\n\t\t")}*/ +/*#- endmacro -#*/ + + +/*#- macro vbuf_cdecls(struct) -#*/ +/*{struct.vbuf_members + | map(attribute="const_cdecl") + | join(",\n\t\t")}*/ +/*#- endmacro -#*/ + + +/*#- macro memb_names(struct) -#*/ +/*{struct + | map(attribute="name") + | join(",\n\t\t")}*/ +/*#- endmacro -#*/ + + +/*#- macro _memb_names(struct) -#*/ +/*{struct + | map(attribute="name") + | map("prefix", "_") + | join(",\n\t\t") }*/ +/*#- endmacro -#*/ + + +/*#- macro buf_lens(struct) -#*/ +/*{struct.c_named_initializers_x1v + | map("wrap", "obj.", ".len") + | join("\n\t\t+ ")}*/ +/*#- endmacro -#*/ + + +/*#- macro naturally_packed(struct) -#*/ +/*#- if struct.needs_vbuf -#*/ +__attribute__((packed)) +/*#- endif -#*/ +/*#- endmacro -#*/ + + +/*#- macro struct_xrec(struct) -#*/ +struct /*{struct.name}*/_xrec +/*#- endmacro -#*/ + + +/*#- macro struct_envelope(struct) -#*/ +struct /*{struct.name}*/_xrec_envelope +/*#- endmacro -#*/ + + +/*#- macro struct_xptrs(struct) -#*/ +struct /*{struct.name}*/_xptrs +/*#- endmacro -#*/ + + +/*#- macro struct_decl(struct) -#*/ + +struct /*{struct.tag}*/ { +// for member in struct + /*{member.cdecl}*/; +// endfor +} /*{naturally_packed(struct)}*/; +/*#- endmacro -#*/ + + +/*#- macro serialized_struct_decl(struct) -#*/ + +/*{struct_xrec(struct)}*/ { +// for member in struct +// if member.needs_vbuf +// if member.is_scalar + x1b_strlen_t /*{member.name}*/; +// else + /*{struct_xrec(member.type.struct)}*/ /*{member.name}*/; +// endif +// else + /*{member.cdecl}*/; +// endif +// endfor +} __attribute__((packed)); + +/*{struct_envelope(struct)}*/ { + x1b_strlen_t tot_len; + /*{struct_xrec(struct)}*/ fixed; + uint8_t buffer[0]; +} __attribute__((packed)); +/*#- endmacro -#*/ + + + +/*#- macro blob_struct_decl(struct) -#*/ + +/*{struct_xptrs(struct)}*/ { +// for member in struct.vbuf_members +// if member.type.is_scalar + const uint8_t * /*{-member.name}*/; +// else + /*{struct_xptrs(member.type.struct)}*/ /*{member.name}*/; +// endif +// endfor +}; +/*#- endmacro -#*/ + + +/*#- macro fixed_ctor_defn(struct) #*/ +#define /*{init_macro_name(struct)}*/( \ + /*{_memb_names(struct)|macro}*/) \ + (/*{struct.ctype}*/){ \ +// for member in struct + ./*{member.name}*/ = _/*{member.name}*/, \ +// endfor + } + +static inline +/*{struct.ctype}*/ +/*{ctor(struct)}*/(/*{cdecls(struct)}*/) +{ + return /*{init_macro_name(struct)}*/( + /*{memb_names(struct)}*/); +} +/*#- endmacro -#*/ + + +/*#- macro vbuf_ctor_defn(struct) #*/ + +static inline +/*{struct_xrec(struct)}*/ +/*{xrec_ctor(struct)}*/(struct /*{struct.tag}*/ obj) +{ + return (/*{struct_xrec(struct)}*/){ +// for member in struct +// if member.needs_vbuf +// if member.is_scalar + ./*{member.name}*/ = obj./*{member.name}*/.len, +// else + ./*{member.name}*/ = /*{xrec_ctor(member.type.struct)}*/(obj./*{member.name}*/), +// endif +// else + ./*{member.name}*/ = obj./*{member.name}*/, +// endif +// endfor + }; +} + +static inline +/*{struct_xptrs(struct)}*/ +/*{xptrs_ctor(struct)}*/(struct /*{struct.tag}*/ obj) +{ + return (/*{struct_xptrs(struct)}*/){ +// for path in struct.c_named_initializers_x1v + ./*{path}*/ = obj./*{path}*/.ptr, +// endfor + }; +} +/*#- endmacro -#*/ + + +// macro ctors(struct) +/*{fixed_ctor_defn(struct)}*/ +// if struct.needs_vbuf +/*{vbuf_ctor_defn(struct)}*/ +// endif +// endmacro + + +/*# macro raw_write_fixed_func(struct) #*/ +static inline +bool /*{raw_write(struct)}*/(xostream_t out, const /*{struct.ctype}*/ obj) +{ + /*{struct.ctype}*/ *ptr; + + ptr = (/*{struct.ctype}*/ *)xostream_prepare(out, sizeof(obj)); + if (x1b_unlikely(ptr == NULL)) { + return false; + } + + *ptr = obj; + + return xostream_commit(out, sizeof(obj)); +} +/*# endmacro #*/ + + +/*# macro raw_write_variable_func(struct) #*/ +static inline +size_t /*{bufreq(struct)}*/(const /*{struct.ctype}*/ obj) +{ + return /*{buf_lens(struct)}*/; +} + +static inline +bool /*{raw_write(struct)}*/(xostream_t out, const /*{struct.ctype}*/ obj) +{ + /*{struct_envelope(struct)}*/ *ptr; + const size_t tot_len = sizeof(*ptr) + /*{bufreq(struct)}*/(obj); + x1b_strlen_t str_len; + uint8_t *buf; + + ptr = (/*{struct_envelope(struct)}*/ *)xostream_prepare(out, tot_len); + if (x1b_unlikely(ptr == NULL)) { + return false; + } + + ptr->tot_len = tot_len; + ptr->fixed = /*{xrec_ctor(struct)}*/(obj); + + buf = ptr->buffer; +/*# for path in struct.c_named_initializers_x1v #*/ + + str_len = obj./*{path}*/.len; + memcpy(buf, obj./*{path}*/.ptr, str_len); + buf += str_len; +/*# endfor #*/ + + return xostream_commit(out, tot_len); +} +/*# endmacro #*/ + +/*#- macro writers(struct) -#*/ +// if struct.needs_vbuf +/*{raw_write_variable_func(struct)}*/ +// else +/*{raw_write_fixed_func(struct)}*/ +// endif + +static inline +bool /*{struct.name}*/_write(xostream_t out, + /*{cdecls(struct)}*/) +{ + return /*{raw_write(struct)}*/(out, /*{ctor(struct)}*/( + /*{memb_names(struct)}*/)); +} +/*#- endmacro -#*/ + + +/*#- macro readers(struct) #*/ + +static inline +const /*{struct_xrec(struct)}*/ * /*{-struct.name}*/_read_fixed(struct xbuf_iter *it) +{ + const /*{struct_envelope(struct)}*/ *e; + const uint8_t *ptr, *end; + + ptr = it->it_ptr; + end = it->it_end; + + e = (/*{struct_envelope(struct)}*/ *)ptr; + if (x1b_unlikely((uint8_t *)&e->fixed > end)) + return NULL; + + ptr += e->tot_len; + if (x1b_unlikely(ptr > end)) + return NULL; + + it->it_ptr = ptr; + return &e->fixed; +} + +static inline +const /*{struct_xrec(struct)}*/ * /*{-struct.name}*/_read(struct xbuf_iter *it, /*{struct_xptrs(struct)}*/ *ptrs) +{ + const /*{struct_envelope(struct)}*/ *e; + const uint8_t *buf, *ptr, *end; + + ptr = it->it_ptr; + end = it->it_end; + + e = (/*{struct_envelope(struct)}*/ *)ptr; + buf = e->buffer; + if (x1b_unlikely((uint8_t *)&e->fixed > end)) + return NULL; + + ptr += e->tot_len; + if (x1b_unlikely(ptr > end)) + return NULL; +/*# for path in struct.c_named_initializers_x1v #*/ + + ptrs->/*{path}*/ = buf; + buf += e->fixed./*{path}*/; +/*# endfor #*/ + + it->it_ptr = ptr; + return &e->fixed; +} + +static inline +/*{struct.ctype}*/ /*{construct(struct)}*/(const /*{struct_xrec(struct)}*/ *rec, const /*{struct_xptrs(struct)}*/ *ptrs) +{ + return (/*{struct.ctype}*/){ +// for member in struct +// if member.needs_vbuf +// if member.is_scalar + ./*{member.name}*/ = xbuf(rec->/*{member.name}*/, ptrs->/*{member.name}*/), +// else + ./*{member.name}*/ = /*{member.type.struct.name}*/_construct(&rec->/*{member.name}*/, &ptrs->/*{member.name}*/), +// endif +// else + ./*{member.name}*/ = rec->/*{member.name}*/, +// endif +// endfor + }; +} +/*#- endmacro -#*/ diff --git a/xpdt/templates/x1b_defns.pyt b/xpdt/templates/x1b_defns.pyt new file mode 100644 index 0000000..490a1e8 --- /dev/null +++ b/xpdt/templates/x1b_defns.pyt @@ -0,0 +1,37 @@ +#%- macro construct_object(elem) -%# +##- if elem.action.value == 'agg_push' +$$elem.member.type.struct_name$$( +##- elif elem.action.value == 'agg_pop' +), +##- else: +$$elem.python_var_name$$, +##- endif +#%- endmacro -%# + +#% macro write_methods(struct) %# + + def __bytes__(self) -> bytes: + # FIXME: This can be more efficient + return self._pack( +## for member in struct.scalar_members + self.$$ ".".join(member) $$, +## endfor + ) + + @classmethod + def _frombuf(cls, + buf: bytes, + off: int = 0, + ) -> _Tuple[int, '$$ struct.name $$']: + ( +## for var in struct.python_var_names + $$var$$, +## endfor + ) = cls._unpack_from(buf, off) + ret = cls( +## for e in struct.construct_recursive() +$$e.indent(' ', 3)$$$$construct_object(e)$$ +## endfor + ) + return cls._bin_size, ret +#%- endmacro -%# diff --git a/xpdt/templates/x1v_defns.pyt b/xpdt/templates/x1v_defns.pyt new file mode 100644 index 0000000..f988460 --- /dev/null +++ b/xpdt/templates/x1v_defns.pyt @@ -0,0 +1,69 @@ +#%- macro construct_object(elem, vbuf_var) -%# +##- if elem.action.value == 'agg_push' +$$elem.member.type.struct_name$$( +##- elif elem.action.value == 'agg_pop' +), +##- else: +##- if elem.member.type.needs_vbuf +buf[off_$$vbuf_var.value$$:off_$$vbuf_var.value + 1$$], +##- set vbuf_var.value = vbuf_var.value + 1 +##- else +$$elem.python_var_name$$, +##- endif +##- endif +#%- endmacro -%# + +#% macro write_methods(struct) %# + + def __bytes__(self) -> bytes: +## for e in struct.all_vbuf_members + $$e.python_var_name$$_buf = self.$$ ".".join(e.full_path_names) $$ + $$e.python_var_name$$_len = len($$e.python_var_name$$_buf) +## endfor + tot_len = sum(( + self._fmt_size, +## for var in struct.python_vbuf_names + $$var$$_len, +## endfor + )) + + return b''.join(( + self._pack( + tot_len, +## for e in struct.all_members +## if e.member.type.needs_vbuf + $$e.python_var_name$$_len, +## else + self.$$ ".".join(e.full_path_names) $$, +## endif +## endfor + ), +## for e in struct.all_vbuf_members + $$e.python_var_name$$_buf, +## endfor + )) + + @classmethod + def _frombuf(cls, + buf: bytes, + off: int = 0, + ) -> _Tuple[int, '$$struct.name$$']: + ( + tot_len, +## for var in struct.python_var_names + $$var$$, +## endfor + ) = cls._unpack_from(buf, off) + + off_0 = off + cls._fmt_size +## for var in struct.python_vbuf_names + off_$$loop.index0 + 1$$ = off_$$loop.index0$$ + $$var$$ +## endfor + ret = cls( +## set vbuf_var = namespace(value=0) +## for e in struct.construct_recursive() +$$e.indent(' ', 3)$$$$construct_object(e, vbuf_var)$$ +## endfor + ) + return tot_len, ret +#%- endmacro -%# diff --git a/xpdt/type.py b/xpdt/type.py new file mode 100644 index 0000000..26b6e90 --- /dev/null +++ b/xpdt/type.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +from typing import NamedTuple, Generator, Tuple, TYPE_CHECKING +from enum import Enum +from abc import ABC, abstractmethod +from struct import calcsize + + +if TYPE_CHECKING: + from .member import MemberDef + + +__all__ = ( + 'ConstructAction', + 'ConstructElement', + 'XpdtType', +) + + +class ConstructAction(Enum): + AGG_PUSH = 'agg_push' + AGG_POP = 'agg_pop' + MEMBER = 'member' + + +class ConstructElement(NamedTuple): + action: ConstructAction + member: MemberDef + path: Tuple[MemberDef, ...] = () + + def indent(self, pat: str = '\t', base: int = 0) -> str: + return pat * (base + len(self.path)) + + @property + def path_names(self) -> Tuple[str, ...]: + return tuple((x.name for x in self.path)) + + @property + def full_path(self) -> Tuple[MemberDef, ...]: + return self.path + (self.member, ) + + @property + def full_path_names(self) -> Tuple[str, ...]: + return tuple((x.name for x in self.full_path)) + + @property + def c_named_initializer(self) -> str: + return '.'.join(self.full_path_names) + + @property + def python_var_name(self) -> str: + # FIXME: this is not prefix-free + return f'_{"_".join(self.full_path_names)}' + + +class XpdtType(ABC): + @property + @abstractmethod + def struct_fmt(self) -> str: + pass + + @property + @abstractmethod + def ctype(self) -> str: + pass + + @property + @abstractmethod + def pytype(self) -> str: + pass + + @property + @abstractmethod + def needs_vbuf(self) -> bool: + pass + + @property + def size(self) -> int: + return calcsize(self.struct_fmt) + + @property + def is_scalar(self) -> bool: + return True + + @property + def struct_name(self) -> str: + # Only relevant for aggregate types + raise NotImplementedError + + def construct_recursive(self, path: Tuple[MemberDef, ...] = ()) \ + -> Generator[ConstructElement, None, None]: + # Only relevant for aggregate types + raise NotImplementedError + + @property + def scalar_members(self) -> Generator[Tuple[str, ...], None, None]: + raise NotImplementedError diff --git a/xpdt/typedef.py b/xpdt/typedef.py new file mode 100644 index 0000000..e6dad0f --- /dev/null +++ b/xpdt/typedef.py @@ -0,0 +1,12 @@ +from typing import NamedTuple + +from .type import XpdtType + +__all__ = ( + 'TypeDef', +) + + +class TypeDef(NamedTuple): + name: str + type: XpdtType diff --git a/xpdt/uuidtype.py b/xpdt/uuidtype.py new file mode 100644 index 0000000..f09e0f6 --- /dev/null +++ b/xpdt/uuidtype.py @@ -0,0 +1,27 @@ +from .type import XpdtType + +__all__ = ( + 'UuidType', +) + + +class UuidType(XpdtType): + __slots__ = () + + pytype = 'bytes' + + @property + def ctype(self) -> str: + return 'xu128_t' + + @property + def struct_fmt(self) -> str: + return '16s' + + @property + def size(self) -> int: + return 16 + + @property + def needs_vbuf(self) -> bool: + return False