diff --git a/Dockerfile b/Dockerfile index 6c19f397..1ee333d8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,11 +22,13 @@ RUN /home/appuser/venv/bin/pip install --no-cache-dir -r requirements.lock # Copy application files +COPY osm2pgsql-query-builder/ /home/appuser/osm2pgsql-query-builder/ COPY README.md setup.py pyproject.toml /home/appuser/ COPY API/ /home/appuser/API/ COPY src/ /home/appuser/src/ -RUN /home/appuser/venv/bin/pip install --no-cache-dir . +RUN /home/appuser/venv/bin/pip install --no-cache-dir ./osm2pgsql-query-builder && \ + /home/appuser/venv/bin/pip install --no-cache-dir . # Final image FROM ghcr.io/osgeo/gdal:ubuntu-small-$GDAL_VERSION diff --git a/osm2pgsql-query-builder/LICENSE.md b/osm2pgsql-query-builder/LICENSE.md new file mode 100644 index 00000000..02687097 --- /dev/null +++ b/osm2pgsql-query-builder/LICENSE.md @@ -0,0 +1,594 @@ +# GNU General Public License + +_Version 3, 29 June 2007_ +_Copyright © 2007 Free Software Foundation, Inc. <>_ + +Everyone is permitted to copy and distribute verbatim copies of this license +document, but changing it is not allowed. + +## Preamble + +The GNU General Public License is a free, copyleft license for software and other +kinds of works. + +The licenses for most software and other practical works are designed to take away +your freedom to share and change the works. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change all versions of a +program--to make sure it remains free software for all its users. We, the Free +Software Foundation, use the GNU General Public License for most of our software; it +applies also to any other work released this way by its authors. You can apply it to +your programs, too. + +When we speak of free software, we are referring to freedom, not price. Our General +Public Licenses are designed to make sure that you have the freedom to distribute +copies of free software (and charge for them if you wish), that you receive source +code or can get it if you want it, that you can change the software or use pieces of +it in new free programs, and that you know you can do these things. + +To protect your rights, we need to prevent others from denying you these rights or +asking you to surrender the rights. Therefore, you have certain responsibilities if +you distribute copies of the software, or if you modify it: responsibilities to +respect the freedom of others. + +For example, if you distribute copies of such a program, whether gratis or for a fee, +you must pass on to the recipients the same freedoms that you received. You must make +sure that they, too, receive or can get the source code. And you must show them these +terms so they know their rights. + +Developers that use the GNU GPL protect your rights with two steps: **(1)** assert +copyright on the software, and **(2)** offer you this License giving you legal permission +to copy, distribute and/or modify it. + +For the developers' and authors' protection, the GPL clearly explains that there is +no warranty for this free software. For both users' and authors' sake, the GPL +requires that modified versions be marked as changed, so that their problems will not +be attributed erroneously to authors of previous versions. + +Some devices are designed to deny users access to install or run modified versions of +the software inside them, although the manufacturer can do so. This is fundamentally +incompatible with the aim of protecting users' freedom to change the software. The +systematic pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we have designed +this version of the GPL to prohibit the practice for those products. If such problems +arise substantially in other domains, we stand ready to extend this provision to +those domains in future versions of the GPL, as needed to protect the freedom of +users. + +Finally, every program is threatened constantly by software patents. States should +not allow patents to restrict development and use of software on general-purpose +computers, but in those that do, we wish to avoid the special danger that patents +applied to a free program could make it effectively proprietary. To prevent this, the +GPL assures that patents cannot be used to render the program non-free. + +The precise terms and conditions for copying, distribution and modification follow. + +## TERMS AND CONDITIONS + +### 0. Definitions + +“This License” refers to version 3 of the GNU General Public License. + +“Copyright” also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + +“The Program” refers to any copyrightable work licensed under this +License. Each licensee is addressed as “you”. “Licensees” and +“recipients” may be individuals or organizations. + +To “modify” a work means to copy from or adapt all or part of the work in +a fashion requiring copyright permission, other than the making of an exact copy. The +resulting work is called a “modified version” of the earlier work or a +work “based on” the earlier work. + +A “covered work” means either the unmodified Program or a work based on +the Program. + +To “propagate” a work means to do anything with it that, without +permission, would make you directly or secondarily liable for infringement under +applicable copyright law, except executing it on a computer or modifying a private +copy. Propagation includes copying, distribution (with or without modification), +making available to the public, and in some countries other activities as well. + +To “convey” a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through a computer +network, with no transfer of a copy, is not conveying. + +An interactive user interface displays “Appropriate Legal Notices” to the +extent that it includes a convenient and prominently visible feature that **(1)** +displays an appropriate copyright notice, and **(2)** tells the user that there is no +warranty for the work (except to the extent that warranties are provided), that +licensees may convey the work under this License, and how to view a copy of this +License. If the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + +### 1. Source Code + +The “source code” for a work means the preferred form of the work for +making modifications to it. “Object code” means any non-source form of a +work. + +A “Standard Interface” means an interface that either is an official +standard defined by a recognized standards body, or, in the case of interfaces +specified for a particular programming language, one that is widely used among +developers working in that language. + +The “System Libraries” of an executable work include anything, other than +the work as a whole, that **(a)** is included in the normal form of packaging a Major +Component, but which is not part of that Major Component, and **(b)** serves only to +enable use of the work with that Major Component, or to implement a Standard +Interface for which an implementation is available to the public in source code form. +A “Major Component”, in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system (if any) on which +the executable work runs, or a compiler used to produce the work, or an object code +interpreter used to run it. + +The “Corresponding Source” for a work in object code form means all the +source code needed to generate, install, and (for an executable work) run the object +code and to modify the work, including scripts to control those activities. However, +it does not include the work's System Libraries, or general-purpose tools or +generally available free programs which are used unmodified in performing those +activities but which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for the work, and +the source code for shared libraries and dynamically linked subprograms that the work +is specifically designed to require, such as by intimate data communication or +control flow between those subprograms and other parts of the work. + +The Corresponding Source need not include anything that users can regenerate +automatically from other parts of the Corresponding Source. + +The Corresponding Source for a work in source code form is that same work. + +### 2. Basic Permissions + +All rights granted under this License are granted for the term of copyright on the +Program, and are irrevocable provided the stated conditions are met. This License +explicitly affirms your unlimited permission to run the unmodified Program. The +output from running a covered work is covered by this License only if the output, +given its content, constitutes a covered work. This License acknowledges your rights +of fair use or other equivalent, as provided by copyright law. + +You may make, run and propagate covered works that you do not convey, without +conditions so long as your license otherwise remains in force. You may convey covered +works to others for the sole purpose of having them make modifications exclusively +for you, or provide you with facilities for running those works, provided that you +comply with the terms of this License in conveying all material for which you do not +control copyright. Those thus making or running the covered works for you must do so +exclusively on your behalf, under your direction and control, on terms that prohibit +them from making any copies of your copyrighted material outside their relationship +with you. + +Conveying under any other circumstances is permitted solely under the conditions +stated below. Sublicensing is not allowed; section 10 makes it unnecessary. + +### 3. Protecting Users' Legal Rights From Anti-Circumvention Law + +No covered work shall be deemed part of an effective technological measure under any +applicable law fulfilling obligations under article 11 of the WIPO copyright treaty +adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention +of such measures. + +When you convey a covered work, you waive any legal power to forbid circumvention of +technological measures to the extent such circumvention is effected by exercising +rights under this License with respect to the covered work, and you disclaim any +intention to limit operation or modification of the work as a means of enforcing, +against the work's users, your or third parties' legal rights to forbid circumvention +of technological measures. + +### 4. Conveying Verbatim Copies + +You may convey verbatim copies of the Program's source code as you receive it, in any +medium, provided that you conspicuously and appropriately publish on each copy an +appropriate copyright notice; keep intact all notices stating that this License and +any non-permissive terms added in accord with section 7 apply to the code; keep +intact all notices of the absence of any warranty; and give all recipients a copy of +this License along with the Program. + +You may charge any price or no price for each copy that you convey, and you may offer +support or warranty protection for a fee. + +### 5. Conveying Modified Source Versions + +You may convey a work based on the Program, or the modifications to produce it from +the Program, in the form of source code under the terms of section 4, provided that +you also meet all of these conditions: + +- **a)** The work must carry prominent notices stating that you modified it, and giving a + relevant date. +- **b)** The work must carry prominent notices stating that it is released under this + License and any conditions added under section 7. This requirement modifies the + requirement in section 4 to “keep intact all notices”. +- **c)** You must license the entire work, as a whole, under this License to anyone who + comes into possession of a copy. This License will therefore apply, along with any + applicable section 7 additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no permission to license the + work in any other way, but it does not invalidate such permission if you have + separately received it. +- **d)** If the work has interactive user interfaces, each must display Appropriate Legal + Notices; however, if the Program has interactive interfaces that do not display + Appropriate Legal Notices, your work need not make them do so. + +A compilation of a covered work with other separate and independent works, which are +not by their nature extensions of the covered work, and which are not combined with +it such as to form a larger program, in or on a volume of a storage or distribution +medium, is called an “aggregate” if the compilation and its resulting +copyright are not used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work in an aggregate +does not cause this License to apply to the other parts of the aggregate. + +### 6. Conveying Non-Source Forms + +You may convey a covered work in object code form under the terms of sections 4 and +5, provided that you also convey the machine-readable Corresponding Source under the +terms of this License, in one of these ways: + +- **a)** Convey the object code in, or embodied in, a physical product (including a + physical distribution medium), accompanied by the Corresponding Source fixed on a + durable physical medium customarily used for software interchange. +- **b)** Convey the object code in, or embodied in, a physical product (including a + physical distribution medium), accompanied by a written offer, valid for at least + three years and valid for as long as you offer spare parts or customer support for + that product model, to give anyone who possesses the object code either **(1)** a copy of + the Corresponding Source for all the software in the product that is covered by this + License, on a durable physical medium customarily used for software interchange, for + a price no more than your reasonable cost of physically performing this conveying of + source, or **(2)** access to copy the Corresponding Source from a network server at no + charge. +- **c)** Convey individual copies of the object code with a copy of the written offer to + provide the Corresponding Source. This alternative is allowed only occasionally and + noncommercially, and only if you received the object code with such an offer, in + accord with subsection 6b. +- **d)** Convey the object code by offering access from a designated place (gratis or for + a charge), and offer equivalent access to the Corresponding Source in the same way + through the same place at no further charge. You need not require recipients to copy + the Corresponding Source along with the object code. If the place to copy the object + code is a network server, the Corresponding Source may be on a different server + (operated by you or a third party) that supports equivalent copying facilities, + provided you maintain clear directions next to the object code saying where to find + the Corresponding Source. Regardless of what server hosts the Corresponding Source, + you remain obligated to ensure that it is available for as long as needed to satisfy + these requirements. +- **e)** Convey the object code using peer-to-peer transmission, provided you inform + other peers where the object code and Corresponding Source of the work are being + offered to the general public at no charge under subsection 6d. + +A separable portion of the object code, whose source code is excluded from the +Corresponding Source as a System Library, need not be included in conveying the +object code work. + +A “User Product” is either **(1)** a “consumer product”, which +means any tangible personal property which is normally used for personal, family, or +household purposes, or **(2)** anything designed or sold for incorporation into a +dwelling. In determining whether a product is a consumer product, doubtful cases +shall be resolved in favor of coverage. For a particular product received by a +particular user, “normally used” refers to a typical or common use of +that class of product, regardless of the status of the particular user or of the way +in which the particular user actually uses, or expects or is expected to use, the +product. A product is a consumer product regardless of whether the product has +substantial commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + +“Installation Information” for a User Product means any methods, +procedures, authorization keys, or other information required to install and execute +modified versions of a covered work in that User Product from a modified version of +its Corresponding Source. The information must suffice to ensure that the continued +functioning of the modified object code is in no case prevented or interfered with +solely because modification has been made. + +If you convey an object code work under this section in, or with, or specifically for +use in, a User Product, and the conveying occurs as part of a transaction in which +the right of possession and use of the User Product is transferred to the recipient +in perpetuity or for a fixed term (regardless of how the transaction is +characterized), the Corresponding Source conveyed under this section must be +accompanied by the Installation Information. But this requirement does not apply if +neither you nor any third party retains the ability to install modified object code +on the User Product (for example, the work has been installed in ROM). + +The requirement to provide Installation Information does not include a requirement to +continue to provide support service, warranty, or updates for a work that has been +modified or installed by the recipient, or for the User Product in which it has been +modified or installed. Access to a network may be denied when the modification itself +materially and adversely affects the operation of the network or violates the rules +and protocols for communication across the network. + +Corresponding Source conveyed, and Installation Information provided, in accord with +this section must be in a format that is publicly documented (and with an +implementation available to the public in source code form), and must require no +special password or key for unpacking, reading or copying. + +### 7. Additional Terms + +“Additional permissions” are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. Additional +permissions that are applicable to the entire Program shall be treated as though they +were included in this License, to the extent that they are valid under applicable +law. If additional permissions apply only to part of the Program, that part may be +used separately under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + +When you convey a copy of a covered work, you may at your option remove any +additional permissions from that copy, or from any part of it. (Additional +permissions may be written to require their own removal in certain cases when you +modify the work.) You may place additional permissions on material, added by you to a +covered work, for which you have or can give appropriate copyright permission. + +Notwithstanding any other provision of this License, for material you add to a +covered work, you may (if authorized by the copyright holders of that material) +supplement the terms of this License with terms: + +- **a)** Disclaiming warranty or limiting liability differently from the terms of + sections 15 and 16 of this License; or +- **b)** Requiring preservation of specified reasonable legal notices or author + attributions in that material or in the Appropriate Legal Notices displayed by works + containing it; or +- **c)** Prohibiting misrepresentation of the origin of that material, or requiring that + modified versions of such material be marked in reasonable ways as different from the + original version; or +- **d)** Limiting the use for publicity purposes of names of licensors or authors of the + material; or +- **e)** Declining to grant rights under trademark law for use of some trade names, + trademarks, or service marks; or +- **f)** Requiring indemnification of licensors and authors of that material by anyone + who conveys the material (or modified versions of it) with contractual assumptions of + liability to the recipient, for any liability that these contractual assumptions + directly impose on those licensors and authors. + +All other non-permissive additional terms are considered “further +restrictions” within the meaning of section 10. If the Program as you received +it, or any part of it, contains a notice stating that it is governed by this License +along with a term that is a further restriction, you may remove that term. If a +license document contains a further restriction but permits relicensing or conveying +under this License, you may add to a covered work material governed by the terms of +that license document, provided that the further restriction does not survive such +relicensing or conveying. + +If you add terms to a covered work in accord with this section, you must place, in +the relevant source files, a statement of the additional terms that apply to those +files, or a notice indicating where to find the applicable terms. + +Additional terms, permissive or non-permissive, may be stated in the form of a +separately written license, or stated as exceptions; the above requirements apply +either way. + +### 8. Termination + +You may not propagate or modify a covered work except as expressly provided under +this License. Any attempt otherwise to propagate or modify it is void, and will +automatically terminate your rights under this License (including any patent licenses +granted under the third paragraph of section 11). + +However, if you cease all violation of this License, then your license from a +particular copyright holder is reinstated **(a)** provisionally, unless and until the +copyright holder explicitly and finally terminates your license, and **(b)** permanently, +if the copyright holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + +Moreover, your license from a particular copyright holder is reinstated permanently +if the copyright holder notifies you of the violation by some reasonable means, this +is the first time you have received notice of violation of this License (for any +work) from that copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + +Termination of your rights under this section does not terminate the licenses of +parties who have received copies or rights from you under this License. If your +rights have been terminated and not permanently reinstated, you do not qualify to +receive new licenses for the same material under section 10. + +### 9. Acceptance Not Required for Having Copies + +You are not required to accept this License in order to receive or run a copy of the +Program. Ancillary propagation of a covered work occurring solely as a consequence of +using peer-to-peer transmission to receive a copy likewise does not require +acceptance. However, nothing other than this License grants you permission to +propagate or modify any covered work. These actions infringe copyright if you do not +accept this License. Therefore, by modifying or propagating a covered work, you +indicate your acceptance of this License to do so. + +### 10. Automatic Licensing of Downstream Recipients + +Each time you convey a covered work, the recipient automatically receives a license +from the original licensors, to run, modify and propagate that work, subject to this +License. You are not responsible for enforcing compliance by third parties with this +License. + +An “entity transaction” is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an organization, or +merging organizations. If propagation of a covered work results from an entity +transaction, each party to that transaction who receives a copy of the work also +receives whatever licenses to the work the party's predecessor in interest had or +could give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if the predecessor +has it or can get it with reasonable efforts. + +You may not impose any further restrictions on the exercise of the rights granted or +affirmed under this License. For example, you may not impose a license fee, royalty, +or other charge for exercise of rights granted under this License, and you may not +initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging +that any patent claim is infringed by making, using, selling, offering for sale, or +importing the Program or any portion of it. + +### 11. Patents + +A “contributor” is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The work thus +licensed is called the contributor's “contributor version”. + +A contributor's “essential patent claims” are all patent claims owned or +controlled by the contributor, whether already acquired or hereafter acquired, that +would be infringed by some manner, permitted by this License, of making, using, or +selling its contributor version, but do not include claims that would be infringed +only as a consequence of further modification of the contributor version. For +purposes of this definition, “control” includes the right to grant patent +sublicenses in a manner consistent with the requirements of this License. + +Each contributor grants you a non-exclusive, worldwide, royalty-free patent license +under the contributor's essential patent claims, to make, use, sell, offer for sale, +import and otherwise run, modify and propagate the contents of its contributor +version. + +In the following three paragraphs, a “patent license” is any express +agreement or commitment, however denominated, not to enforce a patent (such as an +express permission to practice a patent or covenant not to sue for patent +infringement). To “grant” such a patent license to a party means to make +such an agreement or commitment not to enforce a patent against the party. + +If you convey a covered work, knowingly relying on a patent license, and the +Corresponding Source of the work is not available for anyone to copy, free of charge +and under the terms of this License, through a publicly available network server or +other readily accessible means, then you must either **(1)** cause the Corresponding +Source to be so available, or **(2)** arrange to deprive yourself of the benefit of the +patent license for this particular work, or **(3)** arrange, in a manner consistent with +the requirements of this License, to extend the patent license to downstream +recipients. “Knowingly relying” means you have actual knowledge that, but +for the patent license, your conveying the covered work in a country, or your +recipient's use of the covered work in a country, would infringe one or more +identifiable patents in that country that you have reason to believe are valid. + +If, pursuant to or in connection with a single transaction or arrangement, you +convey, or propagate by procuring conveyance of, a covered work, and grant a patent +license to some of the parties receiving the covered work authorizing them to use, +propagate, modify or convey a specific copy of the covered work, then the patent +license you grant is automatically extended to all recipients of the covered work and +works based on it. + +A patent license is “discriminatory” if it does not include within the +scope of its coverage, prohibits the exercise of, or is conditioned on the +non-exercise of one or more of the rights that are specifically granted under this +License. You may not convey a covered work if you are a party to an arrangement with +a third party that is in the business of distributing software, under which you make +payment to the third party based on the extent of your activity of conveying the +work, and under which the third party grants, to any of the parties who would receive +the covered work from you, a discriminatory patent license **(a)** in connection with +copies of the covered work conveyed by you (or copies made from those copies), or **(b)** +primarily for and in connection with specific products or compilations that contain +the covered work, unless you entered into that arrangement, or that patent license +was granted, prior to 28 March 2007. + +Nothing in this License shall be construed as excluding or limiting any implied +license or other defenses to infringement that may otherwise be available to you +under applicable patent law. + +### 12. No Surrender of Others' Freedom + +If conditions are imposed on you (whether by court order, agreement or otherwise) +that contradict the conditions of this License, they do not excuse you from the +conditions of this License. If you cannot convey a covered work so as to satisfy +simultaneously your obligations under this License and any other pertinent +obligations, then as a consequence you may not convey it at all. For example, if you +agree to terms that obligate you to collect a royalty for further conveying from +those to whom you convey the Program, the only way you could satisfy both those terms +and this License would be to refrain entirely from conveying the Program. + +### 13. Use with the GNU Affero General Public License + +Notwithstanding any other provision of this License, you have permission to link or +combine any covered work with a work licensed under version 3 of the GNU Affero +General Public License into a single combined work, and to convey the resulting work. +The terms of this License will continue to apply to the part which is the covered +work, but the special requirements of the GNU Affero General Public License, section +13, concerning interaction through a network will apply to the combination as such. + +### 14. Revised Versions of this License + +The Free Software Foundation may publish revised and/or new versions of the GNU +General Public License from time to time. Such new versions will be similar in spirit +to the present version, but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Program specifies that +a certain numbered version of the GNU General Public License “or any later +version” applies to it, you have the option of following the terms and +conditions either of that numbered version or of any later version published by the +Free Software Foundation. If the Program does not specify a version number of the GNU +General Public License, you may choose any version ever published by the Free +Software Foundation. + +If the Program specifies that a proxy can decide which future versions of the GNU +General Public License can be used, that proxy's public statement of acceptance of a +version permanently authorizes you to choose that version for the Program. + +Later license versions may give you additional or different permissions. However, no +additional obligations are imposed on any author or copyright holder as a result of +your choosing to follow a later version. + +### 15. Disclaimer of Warranty + +THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM “AS IS” WITHOUT WARRANTY OF ANY KIND, EITHER +EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE +QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE +DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + +### 16. Limitation of Liability + +IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY +COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS +PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, +INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE +OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE +WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + +### 17. Interpretation of Sections 15 and 16 + +If the disclaimer of warranty and limitation of liability provided above cannot be +given local legal effect according to their terms, reviewing courts shall apply local +law that most closely approximates an absolute waiver of all civil liability in +connection with the Program, unless a warranty or assumption of liability accompanies +a copy of the Program in return for a fee. + +_END OF TERMS AND CONDITIONS_ + +## How to Apply These Terms to Your New Programs + +If you develop a new program, and you want it to be of the greatest possible use to +the public, the best way to achieve this is to make it free software which everyone +can redistribute and change under these terms. + +To do so, attach the following notices to the program. It is safest to attach them +to the start of each source file to most effectively state the exclusion of warranty; +and each file should have at least the “copyright” line and a pointer to +where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + +If the program does terminal interaction, make it output a short notice like this +when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type 'show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type 'show c' for details. + +The hypothetical commands `show w` and `show c` should show the appropriate parts of +the General Public License. Of course, your program's commands might be different; +for a GUI interface, you would use an “about box”. + +You should also get your employer (if you work as a programmer) or school, if any, to +sign a “copyright disclaimer” for the program, if necessary. For more +information on this, and how to apply and follow the GNU GPL, see +<>. + +The GNU General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may consider it +more useful to permit linking proprietary applications with the library. If this is +what you want to do, use the GNU Lesser General Public License instead of this +License. But first, please read +<>. diff --git a/osm2pgsql-query-builder/README.md b/osm2pgsql-query-builder/README.md new file mode 100644 index 00000000..ef13693d --- /dev/null +++ b/osm2pgsql-query-builder/README.md @@ -0,0 +1,129 @@ +# osm2pgsql-query-builder + +SQL query builder for [osm2pgsql](https://osm2pgsql.org/)-format PostgreSQL databases. + +Takes filter parameters (geometry, tags, attributes, geometry type) and generates PostGIS SQL queries against the standard osm2pgsql table schema (`nodes`, `ways_line`, `ways_poly`, `relations`). + +Extracted from [raw-data-api](https://github.com/hotosm/raw-data-api) so the logic can be reused in other projects that work with osm2pgsql databases (e.g. [PostPass](https://github.com/hotosm/postpass)). + +## Install + +```bash +pip install osm2pgsql-query-builder +``` + +Or from a local checkout (e.g. within the raw-data-api monorepo): + +```bash +pip install ./osm2pgsql-query-builder +``` + +## Usage + +### Snapshot query (quick start) + +Use `SnapshotQueryParams` to build a params object and generate SQL: + +```python +from osm2pgsql_query_builder import SnapshotQueryParams, build_snapshot_query + +params = SnapshotQueryParams( + geometry={ + "type": "Polygon", + "coordinates": [[ + [84.924, 27.766], + [85.319, 27.766], + [85.319, 28.025], + [84.924, 28.025], + [84.924, 27.766], + ]], + }, + filters={ + "tags": {"all_geometry": {"join_or": {"building": []}}}, + "attributes": {"all_geometry": ["name"]}, + }, +) + +sql = build_snapshot_query(params) +``` + +### Custom export categories (YAML/JSON config) + +Use `CategoryBase` for parsing category definitions from config files: + +```python +from osm2pgsql_query_builder import CategoryBase + +# Parse a single category from a YAML/JSON config +category = CategoryBase( + types=["lines", "polygons"], + select=["name", "highway"], + where="tags['highway'] IS NOT NULL", +) + +# Use the parsed values with the builder functions +from osm2pgsql_query_builder import convert_tags_to_postgres + +where_sql = convert_tags_to_postgres(category.where) +# → "tags->>'highway' IS NOT NULL" +``` + +### Extending the models + +`SnapshotQueryParams` and `CategoryBase` are designed to be subclassed for +application-specific fields: + +```python +from osm2pgsql_query_builder import SnapshotQueryParams, CategoryBase +from pydantic import Field +from typing import List, Optional + +class MyParams(SnapshotQueryParams): + file_name: Optional[str] = None + min_zoom: Optional[int] = None + +class MyCategory(CategoryBase): + formats: List[str] = Field(...) +``` + +## Available functions + +| Function | Description | +| --- | --- | +| `build_snapshot_query` | Full snapshot extraction query with tag/attribute/geometry filters | +| `build_geometry_type_query` | Separate queries per geometry type (point/line/polygon) with schemas | +| `build_plain_geojson_query` | Simple query from tag filters and bbox | +| `build_geom_filter` | Geometry intersection filter clause | +| `build_column_select` | Column SELECT clause from attribute list | +| `build_tag_filter` | WHERE clause from tag filters | +| `parse_filters` | Parse filter dict into per-geometry-type filters | +| `build_where_clause` | WHERE clause with grid/country index support | +| `convert_tags_to_postgres` | Convert `tags['key']` syntax to `tags->>'key'` | +| `sanitize_filename` | Sanitize a string for use as a file name | +| `sanitize_column_name` | Sanitize a string for use as a SQL column alias | + +## Models + +| Model | Description | +| --- | --- | +| `SnapshotQueryParams` | Pydantic model for snapshot query parameters (geometry, filters, output type, etc.) | +| `CategoryBase` | Pydantic model for category config (types, select, where) | +| `Filters` | Tag + attribute filter container | +| `TagsFilter` | Per-geometry-type tag filters (point/line/polygon/all_geometry) | +| `AttributeFilter` | Per-geometry-type attribute column lists | +| `SQLFilter` | Single filter with join_or / join_and dicts | +| `GeometryValidatorMixin` | Mixin that extracts Polygon/MultiPolygon from Feature/FeatureCollection | +| `BaseModel` | Pydantic BaseModel with camelCase alias support | + +## Development + +```bash +uv sync --extra dev +uv run pytest +``` + +Build the package: + +```bash +uv build +``` diff --git a/osm2pgsql-query-builder/osm2pgsql_query_builder/__init__.py b/osm2pgsql-query-builder/osm2pgsql_query_builder/__init__.py new file mode 100644 index 00000000..7f636b29 --- /dev/null +++ b/osm2pgsql-query-builder/osm2pgsql_query_builder/__init__.py @@ -0,0 +1,56 @@ +from .builder import ( + build_column_select, + build_geom_filter, + build_geometry_type_query, + build_plain_geojson_query, + build_snapshot_query, + build_tag_condition, + build_tag_filter, + build_where_clause, + convert_tags_to_postgres, + parse_filters, + sanitize_column_name, + sanitize_filename, + wrap_geojson, +) +from .enums import SupportedFilters, SupportedGeometryFilters +from .models import ( + AttributeFilter, + BaseModel, + CategoryBase, + Filters, + GeometryValidatorMixin, + JoinFilterType, + SQLFilter, + SnapshotQueryParams, + TagsFilter, + to_camel, +) + +__all__ = [ + "build_column_select", + "build_geom_filter", + "build_geometry_type_query", + "build_plain_geojson_query", + "build_snapshot_query", + "build_tag_condition", + "build_tag_filter", + "build_where_clause", + "convert_tags_to_postgres", + "parse_filters", + "sanitize_column_name", + "sanitize_filename", + "wrap_geojson", + "SupportedFilters", + "SupportedGeometryFilters", + "AttributeFilter", + "BaseModel", + "CategoryBase", + "Filters", + "GeometryValidatorMixin", + "JoinFilterType", + "SQLFilter", + "SnapshotQueryParams", + "TagsFilter", + "to_camel", +] diff --git a/osm2pgsql-query-builder/osm2pgsql_query_builder/builder.py b/osm2pgsql-query-builder/osm2pgsql_query_builder/builder.py new file mode 100644 index 00000000..9e3e4916 --- /dev/null +++ b/osm2pgsql-query-builder/osm2pgsql_query_builder/builder.py @@ -0,0 +1,737 @@ +# Copyright (C) 2026 Humanitarian OpenStreetmap Team + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +# Humanitarian OpenStreetmap Team +# 1100 13th Street NW Suite 800 Washington, D.C. 20005 +# +"""SQL query builder for osm2pgsql-format PostgreSQL databases.""" + +import logging +import re +from json import dumps, loads + +from .enums import SupportedFilters, SupportedGeometryFilters + +log = logging.getLogger(__name__) + + +def wrap_geojson(query_list, ogr_export=None): + """Wrap queries in ST_AsGeoJSON or pass through for OGR exports.""" + table_base_query = [] + if ogr_export: + table_base_query = query_list + else: + for i in range(len(query_list)): + table_base_query.append( + f"""select ST_AsGeoJSON(t{i}.*) from ({query_list[i]}) t{i}""" + ) + final_query = " UNION ALL ".join(table_base_query) + return final_query + + +def build_geom_filter(geom, geom_lookup_by="ST_intersects"): + """Generate a geometry intersection filter clause.""" + geometry_dump = dumps(loads(geom.model_dump_json())) + return f"""{geom_lookup_by}(geom,(select ST_Union(ST_makeValid(ST_GEOMFROMGEOJSON('{geometry_dump}')))))""" + + +def sanitize_filename(input_str): + # Fixme I need to check every possible special character that can comeup on osm tags + input_str = re.sub(r"\s+", "_", input_str) # putting _ in every space + input_str = re.sub(":", "_", input_str) # putting _ in every : value + input_str = re.sub("-", "_", input_str) # putting _ in every - value + return input_str + + +def sanitize_column_name(input_str): + # Fixme I need to check every possible special character that can comeup on osm tags + input_str = re.sub(r"\s+", "_", input_str) # putting _ in every space + input_str = re.sub(":", "_", input_str) # putting _ in every : value + return input_str + + +def build_column_select( + columns, + create_schema=False, + output_type="geojson", + use_centroid=False, + include_osm_type=True, + include_user_metadata=False, +): + """Build a SELECT clause from a list of attribute columns.""" + + if len(columns) > 0: + filter_col = [] + filter_col.append("osm_id") + if include_osm_type: + filter_col.append("tableoid::regclass AS osm_type") + if include_user_metadata: + filter_col.extend(["uid", f""" "user" """, "timestamp"]) + if create_schema: + schema = {} + schema["osm_id"] = "int64" + schema["type"] = "str" + if include_user_metadata: + schema["uid"] = "int64" + schema["user"] = "str" + schema["timestamp"] = "str" + + if "*" in columns: + filter_col.append("tags") + if create_schema: + schema["tags"] = "str" + else: + for cl in columns: + splitted_cl = [cl] + if "," in cl: + splitted_cl = cl.split(",") + for cl in splitted_cl: + if cl != "": + filter_col.append( + f"""tags ->> '{cl.strip()}' as {sanitize_column_name(cl.strip())}""" + ) + if create_schema: + schema[sanitize_column_name(cl.strip())] = "str" + if output_type == "csv": # if it is csv geom logic is different + filter_col.append("ST_X(ST_Centroid(geom)) as longitude") + filter_col.append("ST_Y(ST_Centroid(geom)) as latitude") + filter_col.append("GeometryType(geom) as geom_type") + else: + filter_col.append("ST_Centroid(geom) as geom" if use_centroid else "geom") + select_condition = " , ".join(filter_col) + if create_schema: + return select_condition, schema + return select_condition + else: + if include_user_metadata: + return f"""osm_id, tableoid::regclass AS osm_type, tags,changeset, uid, "user", timestamp,{'ST_Centroid(geom) as geom' if use_centroid else 'geom'}""" + return f"osm_id, tableoid::regclass AS osm_type, tags,changeset,timestamp,{'ST_Centroid(geom) as geom' if use_centroid else 'geom'}" # this is default attribute that we will deliver to user if user defines his own attribute column then those will be appended with osm_id only + + +def build_tag_condition(key, value, filter_list): + if len(value) > 1: + v_l = [] + for lil in value: + v_l.append(f""" '{lil.strip()}' """) + v_l_join = ", ".join(v_l) + value_tuple = f"""({v_l_join})""" + + k = f""" '{key.strip()}' """ + filter_list.append("""tags ->> """ + k + """IN """ + value_tuple + """""") + elif len(value) == 1: + filter_list.append(f"""tags ->> '{key.strip()}' = '{value[0].strip()}'""") + else: + filter_list.append(f"""tags ? '{key.strip()}'""") + return filter_list + + +def build_tag_filter(filter, join_by=" OR ", plain_query_filter=False): + """Build a WHERE clause from tag filters.""" + final_filter = [] + if plain_query_filter: + for item in filter: + key = item["key"] + value = item["value"] + if len(value) == 1 and value[0] == "*": + value = [] + if len(value) >= 1: + sub_append = [] + pre = """ tags @> '{""" + post = """ }'""" + for v in value: + sub_append.append( + f"""{pre} "{key.strip()}" : "{v.strip()}" {post}""" + ) + sub_append_join = " OR ".join(sub_append) + + final_filter.append(f"({sub_append_join})") + else: + final_filter.append(f"""tags ? '{key.strip()}'""") + tag_filter = join_by.join(final_filter) + return tag_filter + + else: + for key, value in filter.items(): + if key == "join_or": + temp_logic = [] + if value: + for k, v in value.items(): + temp_logic = build_tag_condition(k, v, temp_logic) + final_filter.append(f"""{" OR ".join(temp_logic)}""") + + if key == "join_and": + temp_logic = [] + if value: + for k, v in value.items(): + temp_logic = build_tag_condition(k, v, temp_logic) + if len(temp_logic) == 1: + join_by = " AND " + final_filter.append(f"""{" AND ".join(temp_logic)}""") + + tag_filter = join_by.join(final_filter) + return tag_filter + + +def parse_filters(filters): + """Parse a filter dict into per-geometry-type tag and attribute filters.""" + tags = None + attributes = None + point_tag_filter = None + poly_tag_filter = None + line_tag_filter = None + master_tag_filter = None + point_attribute_filter = None + poly_attribute_filter = None + line_attribute_filter = None + master_attribute_filter = None + if filters: + for key, value in filters.items(): + if key == SupportedFilters.TAGS.value: + if value: + tags = value + for k, v in value.items(): + if k == SupportedGeometryFilters.POINT.value: + point_tag_filter = v + if k == SupportedGeometryFilters.LINE.value: + line_tag_filter = v + if k == SupportedGeometryFilters.POLYGON.value: + poly_tag_filter = v + if k == SupportedGeometryFilters.ALLGEOM.value: + master_tag_filter = v + if key == SupportedFilters.ATTRIBUTES.value: + if value: + attributes = value + for k, v in value.items(): + if k == SupportedGeometryFilters.POINT.value: + point_attribute_filter = v + if k == SupportedGeometryFilters.LINE.value: + line_attribute_filter = v + if k == SupportedGeometryFilters.POLYGON.value: + poly_attribute_filter = v + if k == SupportedGeometryFilters.ALLGEOM.value: + master_attribute_filter = v + return ( + tags, + attributes, + point_attribute_filter, + line_attribute_filter, + poly_attribute_filter, + master_attribute_filter, + point_tag_filter, + line_tag_filter, + poly_tag_filter, + master_tag_filter, + ) + + +def build_where_clause( + geom_filter, g_id, c_id, country_export, table_name="ways_poly" +): + """Build a WHERE clause with optional grid/country index filters.""" + where_clause = geom_filter + if g_id: + if ( + table_name == "ways_poly" + ): # currently grid index is only available for ways_poly + column_name = "grid" + grid_filter_base = [f"""{column_name} = {ind[0]}""" for ind in g_id] + grid_filter = " OR ".join(grid_filter_base) + where_clause = f"({grid_filter}) and ({geom_filter})" + if c_id: + c_id = ",".join(str(num) for num in c_id) + where_clause += f" and (country @> ARRAY[{c_id}])" + if ( + country_export + ): # ignore the geometry take geom from the db itself by using precalculated field + if c_id: + where_clause = f"country @> ARRAY[{c_id}]" + return where_clause + + +def build_geometry_type_query( + params, + ogr_export=False, + g_id=None, + c_id=None, + country_export=False, +): + """Used for specifically focused on export tool, this will generate separate queries for line point and polygon can be used on other datatype support""" + include_user_metadata = params.include_user_metadata + geom_filter = build_geom_filter( + params.geometry, + "ST_within" if params.use_st_within is True else "ST_intersects", + ) + select_condition = f"""osm_id, tableoid::regclass AS osm_type, tags,changeset,timestamp , {'ST_Centroid(geom) as geom' if params.centroid else 'geom'}""" # this is default attribute that we will deliver to user if user defines his own attribute column then those will be appended with osm_id only + schema = { + "osm_id": "int64", + "type": "str", + "tags": "str", + "changeset": "int64", + "timestamp": "str", + } + query_point, query_line, query_poly = None, None, None + ( + attribute_filter, + master_attribute_filter, + master_tag_filter, + poly_attribute_filter, + poly_tag_filter, + ) = (None, None, None, None, None) + point_schema, line_schema, poly_schema = None, None, None + ( + tags, + attributes, + point_attribute_filter, + line_attribute_filter, + poly_attribute_filter, + master_attribute_filter, + point_tag_filter, + line_tag_filter, + poly_tag_filter, + master_tag_filter, + ) = (None, None, None, None, None, None, None, None, None, None) + if params.filters: + params.filters = ( + params.filters.model_dump() + ) # FIXME: temp fix , since validation model got changed + ( + tags, + attributes, + point_attribute_filter, + line_attribute_filter, + poly_attribute_filter, + master_attribute_filter, + point_tag_filter, + line_tag_filter, + poly_tag_filter, + master_tag_filter, + ) = parse_filters(params.filters) + + if ( + master_attribute_filter + ): # if no specific point , line or poly filter is not passed master columns filter will be used , if master columns is also empty then above default select statement will be used + select_condition, schema = build_column_select( + use_centroid=params.centroid, + output_type=params.output_type, + columns=master_attribute_filter, + create_schema=True, + include_user_metadata=include_user_metadata, + ) + if master_tag_filter: + attribute_filter = build_tag_filter(master_tag_filter) + if params.geometry_type is None: # fix me + params.geometry_type = ["point", "line", "polygon"] + + for type in params.geometry_type: + if type == SupportedGeometryFilters.POINT.value: + if point_attribute_filter: + select_condition, schema = build_column_select( + use_centroid=params.centroid, + output_type=params.output_type, + columns=point_attribute_filter, + create_schema=True, + include_user_metadata=include_user_metadata, + ) + where_clause_for_nodes = build_where_clause( + geom_filter, g_id, c_id, country_export, "nodes" + ) + + query_point = f"""select + {select_condition} + from + nodes + where + {where_clause_for_nodes}""" + if point_tag_filter: + attribute_filter = build_tag_filter(point_tag_filter) + if attribute_filter: + query_point += f""" and ({attribute_filter})""" + point_schema = schema + + query_point = wrap_geojson([query_point], ogr_export=ogr_export) + + if type == SupportedGeometryFilters.LINE.value: + query_line_list = [] + if line_attribute_filter: + select_condition, schema = build_column_select( + use_centroid=params.centroid, + output_type=params.output_type, + columns=line_attribute_filter, + create_schema=True, + include_user_metadata=include_user_metadata, + ) + where_clause_for_line = build_where_clause( + geom_filter, g_id, c_id, country_export, "ways_line" + ) + + query_ways_line = f"""select + {select_condition} + from + ways_line + where + {where_clause_for_line}""" + where_clause_for_rel = build_where_clause( + geom_filter, g_id, c_id, country_export, "relations" + ) + + query_relations_line = f"""select + {select_condition} + from + relations + where + {where_clause_for_rel}""" + if line_tag_filter: + attribute_filter = build_tag_filter(line_tag_filter) + if attribute_filter: + query_ways_line += f""" and ({attribute_filter})""" + query_relations_line += f""" and ({attribute_filter})""" + query_relations_line += """ and (geometrytype(geom)='MULTILINESTRING')""" + query_line_list.append(query_ways_line) + query_line_list.append(query_relations_line) + query_line = wrap_geojson(query_line_list, ogr_export=ogr_export) + line_schema = schema + + if type == SupportedGeometryFilters.POLYGON.value: + query_poly_list = [] + if poly_attribute_filter: + select_condition, schema = build_column_select( + use_centroid=params.centroid, + output_type=params.output_type, + columns=poly_attribute_filter, + create_schema=True, + include_user_metadata=include_user_metadata, + ) + + where_clause_for_poly = build_where_clause( + geom_filter, g_id, c_id, country_export, "ways_poly" + ) + + query_ways_poly = f"""select + {select_condition} + from + ways_poly + where + {where_clause_for_poly}""" + where_clause_for_relations = build_where_clause( + geom_filter, g_id, c_id, country_export, "relations" + ) + + query_relations_poly = f"""select + {select_condition} + from + relations + where + {where_clause_for_relations}""" + if poly_tag_filter: + attribute_filter = build_tag_filter(poly_tag_filter) + if attribute_filter: + query_ways_poly += f""" and ({attribute_filter})""" + query_relations_poly += f""" and ({attribute_filter})""" + query_relations_poly += """ and (geometrytype(geom)='POLYGON' or geometrytype(geom)='MULTIPOLYGON')""" + query_poly_list.append(query_ways_poly) + query_poly_list.append(query_relations_poly) + query_poly = wrap_geojson(query_poly_list, ogr_export=ogr_export) + poly_schema = schema + return query_point, query_line, query_poly, point_schema, line_schema, poly_schema + + +def build_snapshot_query( + params, + g_id=None, + c_id=None, + ogr_export=False, + country_export=False, +): + """Default function to support current snapshot extraction with all of the features that export_tool_api has""" + include_user_metadata = params.include_user_metadata + geom_lookup_by = "ST_within" if params.use_st_within is True else "ST_intersects" + geom_filter = build_geom_filter(params.geometry, geom_lookup_by) + + base_query = [] + + ( + tags, + attributes, + point_attribute_filter, + line_attribute_filter, + poly_attribute_filter, + master_attribute_filter, + point_tag_filter, + line_tag_filter, + poly_tag_filter, + master_tag_filter, + ) = (None, None, None, None, None, None, None, None, None, None) + + point_select_condition = None + line_select_condition = None + poly_select_condition = None + + point_tag = None + line_tag = None + poly_tag = None + master_tag = None + use_geomtype_in_relation = True + + # query_table = [] + if params.include_user_metadata: + select_condition = f"""osm_id, tableoid::regclass AS osm_type, version,tags,changeset, uid, "user", timestamp,{'ST_Centroid(geom) as geom' if params.centroid else 'geom'}""" + else: + select_condition = f"""osm_id, tableoid::regclass AS osm_type, version,tags,changeset,timestamp,{'ST_Centroid(geom) as geom' if params.centroid else 'geom'}""" # this is default attribute that we will deliver to user if user defines his own attribute column then those will be appended with osm_id only + + point_select_condition = select_condition # initializing default + line_select_condition = select_condition + poly_select_condition = select_condition + + if params.filters: + params.filters = ( + params.filters.model_dump() + ) # FIXME: temp fix , since validation model got changed + ( + tags, + attributes, + point_attribute_filter, + line_attribute_filter, + poly_attribute_filter, + master_attribute_filter, + point_tag_filter, + line_tag_filter, + poly_tag_filter, + master_tag_filter, + ) = parse_filters(params.filters) + attribute_customization_full_support = ["geojson", "shp"] + + if params.output_type not in attribute_customization_full_support: + log.debug( + "Merging filters since they don't have same no of filters for features" + ) + merged_array = [ + i if i else [] + for i in [ + point_attribute_filter, + line_attribute_filter, + poly_attribute_filter, + ] + ] + merged_result = list({x for l in merged_array for x in l}) + log.debug(merged_result) + if point_attribute_filter: + point_attribute_filter = merged_result + if line_attribute_filter: + line_attribute_filter = merged_result + if poly_attribute_filter: + poly_attribute_filter = merged_result + + if attributes: + if master_attribute_filter: + if len(master_attribute_filter) > 0: + select_condition = build_column_select( + use_centroid=params.centroid, + output_type=params.output_type, + columns=master_attribute_filter, + include_user_metadata=include_user_metadata, + ) + # if master attribute is supplied it will be applied to other geom type as well even though value is supplied they will be ignored + point_select_condition = select_condition + line_select_condition = select_condition + poly_select_condition = select_condition + else: + if point_attribute_filter: + if len(point_attribute_filter) > 0: + point_select_condition = build_column_select( + use_centroid=params.centroid, + output_type=params.output_type, + columns=point_attribute_filter, + include_user_metadata=include_user_metadata, + ) + if line_attribute_filter: + if len(line_attribute_filter) > 0: + line_select_condition = build_column_select( + use_centroid=params.centroid, + output_type=params.output_type, + columns=line_attribute_filter, + include_user_metadata=include_user_metadata, + ) + if poly_attribute_filter: + if len(poly_attribute_filter) > 0: + poly_select_condition = build_column_select( + use_centroid=params.centroid, + output_type=params.output_type, + columns=poly_attribute_filter, + include_user_metadata=include_user_metadata, + ) + + if tags: + if ( + master_tag_filter + ): # if master tag is supplied then other tags should be ignored and master tag will be used + master_tag = build_tag_filter(master_tag_filter) + point_tag = master_tag + line_tag = master_tag + poly_tag = master_tag + else: + if point_tag_filter: + point_tag = build_tag_filter(point_tag_filter) + if line_tag_filter: + line_tag = build_tag_filter(line_tag_filter) + if poly_tag_filter: + poly_tag = build_tag_filter(poly_tag_filter) + + # condition for geometry types + + if params.geometry_type is None or len(params.geometry_type) == 0: + params.geometry_type = ["point", "line", "polygon"] + + if SupportedGeometryFilters.ALLGEOM.value in params.geometry_type: + params.geometry_type = ["point", "line", "polygon"] + if SupportedGeometryFilters.POINT.value in params.geometry_type: + where_clause_for_nodes = build_where_clause( + geom_filter, g_id, c_id, country_export, "nodes" + ) + + query_point = f"""select + {point_select_condition} + from + nodes + where + {where_clause_for_nodes}""" + if point_tag: + query_point += f""" and ({point_tag})""" + base_query.append(query_point) + + if SupportedGeometryFilters.LINE.value in params.geometry_type: + where_clause_for_line = build_where_clause( + geom_filter, g_id, c_id, country_export, "ways_line" + ) + + query_ways_line = f"""select + {line_select_condition} + from + ways_line + where + {where_clause_for_line}""" + if line_tag: + query_ways_line += f""" and ({line_tag})""" + base_query.append(query_ways_line) + + if SupportedGeometryFilters.POLYGON.value in params.geometry_type: + if poly_select_condition == line_select_condition and poly_tag == line_tag: + use_geomtype_in_relation = False + + if use_geomtype_in_relation: + where_clause_for_rel = build_where_clause( + geom_filter, g_id, c_id, country_export, "relations" + ) + + query_relations_line = f"""select + {line_select_condition} + from + relations + where + {where_clause_for_rel}""" + if line_tag: + query_relations_line += f""" and ({line_tag})""" + query_relations_line += """ and (geometrytype(geom)='MULTILINESTRING')""" + base_query.append(query_relations_line) + + if SupportedGeometryFilters.POLYGON.value in params.geometry_type: + where_clause_for_poly = build_where_clause( + geom_filter, g_id, c_id, country_export, "ways_poly" + ) + + query_ways_poly = f"""select + {poly_select_condition} + from + ways_poly + where + {where_clause_for_poly}""" + if poly_tag: + query_ways_poly += f""" and ({poly_tag})""" + base_query.append(query_ways_poly) + where_clause_for_relations = build_where_clause( + geom_filter, g_id, c_id, country_export, "relations" + ) + query_relations_poly = f"""select + {poly_select_condition} + from + relations + where + {where_clause_for_relations}""" + if poly_tag: + query_relations_poly += f""" and ({poly_tag})""" + if use_geomtype_in_relation: + query_relations_poly += """ and (geometrytype(geom)='POLYGON' or geometrytype(geom)='MULTIPOLYGON')""" + base_query.append(query_relations_poly) + + if ogr_export: + # since query will be different for ogr exports and geojson exports because for ogr exports we don't need to grab each row in geojson + table_base_query = base_query + else: + table_base_query = [] + for i in range(len(base_query)): + table_base_query.append( + f"""select ST_AsGeoJSON(t{i}.*) from ({base_query[i]}) t{i}""" + ) + final_query = " UNION ALL ".join(table_base_query) + if params.output_type == "csv": + log.debug(final_query) + + return final_query + + +def build_plain_geojson_query(params, inspect_only=False): + """Build a simple GeoJSON query from tag filters and bbox.""" + geom_filter_cond = None + if params.geometry_type == "polygon": + geom_filter_cond = """ and (geometrytype(geom)='POLYGON' or geometrytype(geom)='MULTIPOLYGON')""" + select_condition = build_column_select(columns=params.select) + where_condition = build_tag_filter(params.where, params.join_by) + if params.bbox: + xmin, ymin, xmax, ymax = ( + params.bbox[0], + params.bbox[1], + params.bbox[2], + params.bbox[3], + ) + geom_condition = f"""ST_intersects(ST_MakeEnvelope({xmin}, {ymin}, {xmax}, {ymax},4326), geom)""" + + query_list = [] + for table_name in params.look_in: + sub_query = f"""select {select_condition} from {table_name} where ({where_condition}) """ + if params.bbox: + sub_query += f""" and {geom_condition}""" + if geom_filter_cond: + sub_query += geom_filter_cond + query_list.append(sub_query) + table_base_query = [] + for i in range(len(query_list)): + table_base_query.append( + f"""select ST_AsGeoJSON(t{i}.*) from ({query_list[i]}) t{i}""" + ) + final_query = " UNION ALL ".join(table_base_query) + if inspect_only: + final_query = f"""EXPLAIN + {final_query}""" + return final_query + + +def convert_tags_to_postgres(query_string): + """Convert tags['key'] syntax to tags->>'key' PostgreSQL syntax.""" + pattern = r"tags\['(.*?)'\]" + + converted_query = re.sub( + pattern, lambda match: f"tags->>'{match.group(1)}'", query_string + ) + + return converted_query diff --git a/osm2pgsql-query-builder/osm2pgsql_query_builder/enums.py b/osm2pgsql-query-builder/osm2pgsql_query_builder/enums.py new file mode 100644 index 00000000..b4275fcf --- /dev/null +++ b/osm2pgsql-query-builder/osm2pgsql_query_builder/enums.py @@ -0,0 +1,23 @@ +from enum import Enum + + +class SupportedFilters(Enum): + TAGS = "tags" + ATTRIBUTES = "attributes" + + @classmethod + def has_value(cls, value): + """Checks value""" + return value in cls._value2member_map_ + + +class SupportedGeometryFilters(Enum): + POINT = "point" + LINE = "line" + POLYGON = "polygon" + ALLGEOM = "all_geometry" + + @classmethod + def has_value(cls, value): + """Checks if the value is supported""" + return value in cls._value2member_map_ diff --git a/osm2pgsql-query-builder/osm2pgsql_query_builder/models.py b/osm2pgsql-query-builder/osm2pgsql_query_builder/models.py new file mode 100644 index 00000000..f45a40e2 --- /dev/null +++ b/osm2pgsql-query-builder/osm2pgsql_query_builder/models.py @@ -0,0 +1,197 @@ +# Copyright (C) 2021 Humanitarian OpenStreetmap Team + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +# Humanitarian OpenStreetmap Team +# 1100 13th Street NW Suite 800 Washington, D.C. 20005 +# +"""Pydantic models for osm2pgsql query parameters and filter configuration.""" + +from typing import Dict, List, Optional, Union + +from geojson_pydantic import Feature, FeatureCollection, MultiPolygon, Polygon +from pydantic import BaseModel as PydanticModel +from pydantic import ConfigDict, Field, field_validator + +from .enums import SupportedGeometryFilters + + +def to_camel(string: str) -> str: + split_string = string.split("_") + + return "".join([split_string[0], *[w.capitalize() for w in split_string[1:]]]) + + +class BaseModel(PydanticModel): + model_config = ConfigDict( + alias_generator=to_camel, + populate_by_name=True, + use_enum_values=True, + ) + + +class JoinFilterType: + OR = "OR" + AND = "AND" + + +class SQLFilter(BaseModel): + join_or: Optional[Dict[str, List[str]]] = Field(default=None) + join_and: Optional[Dict[str, List[str]]] = Field(default=None) + + +class TagsFilter(BaseModel): + point: Optional[SQLFilter] = Field(default=None) + line: Optional[SQLFilter] = Field(default=None) + polygon: Optional[SQLFilter] = Field(default=None) + all_geometry: Optional[SQLFilter] = Field(default=None) + + +class AttributeFilter(BaseModel): + point: Optional[List[str]] = Field(default=None) + line: Optional[List[str]] = Field(default=None) + polygon: Optional[List[str]] = Field(default=None) + all_geometry: Optional[List[str]] = Field(default=None) + + +class Filters(BaseModel): + tags: Optional[TagsFilter] = Field(default=None) + attributes: Optional[AttributeFilter] = Field(default=None) + + +class GeometryValidatorMixin: + @field_validator("geometry") + @classmethod + def validate_geometry(cls, value): + """Validates geometry""" + if value: + if value.type == "Feature": + if value.geometry.type not in ["Polygon", "MultiPolygon"]: + raise ValueError( + f"Feature geometry type {value.geometry.type} must be of type polygon/multipolygon", + ) + return value.geometry + if value.type == "FeatureCollection": + for feature in value.features: + if feature.geometry.type not in ["Polygon", "MultiPolygon"]: + raise ValueError( + f"Feature Collection can't have {feature.type} , should be polygon/multipolygon" + ) + if len(value.features) > 1: + raise ValueError( + "Feature collection with multiple features is not supported yet" + ) + return value.features[0].geometry + return value + + +class SnapshotQueryParams(BaseModel, GeometryValidatorMixin): + """Base parameters for snapshot extraction queries. + + Subclass this to add application-specific fields (e.g. output format enums, + file naming, zoom levels). + """ + + output_type: Optional[str] = Field( + default="geojson", json_schema_extra={"example": "geojson"} + ) + geometry_type: Optional[List[SupportedGeometryFilters]] = Field( + default=None, json_schema_extra={"example": ["point", "polygon"]} + ) + centroid: Optional[bool] = Field( + default=False, description="Exports centroid of features as geom" + ) + use_st_within: Optional[bool] = Field( + default=True, + description="Exports features which are exactly inside the passed polygons (ST_WITHIN) By default features which are intersected with passed polygon is exported", + ) + include_user_metadata: Optional[bool] = Field( + default=False, + description="Include user metadata on exports , Only available to logged in users", + ) + filters: Optional[Filters] = Field( + default=None, + json_schema_extra={ + "example": { + "tags": {"all_geometry": {"join_or": {"building": []}}}, + "attributes": {"all_geometry": ["name"]}, + }, + }, + description="Filter for point,line,polygon/ all geometry for both select and where clause, All geometry filter means : It will apply the same filter to all the geometry type", + ) + geometry: Union[ + Polygon, + MultiPolygon, + Feature, + FeatureCollection, + ] = Field( + json_schema_extra={ + "example": { + "type": "Polygon", + "coordinates": [ + [ + [83.96919250488281, 28.194446860487773], + [83.99751663208006, 28.194446860487773], + [83.99751663208006, 28.214869548073377], + [83.96919250488281, 28.214869548073377], + [83.96919250488281, 28.194446860487773], + ] + ], + }, + }, + ) + + @field_validator("geometry_type") + @classmethod + def return_unique_value(cls, value): + """return unique list""" + if value: + return list(set(value)) + return value + + +class CategoryBase(BaseModel): + """Base model for category configuration. + + Subclass this to add application-specific fields (e.g. hdx config, + export formats with validation). + """ + + types: List[str] = Field( + ..., + description="List of feature types (points, lines, polygons).", + json_schema_extra={"example": ["lines"]}, + ) + select: List[str] = Field( + ..., + description="List of selected fields.", + json_schema_extra={"example": ["name", "highway"]}, + ) + where: str = Field( + ..., + description="SQL-like condition to filter features.", + json_schema_extra={"example": "highway IS NOT NULL"}, + ) + + @field_validator("types") + @classmethod + def validate_types(cls, value): + """validates geom types""" + allowed_types = {"points", "lines", "polygons"} + for item in value: + if item not in allowed_types: + raise ValueError( + f"Invalid type: {item}. Allowed types are {', '.join(allowed_types)}" + ) + return value diff --git a/osm2pgsql-query-builder/pyproject.toml b/osm2pgsql-query-builder/pyproject.toml new file mode 100644 index 00000000..006f38c9 --- /dev/null +++ b/osm2pgsql-query-builder/pyproject.toml @@ -0,0 +1,21 @@ +[project] +name = "osm2pgsql-query-builder" +version = "0.1.0" +description = "SQL query builder for osm2pgsql-format PostgreSQL databases" +license = "GPL-3.0-only" +requires-python = ">=3.10" +dependencies = [ + "geojson-pydantic>=1.0.1", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.4.3", +] + +[tool.setuptools.packages.find] +where = ["."] + +[build-system] +requires = ["setuptools>=61.0.0"] +build-backend = "setuptools.build_meta" diff --git a/osm2pgsql-query-builder/tests/__init__.py b/osm2pgsql-query-builder/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/osm2pgsql-query-builder/tests/test_builder.py b/osm2pgsql-query-builder/tests/test_builder.py new file mode 100644 index 00000000..d2adfe4c --- /dev/null +++ b/osm2pgsql-query-builder/tests/test_builder.py @@ -0,0 +1,408 @@ +# Copyright (C) 2021 Humanitarian OpenStreetmap Team + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import pytest +from geojson_pydantic import Feature, Polygon +from pydantic import ValidationError + +from osm2pgsql_query_builder import ( + CategoryBase, + Filters, + SnapshotQueryParams, + build_snapshot_query, +) +from osm2pgsql_query_builder.models import ( + AttributeFilter, + GeometryValidatorMixin, + SQLFilter, + TagsFilter, +) + + +# Use SnapshotQueryParams directly as the test params model +QueryParams = SnapshotQueryParams + + +def test_snapshot_geometry_query(): + params = QueryParams( + geometry=Polygon( + type="Polygon", + coordinates=[ + [ + [84.92431640625, 27.766190642387496], + [85.31982421875, 27.766190642387496], + [85.31982421875, 28.02592458049937], + [84.92431640625, 28.02592458049937], + [84.92431640625, 27.766190642387496], + ] + ], + ), + output_type="geojson", + use_st_within=False, + filters=Filters( + tags={"point": {"join_or": {"amenity": ["shop", "toilet"]}}}, + attributes={"point": ["name"]}, + ), + ) + expected_query = """select ST_AsGeoJSON(t0.*) from (select + osm_id , tableoid::regclass AS osm_type , tags ->> 'name' as name , geom + from + nodes + where + ST_intersects(geom,(select ST_Union(ST_makeValid(ST_GEOMFROMGEOJSON('{"type": "Polygon", "coordinates": [[[84.92431640625, 27.766190642387496], [85.31982421875, 27.766190642387496], [85.31982421875, 28.02592458049937], [84.92431640625, 28.02592458049937], [84.92431640625, 27.766190642387496]]]}'))))) and (tags ->> 'amenity' IN ( 'shop' , 'toilet' ))) t0 UNION ALL select ST_AsGeoJSON(t1.*) from (select + osm_id, tableoid::regclass AS osm_type, version,tags,changeset,timestamp,geom + from + ways_line + where + ST_intersects(geom,(select ST_Union(ST_makeValid(ST_GEOMFROMGEOJSON('{"type": "Polygon", "coordinates": [[[84.92431640625, 27.766190642387496], [85.31982421875, 27.766190642387496], [85.31982421875, 28.02592458049937], [84.92431640625, 28.02592458049937], [84.92431640625, 27.766190642387496]]]}')))))) t1 UNION ALL select ST_AsGeoJSON(t2.*) from (select + osm_id, tableoid::regclass AS osm_type, version,tags,changeset,timestamp,geom + from + ways_poly + where + ST_intersects(geom,(select ST_Union(ST_makeValid(ST_GEOMFROMGEOJSON('{"type": "Polygon", "coordinates": [[[84.92431640625, 27.766190642387496], [85.31982421875, 27.766190642387496], [85.31982421875, 28.02592458049937], [84.92431640625, 28.02592458049937], [84.92431640625, 27.766190642387496]]]}')))))) t2 UNION ALL select ST_AsGeoJSON(t3.*) from (select + osm_id, tableoid::regclass AS osm_type, version,tags,changeset,timestamp,geom + from + relations + where + ST_intersects(geom,(select ST_Union(ST_makeValid(ST_GEOMFROMGEOJSON('{"type": "Polygon", "coordinates": [[[84.92431640625, 27.766190642387496], [85.31982421875, 27.766190642387496], [85.31982421875, 28.02592458049937], [84.92431640625, 28.02592458049937], [84.92431640625, 27.766190642387496]]]}')))))) t3""" + + query_result = build_snapshot_query(params) + assert query_result.encode("utf-8") == expected_query.encode("utf-8") + + +def test_snapshot_normal_query(): + params = QueryParams( + geometry=Polygon( + type="Polygon", + coordinates=[ + [ + [84.92431640625, 27.766190642387496], + [85.31982421875, 27.766190642387496], + [85.31982421875, 28.02592458049937], + [84.92431640625, 28.02592458049937], + [84.92431640625, 27.766190642387496], + ] + ], + ), + use_st_within=False, + output_type="geojson", + ) + expected_query = """select ST_AsGeoJSON(t0.*) from (select + osm_id, tableoid::regclass AS osm_type, version,tags,changeset,timestamp,geom + from + nodes + where + ST_intersects(geom,(select ST_Union(ST_makeValid(ST_GEOMFROMGEOJSON('{"type": "Polygon", "coordinates": [[[84.92431640625, 27.766190642387496], [85.31982421875, 27.766190642387496], [85.31982421875, 28.02592458049937], [84.92431640625, 28.02592458049937], [84.92431640625, 27.766190642387496]]]}')))))) t0 UNION ALL select ST_AsGeoJSON(t1.*) from (select + osm_id, tableoid::regclass AS osm_type, version,tags,changeset,timestamp,geom + from + ways_line + where + ST_intersects(geom,(select ST_Union(ST_makeValid(ST_GEOMFROMGEOJSON('{"type": "Polygon", "coordinates": [[[84.92431640625, 27.766190642387496], [85.31982421875, 27.766190642387496], [85.31982421875, 28.02592458049937], [84.92431640625, 28.02592458049937], [84.92431640625, 27.766190642387496]]]}')))))) t1 UNION ALL select ST_AsGeoJSON(t2.*) from (select + osm_id, tableoid::regclass AS osm_type, version,tags,changeset,timestamp,geom + from + ways_poly + where + ST_intersects(geom,(select ST_Union(ST_makeValid(ST_GEOMFROMGEOJSON('{"type": "Polygon", "coordinates": [[[84.92431640625, 27.766190642387496], [85.31982421875, 27.766190642387496], [85.31982421875, 28.02592458049937], [84.92431640625, 28.02592458049937], [84.92431640625, 27.766190642387496]]]}')))))) t2 UNION ALL select ST_AsGeoJSON(t3.*) from (select + osm_id, tableoid::regclass AS osm_type, version,tags,changeset,timestamp,geom + from + relations + where + ST_intersects(geom,(select ST_Union(ST_makeValid(ST_GEOMFROMGEOJSON('{"type": "Polygon", "coordinates": [[[84.92431640625, 27.766190642387496], [85.31982421875, 27.766190642387496], [85.31982421875, 28.02592458049937], [84.92431640625, 28.02592458049937], [84.92431640625, 27.766190642387496]]]}')))))) t3""" + query_result = build_snapshot_query(params) + assert query_result.encode("utf-8") == expected_query.encode("utf-8") + + +def test_snapshot_query_st_within(): + params = QueryParams( + geometry=Polygon( + type="Polygon", + coordinates=[ + [ + [84.92431640625, 27.766190642387496], + [85.31982421875, 27.766190642387496], + [85.31982421875, 28.02592458049937], + [84.92431640625, 28.02592458049937], + [84.92431640625, 27.766190642387496], + ] + ], + ), + output_type="geojson", + ) + expected_query = """select ST_AsGeoJSON(t0.*) from (select + osm_id, tableoid::regclass AS osm_type, version,tags,changeset,timestamp,geom + from + nodes + where + ST_within(geom,(select ST_Union(ST_makeValid(ST_GEOMFROMGEOJSON('{"type": "Polygon", "coordinates": [[[84.92431640625, 27.766190642387496], [85.31982421875, 27.766190642387496], [85.31982421875, 28.02592458049937], [84.92431640625, 28.02592458049937], [84.92431640625, 27.766190642387496]]]}')))))) t0 UNION ALL select ST_AsGeoJSON(t1.*) from (select + osm_id, tableoid::regclass AS osm_type, version,tags,changeset,timestamp,geom + from + ways_line + where + ST_within(geom,(select ST_Union(ST_makeValid(ST_GEOMFROMGEOJSON('{"type": "Polygon", "coordinates": [[[84.92431640625, 27.766190642387496], [85.31982421875, 27.766190642387496], [85.31982421875, 28.02592458049937], [84.92431640625, 28.02592458049937], [84.92431640625, 27.766190642387496]]]}')))))) t1 UNION ALL select ST_AsGeoJSON(t2.*) from (select + osm_id, tableoid::regclass AS osm_type, version,tags,changeset,timestamp,geom + from + ways_poly + where + ST_within(geom,(select ST_Union(ST_makeValid(ST_GEOMFROMGEOJSON('{"type": "Polygon", "coordinates": [[[84.92431640625, 27.766190642387496], [85.31982421875, 27.766190642387496], [85.31982421875, 28.02592458049937], [84.92431640625, 28.02592458049937], [84.92431640625, 27.766190642387496]]]}')))))) t2 UNION ALL select ST_AsGeoJSON(t3.*) from (select + osm_id, tableoid::regclass AS osm_type, version,tags,changeset,timestamp,geom + from + relations + where + ST_within(geom,(select ST_Union(ST_makeValid(ST_GEOMFROMGEOJSON('{"type": "Polygon", "coordinates": [[[84.92431640625, 27.766190642387496], [85.31982421875, 27.766190642387496], [85.31982421875, 28.02592458049937], [84.92431640625, 28.02592458049937], [84.92431640625, 27.766190642387496]]]}')))))) t3""" + query_result = build_snapshot_query(params) + assert query_result.encode("utf-8") == expected_query.encode("utf-8") + + +def test_attribute_filter(): + params = QueryParams( + geometry=Polygon( + type="Polygon", + coordinates=[ + [ + [83.502574, 27.569073], + [83.502574, 28.332758], + [85.556417, 28.332758], + [85.556417, 27.569073], + [83.502574, 27.569073], + ] + ], + ), + output_type="geojson", + use_st_within=False, + geometry_type=["polygon", "line"], + filters=Filters( + tags={"all_geometry": {"join_or": {"building": ["yes"]}}}, + attributes={"line": ["name"]}, + ), + ) + expected_query = """select ST_AsGeoJSON(t0.*) from (select + osm_id , tableoid::regclass AS osm_type , tags ->> 'name' as name , geom + from + ways_line + where + ST_intersects(geom,(select ST_Union(ST_makeValid(ST_GEOMFROMGEOJSON('{"type": "Polygon", "coordinates": [[[83.502574, 27.569073], [83.502574, 28.332758], [85.556417, 28.332758], [85.556417, 27.569073], [83.502574, 27.569073]]]}'))))) and (tags ->> 'building' = 'yes')) t0 UNION ALL select ST_AsGeoJSON(t1.*) from (select + osm_id , tableoid::regclass AS osm_type , tags ->> 'name' as name , geom + from + relations + where + ST_intersects(geom,(select ST_Union(ST_makeValid(ST_GEOMFROMGEOJSON('{"type": "Polygon", "coordinates": [[[83.502574, 27.569073], [83.502574, 28.332758], [85.556417, 28.332758], [85.556417, 27.569073], [83.502574, 27.569073]]]}'))))) and (tags ->> 'building' = 'yes') and (geometrytype(geom)='MULTILINESTRING')) t1 UNION ALL select ST_AsGeoJSON(t2.*) from (select + osm_id, tableoid::regclass AS osm_type, version,tags,changeset,timestamp,geom + from + ways_poly + where + (grid = 1187 OR grid = 1188) and (ST_intersects(geom,(select ST_Union(ST_makeValid(ST_GEOMFROMGEOJSON('{"type": "Polygon", "coordinates": [[[83.502574, 27.569073], [83.502574, 28.332758], [85.556417, 28.332758], [85.556417, 27.569073], [83.502574, 27.569073]]]}')))))) and (tags ->> 'building' = 'yes')) t2 UNION ALL select ST_AsGeoJSON(t3.*) from (select + osm_id, tableoid::regclass AS osm_type, version,tags,changeset,timestamp,geom + from + relations + where + ST_intersects(geom,(select ST_Union(ST_makeValid(ST_GEOMFROMGEOJSON('{"type": "Polygon", "coordinates": [[[83.502574, 27.569073], [83.502574, 28.332758], [85.556417, 28.332758], [85.556417, 27.569073], [83.502574, 27.569073]]]}'))))) and (tags ->> 'building' = 'yes') and (geometrytype(geom)='POLYGON' or geometrytype(geom)='MULTIPOLYGON')) t3""" + query_result = build_snapshot_query( + params, + g_id=[[1187], [1188]], + ) + assert query_result.encode("utf-8") == expected_query.encode("utf-8") + + +def test_and_filters(): + params = QueryParams( + geometry=Polygon( + type="Polygon", + coordinates=[ + [ + [36.70588085657477, 37.1979648807274], + [36.70588085657477, 37.1651408422983], + [36.759267544807194, 37.1651408422983], + [36.759267544807194, 37.1979648807274], + [36.70588085657477, 37.1979648807274], + ] + ], + ), + output_type="geojson", + use_st_within=False, + geometry_type=["polygon"], + filters=Filters( + tags={ + "point": {}, + "line": {}, + "polygon": { + "join_or": {}, + "join_and": { + "destroyed:building": ["yes"], + "damage:date": ["2023-02-06"], + }, + }, + }, + attributes={ + "point": [], + "line": [], + "polygon": [ + "building", + "destroyed:building", + "damage:date", + "name", + "source", + ], + }, + ), + ) + expected_query = """select ST_AsGeoJSON(t0.*) from (select + osm_id , tableoid::regclass AS osm_type , tags ->> 'building' as building , tags ->> 'destroyed:building' as destroyed_building , tags ->> 'damage:date' as damage_date , tags ->> 'name' as name , tags ->> 'source' as source , geom + from + ways_poly + where + ST_intersects(geom,(select ST_Union(ST_makeValid(ST_GEOMFROMGEOJSON('{"type": "Polygon", "coordinates": [[[36.70588085657477, 37.1979648807274], [36.70588085657477, 37.1651408422983], [36.759267544807194, 37.1651408422983], [36.759267544807194, 37.1979648807274], [36.70588085657477, 37.1979648807274]]]}'))))) and (tags ->> 'destroyed:building' = 'yes' AND tags ->> 'damage:date' = '2023-02-06')) t0 UNION ALL select ST_AsGeoJSON(t1.*) from (select + osm_id , tableoid::regclass AS osm_type , tags ->> 'building' as building , tags ->> 'destroyed:building' as destroyed_building , tags ->> 'damage:date' as damage_date , tags ->> 'name' as name , tags ->> 'source' as source , geom + from + relations + where + ST_intersects(geom,(select ST_Union(ST_makeValid(ST_GEOMFROMGEOJSON('{"type": "Polygon", "coordinates": [[[36.70588085657477, 37.1979648807274], [36.70588085657477, 37.1651408422983], [36.759267544807194, 37.1651408422983], [36.759267544807194, 37.1979648807274], [36.70588085657477, 37.1979648807274]]]}'))))) and (tags ->> 'destroyed:building' = 'yes' AND tags ->> 'damage:date' = '2023-02-06') and (geometrytype(geom)='POLYGON' or geometrytype(geom)='MULTIPOLYGON')) t1""" + query_result = build_snapshot_query(params) + assert query_result.encode("utf-8") == expected_query.encode("utf-8") + + +def test_convert_tags_to_postgres(): + from osm2pgsql_query_builder import convert_tags_to_postgres + + assert ( + convert_tags_to_postgres("tags['building'] = 'yes'") + == "tags->>'building' = 'yes'" + ) + + +def test_sanitize_filename(): + from osm2pgsql_query_builder import sanitize_filename + + assert sanitize_filename("my file-name:test") == "my_file_name_test" + + +def test_sanitize_column_name(): + from osm2pgsql_query_builder import sanitize_column_name + + assert sanitize_column_name("my tag:name") == "my_tag_name" + + +# --- Model tests --- + + +def test_snapshot_query_params_basic(): + params = SnapshotQueryParams( + geometry=Polygon( + type="Polygon", + coordinates=[ + [ + [84.0, 27.0], + [85.0, 27.0], + [85.0, 28.0], + [84.0, 28.0], + [84.0, 27.0], + ] + ], + ), + ) + assert params.output_type == "geojson" + assert params.use_st_within is True + assert params.centroid is False + assert params.filters is None + + +def test_snapshot_query_params_with_filters(): + params = SnapshotQueryParams( + geometry=Polygon( + type="Polygon", + coordinates=[ + [ + [84.0, 27.0], + [85.0, 27.0], + [85.0, 28.0], + [84.0, 28.0], + [84.0, 27.0], + ] + ], + ), + filters={ + "tags": {"all_geometry": {"join_or": {"building": []}}}, + "attributes": {"all_geometry": ["name"]}, + }, + ) + assert params.filters is not None + assert params.filters.tags.all_geometry.join_or == {"building": []} + assert params.filters.attributes.all_geometry == ["name"] + + +def test_snapshot_query_params_geometry_from_feature(): + params = SnapshotQueryParams( + geometry={ + "type": "Feature", + "properties": {}, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [84.0, 27.0], + [85.0, 27.0], + [85.0, 28.0], + [84.0, 28.0], + [84.0, 27.0], + ] + ], + }, + }, + ) + # GeometryValidatorMixin extracts geometry from Feature + assert params.geometry.type == "Polygon" + + +def test_snapshot_query_params_camel_case_alias(): + params = SnapshotQueryParams( + **{ + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [84.0, 27.0], + [85.0, 27.0], + [85.0, 28.0], + [84.0, 28.0], + [84.0, 27.0], + ] + ], + }, + "outputType": "shp", + "useStWithin": False, + "geometryType": ["point"], + } + ) + assert params.output_type == "shp" + assert params.use_st_within is False + assert params.geometry_type == ["point"] + + +def test_category_base_valid(): + cat = CategoryBase( + types=["points", "lines"], + select=["name", "highway"], + where="tags['highway'] IS NOT NULL", + ) + assert cat.types == ["points", "lines"] + assert cat.select == ["name", "highway"] + + +def test_category_base_invalid_type(): + with pytest.raises(ValidationError): + CategoryBase( + types=["invalid_type"], + select=["name"], + where="tags['building'] IS NOT NULL", + ) diff --git a/src/query_builder/builder.py b/src/query_builder/builder.py index 2df4fd13..cb6fd5b5 100644 --- a/src/query_builder/builder.py +++ b/src/query_builder/builder.py @@ -17,17 +17,27 @@ # 1100 13th Street NW Suite 800 Washington, D.C. 20005 # """Page Contains Query logic required for application""" -# Standard library imports -import re + from json import dumps, loads # Third party imports from geomet import wkt - -# Reader imports from src.config import USE_DUCK_DB_FOR_CUSTOM_EXPORTS -from src.config import logger as logging -from src.validation.models import SupportedFilters, SupportedGeometryFilters + +from osm2pgsql_query_builder import ( + build_column_select, + build_geom_filter, + build_geometry_type_query, + build_snapshot_query, + convert_tags_to_postgres, + sanitize_filename, +) + +# Legacy aliases used by the rest of the codebase. +# TODO Eventually swap callers to the new names. +raw_currentdata_extraction_query = build_snapshot_query +extract_geometry_type_query = build_geometry_type_query +format_file_name_str = sanitize_filename HDX_FILTER_CRITERIA = """ This theme includes all OpenStreetMap features in this area matching ( Learn what tags means [here](https://wiki.openstreetmap.org/wiki/Tags) ) : @@ -84,718 +94,11 @@ def check_exisiting_country(geom): return query -def get_query_as_geojson(query_list, ogr_export=None): - table_base_query = [] - if ogr_export: - table_base_query = query_list - else: - for i in range(len(query_list)): - table_base_query.append( - f"""select ST_AsGeoJSON(t{i}.*) from ({query_list[i]}) t{i}""" - ) - final_query = " UNION ALL ".join(table_base_query) - return final_query - - -def create_geom_filter(geom, geom_lookup_by="ST_intersects"): - """generates geometry intersection filter - Rawdata extraction""" - geometry_dump = dumps(loads(geom.model_dump_json())) - # return f"""{geom_lookup_by}(geom,ST_Buffer((select ST_Union(ST_makeValid(ST_GEOMFROMGEOJSON('{geometry_dump}')))),0.005))""" - return f"""{geom_lookup_by}(geom,(select ST_Union(ST_makeValid(ST_GEOMFROMGEOJSON('{geometry_dump}')))))""" - - -def format_file_name_str(input_str): - # Fixme I need to check every possible special character that can comeup on osm tags - input_str = re.sub("\s+", "_", input_str) # putting _ in every space # noqa - input_str = re.sub(":", "_", input_str) # putting _ in every : value - input_str = re.sub("-", "_", input_str) # putting _ in every - value - - return input_str - - -def remove_spaces(input_str): - # Fixme I need to check every possible special character that can comeup on osm tags - input_str = re.sub("\s+", "_", input_str) # putting _ in every space # noqa - input_str = re.sub(":", "_", input_str) # putting _ in every : value - return input_str - - -def create_column_filter( - columns, - create_schema=False, - output_type="geojson", - use_centroid=False, - include_osm_type=True, - include_user_metadata=False, -): - """generates column filter , which will be used to filter column in output will be used on select query - Rawdata extraction""" - - if len(columns) > 0: - filter_col = [] - filter_col.append("osm_id") - if include_osm_type: - filter_col.append("tableoid::regclass AS osm_type") - if include_user_metadata: - filter_col.extend(["uid", f""" "user" """, "timestamp"]) - if create_schema: - schema = {} - schema["osm_id"] = "int64" - schema["type"] = "str" - if include_user_metadata: - schema["uid"] = "int64" - schema["user"] = "str" - schema["timestamp"] = "str" - - if "*" in columns: - filter_col.append("tags") - if create_schema: - schema["tags"] = "str" - else: - for cl in columns: - splitted_cl = [cl] - if "," in cl: - splitted_cl = cl.split(",") - for cl in splitted_cl: - if cl != "": - filter_col.append( - f"""tags ->> '{cl.strip()}' as {remove_spaces(cl.strip())}""" - ) - if create_schema: - schema[remove_spaces(cl.strip())] = "str" - if output_type == "csv": # if it is csv geom logic is different - filter_col.append("ST_X(ST_Centroid(geom)) as longitude") - filter_col.append("ST_Y(ST_Centroid(geom)) as latitude") - filter_col.append("GeometryType(geom) as geom_type") - else: - filter_col.append("ST_Centroid(geom) as geom" if use_centroid else "geom") - select_condition = " , ".join(filter_col) - if create_schema: - return select_condition, schema - return select_condition - else: - if include_user_metadata: - return f"""osm_id, tableoid::regclass AS osm_type, tags,changeset, uid, "user", timestamp,{'ST_Centroid(geom) as geom' if use_centroid else 'geom'}""" - return f"osm_id, tableoid::regclass AS osm_type, tags,changeset,timestamp,{'ST_Centroid(geom) as geom' if use_centroid else 'geom'}" # this is default attribute that we will deliver to user if user defines his own attribute column then those will be appended with osm_id only - - -def create_tag_sql_logic(key, value, filter_list): - if len(value) > 1: - v_l = [] - for lil in value: - v_l.append(f""" '{lil.strip()}' """) - v_l_join = ", ".join(v_l) - value_tuple = f"""({v_l_join})""" - - k = f""" '{key.strip()}' """ - filter_list.append("""tags ->> """ + k + """IN """ + value_tuple + """""") - elif len(value) == 1: - filter_list.append(f"""tags ->> '{key.strip()}' = '{value[0].strip()}'""") - else: - filter_list.append(f"""tags ? '{key.strip()}'""") - return filter_list - - -def generate_tag_filter_query(filter, join_by=" OR ", plain_query_filter=False): - final_filter = [] - if plain_query_filter: - for item in filter: - key = item["key"] - value = item["value"] - if len(value) == 1 and value[0] == "*": - value = [] - if len(value) >= 1: - sub_append = [] - pre = """ tags @> '{""" - post = """ }'""" - for v in value: - sub_append.append( - f"""{pre} "{key.strip()}" : "{v.strip()}" {post}""" - ) - sub_append_join = " OR ".join(sub_append) - - final_filter.append(f"({sub_append_join})") - else: - final_filter.append(f"""tags ? '{key.strip()}'""") - tag_filter = join_by.join(final_filter) - return tag_filter - - else: - for key, value in filter.items(): - if key == "join_or": - temp_logic = [] - if value: - for k, v in value.items(): - temp_logic = create_tag_sql_logic(k, v, temp_logic) - final_filter.append(f"""{" OR ".join(temp_logic)}""") - - if key == "join_and": - temp_logic = [] - if value: - for k, v in value.items(): - temp_logic = create_tag_sql_logic(k, v, temp_logic) - if len(temp_logic) == 1: - join_by = " AND " - final_filter.append(f"""{" AND ".join(temp_logic)}""") - - tag_filter = join_by.join(final_filter) - return tag_filter - - -def extract_geometry_type_query( - params, - ogr_export=False, - g_id=None, - c_id=None, - country_export=False, -): - """used for specifically focused on export tool , this will generate separate queries for line point and polygon can be used on other datatype support - Rawdata extraction""" - include_user_metadata = params.include_user_metadata - geom_filter = create_geom_filter( - params.geometry, - "ST_within" if params.use_st_within is True else "ST_intersects", - ) - select_condition = f"""osm_id, tableoid::regclass AS osm_type, tags,changeset,timestamp , {'ST_Centroid(geom) as geom' if params.centroid else 'geom'}""" # this is default attribute that we will deliver to user if user defines his own attribute column then those will be appended with osm_id only - schema = { - "osm_id": "int64", - "type": "str", - "tags": "str", - "changeset": "int64", - "timestamp": "str", - } - query_point, query_line, query_poly = None, None, None - ( - attribute_filter, - master_attribute_filter, - master_tag_filter, - poly_attribute_filter, - poly_tag_filter, - ) = (None, None, None, None, None) - point_schema, line_schema, poly_schema = None, None, None - ( - tags, - attributes, - point_attribute_filter, - line_attribute_filter, - poly_attribute_filter, - master_attribute_filter, - point_tag_filter, - line_tag_filter, - poly_tag_filter, - master_tag_filter, - ) = (None, None, None, None, None, None, None, None, None, None) - if params.filters: - params.filters = ( - params.filters.model_dump() - ) # FIXME: temp fix , since validation model got changed - ( - tags, - attributes, - point_attribute_filter, - line_attribute_filter, - poly_attribute_filter, - master_attribute_filter, - point_tag_filter, - line_tag_filter, - poly_tag_filter, - master_tag_filter, - ) = extract_attributes_tags(params.filters) - - if ( - master_attribute_filter - ): # if no specific point , line or poly filter is not passed master columns filter will be used , if master columns is also empty then above default select statement will be used - select_condition, schema = create_column_filter( - use_centroid=params.centroid, - output_type=params.output_type, - columns=master_attribute_filter, - create_schema=True, - include_user_metadata=include_user_metadata, - ) - if master_tag_filter: - attribute_filter = generate_tag_filter_query(master_tag_filter) - if params.geometry_type is None: # fix me - params.geometry_type = ["point", "line", "polygon"] - - for type in params.geometry_type: - if type == SupportedGeometryFilters.POINT.value: - if point_attribute_filter: - select_condition, schema = create_column_filter( - use_centroid=params.centroid, - output_type=params.output_type, - columns=point_attribute_filter, - create_schema=True, - include_user_metadata=include_user_metadata, - ) - where_clause_for_nodes = generate_where_clause_indexes_case( - geom_filter, g_id, c_id, country_export, "nodes" - ) - - query_point = f"""select - {select_condition} - from - nodes - where - {where_clause_for_nodes}""" - if point_tag_filter: - attribute_filter = generate_tag_filter_query(point_tag_filter) - if attribute_filter: - query_point += f""" and ({attribute_filter})""" - point_schema = schema - - query_point = get_query_as_geojson([query_point], ogr_export=ogr_export) - - if type == SupportedGeometryFilters.LINE.value: - query_line_list = [] - if line_attribute_filter: - select_condition, schema = create_column_filter( - use_centroid=params.centroid, - output_type=params.output_type, - columns=line_attribute_filter, - create_schema=True, - include_user_metadata=include_user_metadata, - ) - where_clause_for_line = generate_where_clause_indexes_case( - geom_filter, g_id, c_id, country_export, "ways_line" - ) - - query_ways_line = f"""select - {select_condition} - from - ways_line - where - {where_clause_for_line}""" - where_clause_for_rel = generate_where_clause_indexes_case( - geom_filter, g_id, c_id, country_export, "relations" - ) - - query_relations_line = f"""select - {select_condition} - from - relations - where - {where_clause_for_rel}""" - if line_tag_filter: - attribute_filter = generate_tag_filter_query(line_tag_filter) - if attribute_filter: - query_ways_line += f""" and ({attribute_filter})""" - query_relations_line += f""" and ({attribute_filter})""" - query_relations_line += """ and (geometrytype(geom)='MULTILINESTRING')""" - query_line_list.append(query_ways_line) - query_line_list.append(query_relations_line) - query_line = get_query_as_geojson(query_line_list, ogr_export=ogr_export) - line_schema = schema - - if type == SupportedGeometryFilters.POLYGON.value: - query_poly_list = [] - if poly_attribute_filter: - select_condition, schema = create_column_filter( - use_centroid=params.centroid, - output_type=params.output_type, - columns=poly_attribute_filter, - create_schema=True, - include_user_metadata=include_user_metadata, - ) - - where_clause_for_poly = generate_where_clause_indexes_case( - geom_filter, g_id, c_id, country_export, "ways_poly" - ) - - query_ways_poly = f"""select - {select_condition} - from - ways_poly - where - {where_clause_for_poly}""" - where_clause_for_relations = generate_where_clause_indexes_case( - geom_filter, g_id, c_id, country_export, "relations" - ) - - query_relations_poly = f"""select - {select_condition} - from - relations - where - {where_clause_for_relations}""" - if poly_tag_filter: - attribute_filter = generate_tag_filter_query(poly_tag_filter) - if attribute_filter: - query_ways_poly += f""" and ({attribute_filter})""" - query_relations_poly += f""" and ({attribute_filter})""" - query_relations_poly += """ and (geometrytype(geom)='POLYGON' or geometrytype(geom)='MULTIPOLYGON')""" - query_poly_list.append(query_ways_poly) - query_poly_list.append(query_relations_poly) - query_poly = get_query_as_geojson(query_poly_list, ogr_export=ogr_export) - poly_schema = schema - return query_point, query_line, query_poly, point_schema, line_schema, poly_schema - - -def extract_attributes_tags(filters): - tags = None - attributes = None - point_tag_filter = None - poly_tag_filter = None - line_tag_filter = None - master_tag_filter = None - point_attribute_filter = None - poly_attribute_filter = None - line_attribute_filter = None - master_attribute_filter = None - if filters: - for key, value in filters.items(): - if key == SupportedFilters.TAGS.value: - if value: - tags = value - for k, v in value.items(): - if k == SupportedGeometryFilters.POINT.value: - point_tag_filter = v - if k == SupportedGeometryFilters.LINE.value: - line_tag_filter = v - if k == SupportedGeometryFilters.POLYGON.value: - poly_tag_filter = v - if k == SupportedGeometryFilters.ALLGEOM.value: - master_tag_filter = v - if key == SupportedFilters.ATTRIBUTES.value: - if value: - attributes = value - for k, v in value.items(): - if k == SupportedGeometryFilters.POINT.value: - point_attribute_filter = v - if k == SupportedGeometryFilters.LINE.value: - line_attribute_filter = v - if k == SupportedGeometryFilters.POLYGON.value: - poly_attribute_filter = v - if k == SupportedGeometryFilters.ALLGEOM.value: - master_attribute_filter = v - return ( - tags, - attributes, - point_attribute_filter, - line_attribute_filter, - poly_attribute_filter, - master_attribute_filter, - point_tag_filter, - line_tag_filter, - poly_tag_filter, - master_tag_filter, - ) - - -def generate_where_clause_indexes_case( - geom_filter, g_id, c_id, country_export, table_name="ways_poly" -): - where_clause = geom_filter - if g_id: - if ( - table_name == "ways_poly" - ): # currently grid index is only available for ways_poly - column_name = "grid" - grid_filter_base = [f"""{column_name} = {ind[0]}""" for ind in g_id] - grid_filter = " OR ".join(grid_filter_base) - where_clause = f"({grid_filter}) and ({geom_filter})" - if c_id: - c_id = ",".join(str(num) for num in c_id) - # if table_name == "ways_poly" or table_name == "nodes": - # where_clause += f" and (country IN ({c_id}))" - # else: - where_clause += f" and (country @> ARRAY[{c_id}])" - if ( - country_export - ): # ignore the geometry take geom from the db itself by using precalculated field - if c_id: - # if table_name == "ways_poly" or table_name == "nodes": - # where_clause = f"country IN ({c_id})" - # else: - where_clause = f"country @> ARRAY[{c_id}]" - return where_clause - - -def get_country_geojson(c_id): - query = f"SELECT ST_AsGeoJSON(geometry) as geom from countries where id={c_id}" - return query - - -def raw_currentdata_extraction_query( - params, - g_id=None, - c_id=None, - ogr_export=False, - country_export=False, -): - """Default function to support current snapshot extraction with all of the feature that export_tool_api has""" - include_user_metadata = params.include_user_metadata - geom_lookup_by = "ST_within" if params.use_st_within is True else "ST_intersects" - geom_filter = create_geom_filter(params.geometry, geom_lookup_by) - - base_query = [] - - ( - tags, - attributes, - point_attribute_filter, - line_attribute_filter, - poly_attribute_filter, - master_attribute_filter, - point_tag_filter, - line_tag_filter, - poly_tag_filter, - master_tag_filter, - ) = (None, None, None, None, None, None, None, None, None, None) - - point_select_condition = None - line_select_condition = None - poly_select_condition = None - - point_tag = None - line_tag = None - poly_tag = None - master_tag = None - use_geomtype_in_relation = True - - # query_table = [] - if params.include_user_metadata: - select_condition = f"""osm_id, tableoid::regclass AS osm_type, version,tags,changeset, uid, "user", timestamp,{'ST_Centroid(geom) as geom' if params.centroid else 'geom'}""" - else: - select_condition = f"""osm_id, tableoid::regclass AS osm_type, version,tags,changeset,timestamp,{'ST_Centroid(geom) as geom' if params.centroid else 'geom'}""" # this is default attribute that we will deliver to user if user defines his own attribute column then those will be appended with osm_id only - - point_select_condition = select_condition # initializing default - line_select_condition = select_condition - poly_select_condition = select_condition - - if params.filters: - params.filters = ( - params.filters.model_dump() - ) # FIXME: temp fix , since validation model got changed - ( - tags, - attributes, - point_attribute_filter, - line_attribute_filter, - poly_attribute_filter, - master_attribute_filter, - point_tag_filter, - line_tag_filter, - poly_tag_filter, - master_tag_filter, - ) = extract_attributes_tags(params.filters) - attribute_customization_full_support = ["geojson", "shp"] - - if params.output_type not in attribute_customization_full_support: - logging.debug( - "Merging filters since they don't have same no of filters for features" - ) - merged_array = [ - i if i else [] - for i in [ - point_attribute_filter, - line_attribute_filter, - poly_attribute_filter, - ] - ] - merged_result = list({x for l in merged_array for x in l}) - logging.debug(merged_result) - if point_attribute_filter: - point_attribute_filter = merged_result - if line_attribute_filter: - line_attribute_filter = merged_result - if poly_attribute_filter: - poly_attribute_filter = merged_result - - if attributes: - if master_attribute_filter: - if len(master_attribute_filter) > 0: - select_condition = create_column_filter( - use_centroid=params.centroid, - output_type=params.output_type, - columns=master_attribute_filter, - include_user_metadata=include_user_metadata, - ) - # if master attribute is supplied it will be applied to other geom type as well even though value is supplied they will be ignored - point_select_condition = select_condition - line_select_condition = select_condition - poly_select_condition = select_condition - else: - if point_attribute_filter: - if len(point_attribute_filter) > 0: - point_select_condition = create_column_filter( - use_centroid=params.centroid, - output_type=params.output_type, - columns=point_attribute_filter, - include_user_metadata=include_user_metadata, - ) - if line_attribute_filter: - if len(line_attribute_filter) > 0: - line_select_condition = create_column_filter( - use_centroid=params.centroid, - output_type=params.output_type, - columns=line_attribute_filter, - include_user_metadata=include_user_metadata, - ) - if poly_attribute_filter: - if len(poly_attribute_filter) > 0: - poly_select_condition = create_column_filter( - use_centroid=params.centroid, - output_type=params.output_type, - columns=poly_attribute_filter, - include_user_metadata=include_user_metadata, - ) - - if tags: - if ( - master_tag_filter - ): # if master tag is supplied then other tags should be ignored and master tag will be used - master_tag = generate_tag_filter_query(master_tag_filter) - point_tag = master_tag - line_tag = master_tag - poly_tag = master_tag - else: - if point_tag_filter: - point_tag = generate_tag_filter_query(point_tag_filter) - if line_tag_filter: - line_tag = generate_tag_filter_query(line_tag_filter) - if poly_tag_filter: - poly_tag = generate_tag_filter_query(poly_tag_filter) - - # condition for geometry types - - if params.geometry_type is None or len(params.geometry_type) == 0: - params.geometry_type = ["point", "line", "polygon"] - - if SupportedGeometryFilters.ALLGEOM.value in params.geometry_type: - params.geometry_type = ["point", "line", "polygon"] - if SupportedGeometryFilters.POINT.value in params.geometry_type: - where_clause_for_nodes = generate_where_clause_indexes_case( - geom_filter, g_id, c_id, country_export, "nodes" - ) - - query_point = f"""select - {point_select_condition} - from - nodes - where - {where_clause_for_nodes}""" - if point_tag: - query_point += f""" and ({point_tag})""" - base_query.append(query_point) - - if SupportedGeometryFilters.LINE.value in params.geometry_type: - where_clause_for_line = generate_where_clause_indexes_case( - geom_filter, g_id, c_id, country_export, "ways_line" - ) - - query_ways_line = f"""select - {line_select_condition} - from - ways_line - where - {where_clause_for_line}""" - if line_tag: - query_ways_line += f""" and ({line_tag})""" - base_query.append(query_ways_line) - - if SupportedGeometryFilters.POLYGON.value in params.geometry_type: - if poly_select_condition == line_select_condition and poly_tag == line_tag: - use_geomtype_in_relation = False - - if use_geomtype_in_relation: - where_clause_for_rel = generate_where_clause_indexes_case( - geom_filter, g_id, c_id, country_export, "relations" - ) - - query_relations_line = f"""select - {line_select_condition} - from - relations - where - {where_clause_for_rel}""" - if line_tag: - query_relations_line += f""" and ({line_tag})""" - query_relations_line += """ and (geometrytype(geom)='MULTILINESTRING')""" - base_query.append(query_relations_line) - - if SupportedGeometryFilters.POLYGON.value in params.geometry_type: - where_clause_for_poly = generate_where_clause_indexes_case( - geom_filter, g_id, c_id, country_export, "ways_poly" - ) - - query_ways_poly = f"""select - {poly_select_condition} - from - ways_poly - where - {where_clause_for_poly}""" - if poly_tag: - query_ways_poly += f""" and ({poly_tag})""" - base_query.append(query_ways_poly) - where_clause_for_relations = generate_where_clause_indexes_case( - geom_filter, g_id, c_id, country_export, "relations" - ) - query_relations_poly = f"""select - {poly_select_condition} - from - relations - where - {where_clause_for_relations}""" - if poly_tag: - query_relations_poly += f""" and ({poly_tag})""" - if use_geomtype_in_relation: - query_relations_poly += """ and (geometrytype(geom)='POLYGON' or geometrytype(geom)='MULTIPOLYGON')""" - base_query.append(query_relations_poly) - - if ogr_export: - # since query will be different for ogr exports and geojson exports because for ogr exports we don't need to grab each row in geojson - table_base_query = base_query - else: - table_base_query = [] - for i in range(len(base_query)): - table_base_query.append( - f"""select ST_AsGeoJSON(t{i}.*) from ({base_query[i]}) t{i}""" - ) - final_query = " UNION ALL ".join(table_base_query) - if params.output_type == "csv": - logging.debug(final_query) - - return final_query - - def check_last_updated_rawdata(): query = """select importdate as last_updated from planet_osm_replication_status""" return query -def raw_extract_plain_geojson(params, inspect_only=False): - geom_filter_cond = None - if params.geometry_type == "polygon": - geom_filter_cond = """ and (geometrytype(geom)='POLYGON' or geometrytype(geom)='MULTIPOLYGON')""" - select_condition = create_column_filter(columns=params.select) - where_condition = generate_tag_filter_query(params.where, params.join_by) - if params.bbox: - xmin, ymin, xmax, ymax = ( - params.bbox[0], - params.bbox[1], - params.bbox[2], - params.bbox[3], - ) - geom_condition = f"""ST_intersects(ST_MakeEnvelope({xmin}, {ymin}, {xmax}, {ymax},4326), geom)""" - - query_list = [] - for table_name in params.look_in: - sub_query = f"""select {select_condition} from {table_name} where ({where_condition}) """ - if params.bbox: - sub_query += f""" and {geom_condition}""" - if geom_filter_cond: - sub_query += geom_filter_cond - query_list.append(sub_query) - table_base_query = [] - for i in range(len(query_list)): - table_base_query.append( - f"""select ST_AsGeoJSON(t{i}.*) from ({query_list[i]}) t{i}""" - ) - final_query = " UNION ALL ".join(table_base_query) - if inspect_only: - final_query = f"""EXPLAIN - {final_query}""" - return final_query - - def get_countries_query(q): query = "Select ST_AsGeoJSON(cf.*) FROM countries cf" if q: @@ -813,19 +116,19 @@ def get_osm_feature_query(osm_id): "osm_id, tableoid::regclass AS osm_type, tags,changeset,timestamp,geom" ) query = f"""SELECT ST_AsGeoJSON(n.*) - FROM (select {select_condition} from nodes) n + FROM (select {select_condition} from nodes) n WHERE osm_id = {osm_id} UNION SELECT ST_AsGeoJSON(wl.*) - FROM (select {select_condition} from ways_line) wl + FROM (select {select_condition} from ways_line) wl WHERE osm_id = {osm_id} UNION SELECT ST_AsGeoJSON(wp.*) - FROM (select {select_condition} from ways_poly) wp + FROM (select {select_condition} from ways_poly) wp WHERE osm_id = {osm_id} UNION SELECT ST_AsGeoJSON(r.*) - FROM (select {select_condition} from relations) r + FROM (select {select_condition} from relations) r WHERE osm_id = {osm_id}""" return query @@ -893,16 +196,6 @@ def get_country_from_iso(iso3): return query -def convert_tags_pattern_to_postgres(query_string): - pattern = r"tags\['(.*?)'\]" - - converted_query = re.sub( - pattern, lambda match: f"tags->>'{match.group(1)}'", query_string - ) - - return converted_query - - def postgres2duckdb_query( base_table_name, table, @@ -941,7 +234,7 @@ def postgres2duckdb_query( postgres_query = f"""select {select_query} from (select * , tableoid::regclass as osm_type from {table} where {row_filter_condition}) as sub_query""" if single_category_where: postgres_query += ( - f" where {convert_tags_pattern_to_postgres(single_category_where)}" + f" where {convert_tags_to_postgres(single_category_where)}" ) duck_db_create = f"""CREATE TABLE {base_table_name}_{table} AS SELECT {create_select_duck_db} FROM postgres_query("postgres_db", "{postgres_query}") """ @@ -955,11 +248,11 @@ def extract_custom_features_from_postgres( """ Generates Postgresql query for custom feature extraction """ - geom_filter = f"""(country @> ARRAY [{cid}])""" if cid else create_geom_filter(geom) + geom_filter = f"""(country @> ARRAY [{cid}])""" if cid else build_geom_filter(geom) postgres_query = f"""select {select_q} from (select * , tableoid::regclass as osm_type from {from_q} where {geom_filter}) as sub_query""" if where_q: - postgres_query += f" where {convert_tags_pattern_to_postgres(where_q)}" + postgres_query += f" where {convert_tags_to_postgres(where_q)}" return postgres_query @@ -1003,7 +296,7 @@ def extract_features_custom_exports( select += ["osm_id", "osm_type", "geom"] select_query = ", ".join(select) else: - select_query = create_column_filter(select, include_osm_type=False) + select_query = build_column_select(select, include_osm_type=False) from_query = map_tables[feature_type]["table"] @@ -1026,6 +319,11 @@ def extract_features_custom_exports( return " UNION ALL ".join(base_query) +def get_country_geojson(c_id): + query = f"SELECT ST_AsGeoJSON(geometry) as geom from countries where id={c_id}" + return query + + def get_country_geom_from_iso(iso3): """ Generate a SQL query to retrieve country geometry based on ISO3 code. diff --git a/src/validation/models.py b/src/validation/models.py index e43b6c75..23aaaac9 100644 --- a/src/validation/models.py +++ b/src/validation/models.py @@ -23,8 +23,15 @@ # Third party imports from geojson_pydantic import Feature, FeatureCollection, MultiPolygon, Polygon -from pydantic import BaseModel as PydanticModel -from pydantic import Field, validator +from pydantic import ConfigDict, Field, ValidationInfo, field_validator, model_validator + +# Shared models from osm2pgsql-query-builder +from osm2pgsql_query_builder import ( + BaseModel, + CategoryBase, + GeometryValidatorMixin, + SnapshotQueryParams, +) # Reader imports from src.config import ( @@ -39,18 +46,11 @@ from src.config import ALLOWED_HDX_TAGS, ALLOWED_HDX_UPDATE_FREQUENCIES -def to_camel(string: str) -> str: - split_string = string.split("_") - - return "".join([split_string[0], *[w.capitalize() for w in split_string[1:]]]) - - -class BaseModel(PydanticModel): - class Config: - alias_generator = to_camel - populate_by_name = True - use_enum_values = True - # extra = "forbid" +def _validate_geometry_or_iso3(geometry, iso3): + if geometry is not None and iso3 is not None: + raise ValueError("Only one of geometry or iso3 should be supplied.") + if geometry is None and iso3 is None: + raise ValueError("Either geometry or iso3 should be supplied.") class RawDataOutputType(Enum): @@ -68,113 +68,15 @@ class RawDataOutputType(Enum): MVT = "mvt" ## Experimental -class SupportedFilters(Enum): - TAGS = "tags" - ATTRIBUTES = "attributes" - - @classmethod - def has_value(cls, value): - """Checks value""" - return value in cls._value2member_map_ - - -class SupportedGeometryFilters(Enum): - POINT = "point" - LINE = "line" - POLYGON = "polygon" - ALLGEOM = "all_geometry" - - @classmethod - def has_value(cls, value): - """Checks if the value is supported""" - return value in cls._value2member_map_ - - -class JoinFilterType(Enum): - OR = "OR" - AND = "AND" - - -class SQLFilter(BaseModel): - join_or: Optional[Dict[str, List[str]]] = Field(default=None) - join_and: Optional[Dict[str, List[str]]] = Field(default=None) - - -class TagsFilter(BaseModel): - point: Optional[SQLFilter] = Field(default=None) - line: Optional[SQLFilter] = Field(default=None) - polygon: Optional[SQLFilter] = Field(default=None) - all_geometry: Optional[SQLFilter] = Field(default=None) - - -class AttributeFilter(BaseModel): - point: Optional[List[str]] = Field(default=None) - line: Optional[List[str]] = Field(default=None) - polygon: Optional[List[str]] = Field(default=None) - all_geometry: Optional[List[str]] = Field(default=None) - - -class Filters(BaseModel): - tags: Optional[TagsFilter] = Field(default=None) - attributes: Optional[AttributeFilter] = Field(default=None) - - -class GeometryValidatorMixin: - @validator("geometry") - def validate_geometry(cls, value): - """Validates geometry""" - if value: - if value.type == "Feature": - if value.geometry.type not in ["Polygon", "MultiPolygon"]: - raise ValueError( - f"Feature geometry type {value.geometry.type} must be of type polygon/multipolygon", - ) - return value.geometry - if value.type == "FeatureCollection": - for feature in value.features: - if feature.geometry.type not in ["Polygon", "MultiPolygon"]: - raise ValueError( - f"Feature Collection can't have {feature.type} , should be polygon/multipolygon" - ) - if len(value.features) > 1: - raise ValueError( - "Feature collection with multiple features is not supported yet" - ) - return value.features[0].geometry - return value - - -class RawDataCurrentParamsBase(BaseModel, GeometryValidatorMixin): +class RawDataCurrentParamsBase(SnapshotQueryParams): output_type: Optional[RawDataOutputType] = Field( default=RawDataOutputType.GEOJSON.value, example="geojson" ) - geometry_type: Optional[List[SupportedGeometryFilters]] = Field( - default=None, example=["point", "polygon"] - ) - centroid: Optional[bool] = Field( - default=False, description="Exports centroid of features as geom" - ) - use_st_within: Optional[bool] = Field( - default=True, - description="Exports features which are exactly inside the passed polygons (ST_WITHIN) By default features which are intersected with passed polygon is exported", - ) - include_user_metadata: Optional[bool] = Field( - default=False, - description="Include user metadata on exports , Only available to logged in users", - ) if ENABLE_POLYGON_STATISTICS_ENDPOINTS: include_stats: Optional[bool] = Field( default=False, description="Includes detailed stats about the polygon passed such as buildings count , road count along with summary about data completeness in the area", ) - filters: Optional[Filters] = Field( - default=None, - example={ - "tags": {"all_geometry": {"join_or": {"building": []}}}, - "attributes": {"all_geometry": ["name"]}, - }, - description="Filter for point,line,polygon/ all geometry for both select and where clause, All geometry filter means : It will apply the same filter to all the geometry type", - ) include_stats_html: Optional[bool] = Field( default=False, description="Includes detailed stats about the polygon passed such as buildings count , road count along with summary about data completeness in the area", @@ -183,32 +85,6 @@ class RawDataCurrentParamsBase(BaseModel, GeometryValidatorMixin): default=False, description="Includes transliterations", ) - geometry: Union[ - Polygon, - MultiPolygon, - Feature, - FeatureCollection, - ] = Field( - example={ - "type": "Polygon", - "coordinates": [ - [ - [83.96919250488281, 28.194446860487773], - [83.99751663208006, 28.194446860487773], - [83.99751663208006, 28.214869548073377], - [83.96919250488281, 28.214869548073377], - [83.96919250488281, 28.194446860487773], - ] - ], - }, - ) - - @validator("geometry_type", allow_reuse=True) - def return_unique_value(cls, value): - """return unique list""" - if value: - return list(set(value)) - return value class RawDataCurrentParams(RawDataCurrentParamsBase): @@ -232,20 +108,22 @@ class RawDataCurrentParams(RawDataCurrentParamsBase): if ALLOW_BIND_ZIP_FILTER: bind_zip: Optional[bool] = True - @validator("bind_zip", allow_reuse=True) - def check_bind_option(cls, value, values): + @field_validator("bind_zip") + @classmethod + def check_bind_option(cls, value, info: ValidationInfo): """Checks if cloud optimized output format or geoJSON is selected along with bind to zip file""" if value is False: - if values.get("output_type") not in ( - ( - [ - RawDataOutputType.GEOJSON.value, - RawDataOutputType.FLATGEOBUF.value, - RawDataOutputType.GEOPARQUET.value, - ] - + ([RawDataOutputType.PMTILES.value] if ENABLE_TILES else []) - ) - ): + output_type = info.data.get("output_type") + if output_type not in [ + RawDataOutputType.GEOJSON.value, + RawDataOutputType.FLATGEOBUF.value, + RawDataOutputType.GEOPARQUET.value, + *( + [RawDataOutputType.PMTILES.value] + if ENABLE_TILES + else [] + ), + ]: raise ValueError( "Only Cloud Optimized format and GeoJSON is supported for streaming" ) @@ -256,13 +134,14 @@ class SnapshotResponse(BaseModel): task_id: str track_link: str - class Config: - json_schema_extra = { + model_config = ConfigDict( + json_schema_extra={ "example": { "task_id": "aa539af6-83d4-4aa3-879e-abf14fffa03f", "track_link": "/tasks/status/aa539af6-83d4-4aa3-879e-abf14fffa03f/", } } + ) class SnapshotTaskResult(BaseModel): @@ -279,8 +158,8 @@ class SnapshotTaskResponse(BaseModel): status: str result: SnapshotTaskResult - class Config: - json_schema_extra = { + model_config = ConfigDict( + json_schema_extra={ "example": { "id": "3fded368-456f-4ef4-a1b8-c099a7f77ca4", "status": "SUCCESS", @@ -294,13 +173,15 @@ class Config: }, } } + ) class StatusResponse(BaseModel): last_updated: str - class Config: - json_schema_extra = {"example": {"lastUpdated": "2022-06-27 19:59:24+05:45"}} + model_config = ConfigDict( + json_schema_extra={"example": {"lastUpdated": "2022-06-27 19:59:24+05:45"}} + ) class StatsRequestParams(BaseModel, GeometryValidatorMixin): @@ -329,14 +210,10 @@ class StatsRequestParams(BaseModel, GeometryValidatorMixin): }, ) - @validator("geometry", pre=True, always=True) - def set_geometry_or_iso3(cls, value, values): - """Either geometry or iso3 should be supplied.""" - if value is not None and values.get("iso3") is not None: - raise ValueError("Only one of geometry or iso3 should be supplied.") - if value is None and values.get("iso3") is None: - raise ValueError("Either geometry or iso3 should be supplied.") - return value + @model_validator(mode="after") + def validate_geometry_or_iso3_required(self): + _validate_geometry_or_iso3(self.geometry, self.iso3) + return self ### HDX BLock @@ -368,7 +245,8 @@ class HDXModel(BaseModel): example="Sample notes to append", ) - @validator("tags") + @field_validator("tags") + @classmethod def validate_tags(cls, value): """Validates tags if they are allowed from hdx allowed approved tags @@ -391,7 +269,7 @@ def validate_tags(cls, value): return value -class CategoryModel(BaseModel): +class CategoryModel(CategoryBase): """ Model for category configuration settings. @@ -406,49 +284,14 @@ class CategoryModel(BaseModel): hdx: Optional[HDXModel] = Field( default=None, description="HDX Specific configurations" ) - types: List[str] = Field( - ..., - description="List of feature types (points, lines, polygons).", - example=["lines"], - ) - select: List[str] = Field( - ..., - description="List of selected fields.", - example=["name", "highway"], - ) - where: str = Field( - ..., - description="SQL-like condition to filter features.", - example="highway IS NOT NULL", - ) formats: List[str] = Field( ..., description="List of Export Formats (suffixes).", example=["gpkg", "geojson"], ) - @validator("types") - def validate_types(cls, value): - """validates geom types - - Args: - value (_type_): _description_ - - Raises: - ValueError: _description_ - - Returns: - _type_: _description_ - """ - allowed_types = {"points", "lines", "polygons"} - for item in value: - if item not in allowed_types: - raise ValueError( - f"Invalid type: {item}. Allowed types are {', '.join(allowed_types)}" - ) - return value - - @validator("formats") + @field_validator("formats") + @classmethod def validate_export_types(cls, value): """Validates export types if they are supported @@ -554,7 +397,8 @@ class DatasetConfig(BaseModel): example="[{'url': 'https://something.org/datasetviz.html'}]", ) - @validator("update_frequency") + @field_validator("update_frequency") + @classmethod def validate_frequency(cls, value): """Validates frequency @@ -670,29 +514,31 @@ class DynamicCategoriesModel(CategoriesBase, GeometryValidatorMixin): }, ) - @validator("geometry", pre=True, always=True) - def set_geometry_or_iso3(cls, value, values): + @field_validator("geometry") + @classmethod + def set_geometry_or_iso3(cls, value, info: ValidationInfo): """Either geometry or iso3 should be supplied.""" - if value is not None and values.get("iso3") is not None: - raise ValueError("Only one of geometry or iso3 should be supplied.") - if value is None and values.get("iso3") is None: - raise ValueError("Either geometry or iso3 should be supplied.") if value is not None: - dataset = values.get("dataset") - if values.get("hdx_upload"): - for category in values.get("categories"): - category_name, category_data = list(category.items())[0] + dataset = info.data.get("dataset") + if info.data.get("hdx_upload"): + for category in info.data.get("categories"): + _, category_data = next(iter(category.items())) if category_data.hdx is None: raise ValueError(f"HDX is missing for category {category}") - if dataset is None and values.get("hdx_upload"): + if dataset is None and info.data.get("hdx_upload"): raise ValueError("Dataset config should be supplied for custom polygon") - if values.get("hdx_upload"): + if info.data.get("hdx_upload"): for item in dataset: if item is None: raise ValueError(f"Missing, Dataset config : {item}") return value + @model_validator(mode="after") + def validate_geometry_or_iso3_required(self): + _validate_geometry_or_iso3(self.geometry, self.iso3) + return self + class CustomRequestsYaml(CategoriesBase): geometry: Union[Polygon, MultiPolygon, Feature, FeatureCollection] = Field(