diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..1d36346 --- /dev/null +++ b/.flake8 @@ -0,0 +1,2 @@ +[flake8] +max-line-length = 88 \ No newline at end of file diff --git a/.gitignore b/.gitignore index 44e36a3..8fd3351 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,7 @@ __pycache__ *.pyc *.egg-info +# scikit-build artifacts +_skbuild/ +python/upsp/_version.py + diff --git a/CMakeLists.txt b/CMakeLists.txt index a6cfa7e..fce7719 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,20 +2,50 @@ cmake_minimum_required(VERSION 3.20) project(upsp) cmake_policy(SET CMP0074 NEW) -list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/Modules") - -set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib") -set(CMAKE_INSTALL_RPATH_USE_LINK_PATH True) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_POSITION_INDEPENDENT_CODE ON) -# Direct inclusion of open-source tools for -# integrating pandoc w/ CMake, copied into cmake/Modules. Ref: -# https://github.com/jeetsukumaran/cmake-pandocology/ -# (SHA: 10900f9aec4431b504fa8979576f950533cf20d9) -include(pandocology) +list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/Modules") +# This ensures any link paths used at build time are preserved in +# the installed targets. This supports cases where the user has +# supplied third-party libraries in non-system locations (for +# instance, if making use of a local vcpkg setup) +set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) +set(CMAKE_SKIP_BUILD_RPATH FALSE) + +if (DEFINED SKBUILD) + # prevent an unused variable warning + set(ignoreMe "${SKBUILD}") + + # TODO this is ugly but it works for now. + # + # When the project is built by scikit-build for deployment as a + # Python package, libraries are dropped in the lib/ folder and there's + # multiple consumers of those libraries that need to resolve them at runtime: + # + # - Executables, installed to bin/, need to look in $ORIGIN/../lib + # - Python extension modules, installed to lib/pythonX/site-packages/upsp, + # need to look in $ORIGIN/../../.. + # + # Lastly, there's some build-tree-only artifacts (eg gtest unit tests), + # those need to look in their own build directory ($ORIGIN). + # + # We could do this on a per-target basis but I'd rather just leave + # the hack in one spot for now instead of being peppered all over + # this build file. + # + # A "better/cleaner" example of this setup can be found here (although + # I don't think it handles all the use cases I listed above, it's + # a small sample project): + # + # https://github.com/scikit-build/scikit-build-sample-projects/blob/master/projects/hello-cmake-package/CMakeLists.txt#L92 + # + set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) + set(CMAKE_INSTALL_RPATH "\$ORIGIN:\$ORIGIN/../lib:\$ORIGIN/../../../") +endif() + find_package(Eigen3 CONFIG REQUIRED) find_package(Boost REQUIRED) find_package(OpenCV CONFIG REQUIRED COMPONENTS core imgproc imgcodecs calib3d videoio tracking) @@ -26,14 +56,16 @@ find_package(pybind11 REQUIRED) find_library(LIBRT rt) find_package(hdf5 CONFIG REQUIRED) find_package(IlmBase CONFIG REQUIRED) +find_package(PythonExtensions) + +include_directories(cpp/include) pybind11_add_module(cine cpp/pybind11/cine.cpp) target_link_libraries(cine PRIVATE upsp_video) + pybind11_add_module(raycast cpp/pybind11/raycast.cpp) target_link_libraries(raycast PRIVATE upsp_kdtree) -include_directories(cpp/include) - add_library( upsp_video SHARED @@ -121,9 +153,6 @@ add_executable(upsp_matrix_transpose cpp/exec/upsp_matrix_transpose.cpp) target_link_libraries(upsp_matrix_transpose MPI::MPI_CXX) target_link_libraries(upsp_matrix_transpose OpenMP::OpenMP_CXX) - -# GTest::gmock GTest::gtest GTest::gmock_main GTest::gtest_main -# TODO could automate smoke tests and gtest unit tests leveraging CTest find_package(GTest CONFIG REQUIRED) upsp_add_executable( run_tests @@ -143,91 +172,6 @@ target_link_libraries(run_tests GTest::gtest GTest::gtest_main) target_link_libraries(run_tests OpenMP::OpenMP_CXX) target_link_libraries(run_tests hdf5::hdf5_cpp-shared hdf5::hdf5_hl_cpp-shared) -# Documentation (custom cmake macros from pandocology) -# NOTE: pandoc options ref: https://pandoc.org/MANUAL.html -string(TIMESTAMP TODAY "%Y-%m-%d") -function(upsp_add_document TARGET FILENAME) - list(APPEND UPSP_ADD_DOCUMENT_TARGETS ${TARGET}) - set(UPSP_ADD_DOCUMENT_TARGETS ${UPSP_ADD_DOCUMENT_TARGETS} PARENT_SCOPE) - get_filename_component(BASENAME_WE ${FILENAME} NAME_WE) - add_document( - TARGET ${TARGET}_docx - OUTPUT_FILE ${BASENAME_WE}.docx - SOURCES ${FILENAME} - RESOURCE_DIRS docs/md/static - PANDOC_DIRECTIVES --to docx - --from markdown+pandoc_title_block+table_captions+simple_tables+yaml_metadata_block - --filter pandoc-xnos - --mathjax - --standalone - --toc - --number-sections - --metadata date=${TODAY} - NO_EXPORT_PRODUCT - ) - - add_document( - TARGET ${TARGET}_html - OUTPUT_FILE ${BASENAME_WE}.html - SOURCES ${FILENAME} - RESOURCE_DIRS docs/md/static - PANDOC_DIRECTIVES --to html - --from markdown+pandoc_title_block+table_captions+simple_tables+yaml_metadata_block - --filter pandoc-xnos - --mathjax - --standalone - --toc - --number-sections - --metadata date=${TODAY} - -c static/upsp-styles.css - -A static/upsp-footer.html - NO_EXPORT_PRODUCT - ) - - add_dependencies(${TARGET}_html ${TARGET}_docx) -endfunction() - -function(upsp_serialize_document_depends) - list(LENGTH UPSP_ADD_DOCUMENT_TARGETS NUMBER_TARGETS) - if (${NUMBER_TARGETS} LESS_EQUAL 1) - return() - endif() - math(EXPR START "1") - math(EXPR STOP "${NUMBER_TARGETS} - 1") - foreach(THIS_IDX RANGE ${START} ${STOP}) - math(EXPR PREV_IDX "${THIS_IDX} - 1") - list(GET UPSP_ADD_DOCUMENT_TARGETS ${THIS_IDX} THIS_TGT) - list(GET UPSP_ADD_DOCUMENT_TARGETS ${PREV_IDX} PREV_TGT) - add_dependencies(${THIS_TGT}_docx ${PREV_TGT}_html) - endforeach() -endfunction() - -upsp_add_document(upsp_user_manual docs/md/upsp-user-manual.md) -upsp_add_document(upsp_swdd docs/md/upsp-swdd.md) -upsp_add_document(upsp_third_party_dependencies docs/md/upsp-third-party-dependencies.md) -upsp_serialize_document_depends() - -# INSTALLATION TARGETS - -# Documentation install targets -# A bit hacky but it works. Grab outputs from the build directory -# after pandoc has been run (pandoc configured by the various -# cmake pandocology macros) and copy them "by hand" to install location. -install( - FILES - ${CMAKE_CURRENT_BINARY_DIR}/upsp-swdd.docx - ${CMAKE_CURRENT_BINARY_DIR}/upsp-swdd.html - ${CMAKE_CURRENT_BINARY_DIR}/upsp-third-party-dependencies.docx - ${CMAKE_CURRENT_BINARY_DIR}/upsp-third-party-dependencies.html - ${CMAKE_CURRENT_BINARY_DIR}/upsp-user-manual.docx - ${CMAKE_CURRENT_BINARY_DIR}/upsp-user-manual.html - DESTINATION docs -) -install( - DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/static - DESTINATION docs -) - install( TARGETS add_field @@ -248,120 +192,20 @@ install( scripts/upsp-make-processing-tree scripts/upsp-kulite-comparison scripts/upsp-plotting - scripts/upsp-unity-export + scripts/upsp-qsub-args DESTINATION bin ) -# Python install process could be a bit cleaner. -# - Ideally we would make use of setuputils. -# - However, there are a couple modules built with -# pybind11 (e.g., raycast.so) that rely on other -# solibs built by the project. If a user wants -# to import them in Python, then either -# a) raycast.so has to have its RPATH populated with -# the path to the folder containing the solib -# dependencies, or -# b) the user has to have their LD_LIBRARY_PATH set. -# Clearly, option (a) is preferred for Python -# development, and option (b) is an annoying -# workaround. -# - By using the CMake install() command on the pybind11 -# CMake targets, their RPATH's are updated correctly -# so option (a) works. However, this precludes using -# setuputils. -# - There is probably a way to configure CMake via -# setuputils calls to properly replicate the behavior -# but I haven't figured it out yet. -install(TARGETS cine raycast LIBRARY DESTINATION python/upsp) - -# The rest of the non-pybind11 Python modules are then -# just manually copied here. We explicitly list out all files -# for safety (in general, globs in build scripts aren't best -# practice). +# TODO I think this works in a fairly brittle fashion right now. +# It just so happens that the cmake_install_dir staging directory +# created by scikit-build has a "python/upsp" folder mimicking the +# source tree layout... so these extensions are just "dropped" into the +# right spot. There's probably a more robust way to specify their +# install location, probably some CMAKE_ variable I'm ignorant of. install( - FILES - python/upsp/__init__.py - DESTINATION - python/upsp -) - -install( - FILES - python/upsp/processing/context.py - python/upsp/processing/grids.py - python/upsp/processing/io.py - python/upsp/processing/kulite_processing.py - python/upsp/processing/kulite_utilities.py - python/upsp/processing/p3d_conversions.py - python/upsp/processing/p3d_utilities.py - python/upsp/processing/plot3d.py - python/upsp/processing/tree.py - python/upsp/processing/unity_conversions.py - DESTINATION - python/upsp/processing -) - -install( - FILES - python/upsp/cam_cal_utils/external_calibrate.py - python/upsp/cam_cal_utils/img_utils.py - python/upsp/cam_cal_utils/parsers.py - python/upsp/cam_cal_utils/photogrammetry.py - python/upsp/cam_cal_utils/target_bumping.py - python/upsp/cam_cal_utils/visibility.py - python/upsp/cam_cal_utils/visualization.py - DESTINATION - python/upsp/cam_cal_utils -) - -install( - FILES - python/upsp/target_localization/blob_detector_methods.py - python/upsp/target_localization/gaussian_fitting_methods.py - DESTINATION - python/upsp/target_localization -) - -install( - FILES - python/upsp/kulite_comparison/plotting.py - python/upsp/kulite_comparison/selection.py - python/upsp/kulite_comparison/spatial_queries.py - DESTINATION - python/upsp/kulite_comparison -) - -install( - FILES - python/upsp/processing/templates/add-field.sh.template - python/upsp/processing/templates/gltf-viewer.html.template - python/upsp/processing/templates/launcher.sh.template - python/upsp/processing/templates/run-step-parallel.sh.template - python/upsp/processing/templates/run-step-serial.sh.template - DESTINATION - python/upsp/processing/templates -) - -# Define the two required variables before including -# the source code for watching a git repository. -set(PRE_CONFIGURE_FILE "scripts/version") -set(POST_CONFIGURE_FILE "${CMAKE_BINARY_DIR}/version") -include(cmake/git_watcher.cmake) - -install( - PROGRAMS - ${CMAKE_BINARY_DIR}/version - DESTINATION - ${CMAKE_INSTALL_PREFIX} -) - -install( - FILES - scripts/activate.sh - scripts/activate.csh - RELEASE.md - DESTINATION - ${CMAKE_INSTALL_PREFIX} + TARGETS + cine + raycast + LIBRARY DESTINATION python/upsp ) - diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..2465e47 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,95 @@ +NASA OPEN SOURCE AGREEMENT VERSION 1.3 + +THIS OPEN SOURCE AGREEMENT (“AGREEMENT”) DEFINES THE RIGHTS OF USE, REPRODUCTION, DISTRIBUTION, MODIFICATION AND REDISTRIBUTION OF CERTAIN COMPUTER SOFTWARE ORIGINALLY RELEASED BY THE UNITED STATES GOVERNMENT AS REPRESENTED BY THE GOVERNMENT AGENCY LISTED BELOW ("GOVERNMENT AGENCY"). +THE UNITED STATES GOVERNMENT, AS REPRESENTED BY GOVERNMENT AGENCY, IS AN INTENDED THIRD-PARTY BENEFICIARY OF ALL SUBSEQUENT DISTRIBUTIONS OR REDISTRIBUTIONS OF THE SUBJECT SOFTWARE. ANYONE WHO USES, REPRODUCES, DISTRIBUTES, MODIFIES OR REDISTRIBUTES THE SUBJECT SOFTWARE, AS DEFINED HEREIN, OR ANY PART THEREOF, IS, BY THAT ACTION, ACCEPTING IN FULL THE RESPONSIBILITIES AND OBLIGATIONS CONTAINED IN THIS AGREEMENT. + + +1. DEFINITIONS + +A. “Contributor” means Government Agency, as the developer of the Original Software, and any entity that makes a Modification. +B. “Covered Patents” mean patent claims licensable by a Contributor that are necessarily infringed by the use or sale of its Modification alone or when combined with the Subject Software. +C. “Display” means the showing of a copy of the Subject Software, either directly or by means of an image, or any other device. +D. “Distribution” means conveyance or transfer of the Subject Software, regardless of means, to another. +E. “Larger Work” means computer software that combines Subject Software, or portions thereof, with software separate from the Subject Software that is not governed by the terms of this Agreement. +F. “Modification” means any alteration of, including addition to or deletion from, the substance or structure of either the Original Software or Subject Software, and includes derivative works, as that term is defined in the Copyright Statute, 17 USC 101. However, the act of including Subject Software as part of a Larger Work does not in and of itself constitute a Modification. +G. “Original Software” means the computer software first released under this Agreement by Government Agency with Government Agency designation NASA Ames Research Center and entitled Support Libraries for Cart3D I/O Functions and Extensible Design Description Markup +including source code, object code and accompanying documentation, if any. +H. “Recipient” means anyone who acquires the Subject Software under this Agreement, including all Contributors. +I. “Redistribution” means Distribution of the Subject Software after a Modification has been made. +J. “Reproduction” means the making of a counterpart, image or copy of the Subject Software. +K. “Sale” means the exchange of the Subject Software for money or equivalent value. +L. “Subject Software” means the Original Software, Modifications, or any respective parts thereof. +M. “Use” means the application or employment of the Subject Software for any purpose. + +2. GRANT OF RIGHTS + +A. Under Non-Patent Rights: Subject to the terms and conditions of this Agreement, each Contributor, with respect to its own contribution to the Subject Software, hereby grants to each Recipient a non-exclusive, world-wide, royalty-free license to engage in the following activities pertaining to the Subject Software: + +1. Use +2. Distribution +3. Reproduction +4. Modification +5. Redistribution +6. Display + +B. Under Patent Rights: Subject to the terms and conditions of this Agreement, each Contributor, with respect to its own contribution to the Subject Software, hereby grants to each Recipient under Covered Patents a non-exclusive, world-wide, royalty-free license to engage in the following activities pertaining to the Subject Software: + +1. Use +2. Distribution +3. Reproduction +4. Sale +5. Offer for Sale + +C. The rights granted under Paragraph B. also apply to the combination of a Contributor’s Modification and the Subject Software if, at the time the Modification is added by the Contributor, the addition of such Modification causes the combination to be covered by the Covered +Patents. It does not apply to any other combinations that include a Modification. + +D. The rights granted in Paragraphs A. and B. allow the Recipient to sublicense those same rights. Such sublicense must be under the same terms and conditions of this Agreement. + +3. OBLIGATIONS OF RECIPIENT + +A. Distribution or Redistribution of the Subject Software must be made under this Agreement except for additions covered under paragraph 3H. + +1. Whenever a Recipient distributes or redistributes the Subject Software, a copy of this Agreement must be included with each copy of the Subject Software; and +2. If Recipient distributes or redistributes the Subject Software in any form other than source code, Recipient must also make the source code freely available, and must provide with each copy of the Subject Software information on how to obtain the source code in a reasonable manner on or through a medium customarily used for software exchange. + +B. Each Recipient must ensure that the following copyright notice appears prominently in the Subject Software: + +Copyright © 2022 United States Government as represented by the Administrator of the National Aeronautics and Space Administration. +All Rights Reserved. + +C. Each Contributor must characterize its alteration of the Subject Software as a Modification and must identify itself as the originator of its Modification in a manner that reasonably allows subsequent Recipients to identify the originator of the Modification. In fulfillment of these requirements, Contributor must include a file (e.g., a change log file) that describes the alterations made and the date of the alterations, identifies Contributor as originator of the alterations, and consents to characterization of the alterations as a Modification, for example, by including a statement that the Modification is derived, directly or indirectly, from Original Software provided by Government Agency. Once consent is granted, it may not thereafter be revoked. + +D. A Contributor may add its own copyright notice to the Subject Software. Once a copyright notice has been added to the Subject Software, a Recipient may not remove it without the express permission of the Contributor who added the notice. + +E. A Recipient may not make any representation in the Subject Software or in any promotional, advertising or other material that may be construed as an endorsement by Government Agency or by any prior Recipient of any product or service provided by Recipient, or that may seek to obtain commercial advantage by the fact of Government Agency's or a prior Recipient’s participation in this Agreement. + +F. In an effort to track usage and maintain accurate records of the Subject Software, each Recipient, upon receipt of the Subject Software, is requested to provide Government Agency, by e-mail to the Government Agency Point of Contact listed in clause 5.F., the following information: name and email. Recipient’s name and personal information shall be used for statistical purposes only. Once a Recipient makes a Modification available, it is requested that the Recipient inform Government Agency, by e-mail to the Government Agency Point of Contact listed in clause 5.F., how to access the Modification. + +G. Each Contributor represents that that its Modification is believed to be Contributor’s original creation and does not violate any existing agreements, regulations, statutes or rules, and further that Contributor has sufficient rights to grant the rights conveyed by this Agreement. + +H. A Recipient may choose to offer, and to charge a fee for, warranty, support, indemnity and/or liability obligations to one or more other Recipients of the Subject Software. A Recipient may do so, however, only on its own behalf and not on behalf of Government Agency or any other Recipient. Such a Recipient must make it absolutely clear that any such warranty, support, indemnity and/or liability obligation is offered by that Recipient alone. Further, such Recipient agrees to indemnify Government Agency and every other Recipient for any liability incurred by them as a result of warranty, support, indemnity and/or liability offered by such Recipient. + +I. A Recipient may create a Larger Work by combining Subject Software with separate software not governed by the terms of this agreement and distribute the Larger Work as a single product. In such case, the Recipient must make sure Subject Software, or portions thereof, included in the Larger Work is subject to this Agreement. + +J. Notwithstanding any provisions contained herein, Recipient is hereby put on notice that export of any goods or technical data from the United States may require some form of export license from the U.S. Government. Failure to obtain necessary export licenses may result in criminal liability under U.S. laws. Government Agency neither +represents that a license shall not be required nor that, if required, it shall be issued. Nothing granted herein provides any such export license. + +4. DISCLAIMER OF WARRANTIES AND LIABILITIES; WAIVER AND INDEMNIFICATION + +A. No Warranty: THE SUBJECT SOFTWARE IS PROVIDED “AS IS” WITHOUT ANY WARRANTY OF ANY KIND, EITHER EXPRESSED, IMPLIED, OR STATUTORY, INCLUDING, BUT NOT LIMITED TO, ANY WARRANTY THAT THE SUBJECT SOFTWARE WILL CONFORM TO SPECIFICATIONS, ANY IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR FREEDOM FROM INFRINGEMENT, ANY WARRANTY THAT THE SUBJECT SOFTWARE WILL BE ERROR FREE, OR ANY WARRANTY THAT DOCUMENTATION, IF PROVIDED, WILL CONFORM TO THE SUBJECT SOFTWARE. THIS AGREEMENT DOES NOT, IN ANY MANNER, CONSTITUTE AN ENDORSEMENT BY GOVERNMENT AGENCY OR ANY PRIOR RECIPIENT OF ANY RESULTS, RESULTING DESIGNS, HARDWARE, SOFTWARE PRODUCTS OR ANY OTHER APPLICATIONS RESULTING FROM USE OF THE SUBJECT SOFTWARE. FURTHER, GOVERNMENT AGENCY DISCLAIMS ALL WARRANTIES AND LIABILITIES REGARDING THIRD-PARTY SOFTWARE, IF PRESENT IN THE ORIGINAL SOFTWARE, AND DISTRIBUTES IT “AS IS.” + +B. Waiver and Indemnity: RECIPIENT AGREES TO WAIVE ANY AND ALL CLAIMS AGAINST THE UNITED STATES GOVERNMENT, ITS CONTRACTORS AND SUBCONTRACTORS, AS WELL AS ANY PRIOR RECIPIENT. IF RECIPIENT'S USE OF THE SUBJECT SOFTWARE RESULTS IN ANY LIABILITIES, DEMANDS, DAMAGES, EXPENSES OR LOSSES ARISING FROM SUCH USE, INCLUDING ANY DAMAGES FROM PRODUCTS BASED ON, OR RESULTING FROM, RECIPIENT'S USE OF THE SUBJECT SOFTWARE, RECIPIENT SHALL INDEMNIFY AND HOLD HARMLESS THE UNITED STATES GOVERNMENT, ITS CONTRACTORS AND SUBCONTRACTORS, AS WELL AS ANY PRIOR RECIPIENT, TO THE EXTENT PERMITTED BY LAW. RECIPIENT'S SOLE REMEDY FOR ANY SUCH MATTER SHALL BE THE IMMEDIATE, UNILATERAL TERMINATION OF THIS AGREEMENT. +5. GENERAL TERMS + +A. Termination: This Agreement and the rights granted hereunder will terminate automatically if a Recipient fails to comply with these terms and conditions, and fails to cure such noncompliance within thirty (30) days of becoming aware of such noncompliance. Upon termination, a Recipient agrees to immediately cease use and distribution of the Subject Software. All sublicenses to the Subject Software properly granted by the breaching Recipient shall survive any such termination of this Agreement. + +B. Severability: If any provision of this Agreement is invalid or unenforceable under applicable law, it shall not affect the validity or enforceability of the remainder of the terms of this Agreement. + +C. Applicable Law: This Agreement shall be subject to United States federal law only for all purposes, including, but not limited to, determining the validity of this Agreement, the meaning of its provisions and the rights, obligations and remedies of the parties. + +D. Entire Understanding: This Agreement constitutes the entire understanding and agreement of the parties relating to release of the Subject Software and may not be superseded, modified or amended except by further written agreement duly executed by the parties. + +E. Binding Authority: By accepting and using the Subject Software under this Agreement, a Recipient affirms its authority to bind the Recipient to all terms and conditions of this Agreement and that that Recipient hereby agrees to all terms and conditions herein. + +F. Point of Contact: Any Recipient contact with Government Agency is to be directed to the designated representative as follows: +Marian Nemec marian.nemec@nasa.gov diff --git a/docs/sphinx/.gitignore b/docs/sphinx/.gitignore new file mode 100644 index 0000000..ebb2a12 --- /dev/null +++ b/docs/sphinx/.gitignore @@ -0,0 +1,3 @@ +_build/ +_apidoc/ +.venv/ diff --git a/docs/sphinx/Makefile b/docs/sphinx/Makefile new file mode 100644 index 0000000..b8bc907 --- /dev/null +++ b/docs/sphinx/Makefile @@ -0,0 +1,34 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + # generate autodoc pages + sphinx-apidoc \ + --force \ + --module-first \ + --no-toc \ + --templatedir _templates/ \ + --separate \ + -o _apidoc \ + ../../python/upsp + + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +clean: Makefile + rm -rf api/* + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile clean diff --git a/docs/sphinx/_static/flowchart-batch-processing.png b/docs/sphinx/_static/flowchart-batch-processing.png new file mode 100644 index 0000000..e20bd30 Binary files /dev/null and b/docs/sphinx/_static/flowchart-batch-processing.png differ diff --git a/docs/sphinx/_static/flowchart-external-calibrate-coarse.png b/docs/sphinx/_static/flowchart-external-calibrate-coarse.png new file mode 100644 index 0000000..700004f Binary files /dev/null and b/docs/sphinx/_static/flowchart-external-calibrate-coarse.png differ diff --git a/docs/sphinx/_static/flowchart-external-calibrate-refined.png b/docs/sphinx/_static/flowchart-external-calibrate-refined.png new file mode 100644 index 0000000..49d4dba Binary files /dev/null and b/docs/sphinx/_static/flowchart-external-calibrate-refined.png differ diff --git a/docs/sphinx/_static/flowchart-psp-process.png b/docs/sphinx/_static/flowchart-psp-process.png new file mode 100755 index 0000000..271585c Binary files /dev/null and b/docs/sphinx/_static/flowchart-psp-process.png differ diff --git a/docs/sphinx/_static/memory-model-psp-process.png b/docs/sphinx/_static/memory-model-psp-process.png new file mode 100755 index 0000000..40d0678 Binary files /dev/null and b/docs/sphinx/_static/memory-model-psp-process.png differ diff --git a/docs/sphinx/_static/sls-pressure-time-history-screenshot.png b/docs/sphinx/_static/sls-pressure-time-history-screenshot.png new file mode 100644 index 0000000..a596ba2 Binary files /dev/null and b/docs/sphinx/_static/sls-pressure-time-history-screenshot.png differ diff --git a/docs/sphinx/_templates/module.rst_t b/docs/sphinx/_templates/module.rst_t new file mode 100644 index 0000000..fa64898 --- /dev/null +++ b/docs/sphinx/_templates/module.rst_t @@ -0,0 +1,9 @@ +{%- if show_headings %} +{{- ["``", basename, "``"] | join("") | heading }} + +{% endif -%} +.. automodule:: {{ qualname }} +{%- for option in automodule_options %} + :{{ option }}: + +{%- endfor %} diff --git a/docs/sphinx/_templates/package.rst_t b/docs/sphinx/_templates/package.rst_t new file mode 100644 index 0000000..2263210 --- /dev/null +++ b/docs/sphinx/_templates/package.rst_t @@ -0,0 +1,68 @@ +{%- macro automodule(modname, options) -%} +.. automodule:: {{ modname }} +{%- for option in options %} + :{{ option }}: +{%- endfor %} +{%- endmacro %} + +{%- macro toctree(docnames) -%} +.. toctree:: + :maxdepth: 2 +{% for docname in docnames %} + {{ docname }} +{%- endfor %} +{%- endmacro %} + + +{%- if is_namespace %} +{{- [pkgname, "namespace"] | join(" ") | escape | heading }} +{% else %} +{% if "." in pkgname %} +{{- ["``", pkgname, "``"] | join("") | heading }} +{% else %} +{{- ["``", pkgname, "`` Python API"] | join("") | heading }} +{% endif %} +{% endif %} + +{%- if is_namespace %} +.. py:module:: {{ pkgname }} +{% endif %} + +{%- if modulefirst and not is_namespace %} +{{ automodule(pkgname, automodule_options) }} +{% endif %} + + +{%- if subpackages %} +.. rubric:: Subpackages: + +{{ toctree(subpackages) }} +{% endif %} + + +{%- if submodules %} +{% if separatemodules %} +.. rubric:: Submodules + +{{ toctree(submodules) }} +{% else %} +.. contents:: Submodules: + :depth: 1 + :local: + +{% for submodule in submodules %} +{% if show_headings %} +{{- ["``", submodule, "``"] | join("") | heading(3) }} +{% endif %} +{{ automodule(submodule, automodule_options) }} +{% endfor %} +{%- endif %} +{%- endif %} + + +{%- if not modulefirst and not is_namespace %} +Module contents +--------------- + +{{ automodule(pkgname, automodule_options) }} +{% endif %} diff --git a/docs/sphinx/applications.rst b/docs/sphinx/applications.rst new file mode 100644 index 0000000..b60942e --- /dev/null +++ b/docs/sphinx/applications.rst @@ -0,0 +1,80 @@ +============ +Applications +============ + +The following sections provide an overview of the applications +distributed as part of the `upsp` software package. + +Running each application with the ``-h`` option will also provide +brief usage instructions to ``stdout``. + +``upsp-extract-frames`` +----------------------- + +The ``upsp-extract-frames`` application is a utility to dump high-speed +camera video frames to “friendlier” formats. Video frames can be output +to common image formats (e.g., jpg, png) or video formats (e.g., avi, +mp4). + +``upsp-external-calibration`` +----------------------------- + +The ``upsp-external-calibration`` application locates model targets in +the first frame, then finds the position and orientation of the wind +tunnel model relative to the camera using photogrammetry techniques. + +``psp_process`` +--------------- + +The majority of “computational heavy lifting” is performed by the +``psp_process`` application. Conceptually, the processing is split into +three steps: + +- **Initialization**: Video files are opened; first-frame camera + calibration parameters are loaded from file; fiducial patching is + initialized (see +@sec:fiducial-patching); diagnostic images for each + camera are saved; first-frame image-to-grid projections are + initialized +- **Phase 1**: For each time step, pixel (“intensity”) data from each + camera frame is projected to its corresponding vertex of the model + grid. Intensity data from multiple cameras are combined. +- **Phase 2**: The intensity time series at each grid vertex is + detrended and converted to a surface pressure measurement, using the + provided unsteady gain calibration and the reference steady-state + pressure. + +For more detail on the algorithms used in each step, please see the uPSP +SwDD. + +.. _`sec:fiducial-patching`: + +Fiducial Patching +~~~~~~~~~~~~~~~~~ + +In many cases, wind tunnel test models will have surface area not +covered by pressure-sensitive paint. Particularly troublesome cases are +small regions within larger painted areas, referred to as “fiducials.” +Example fiducials could be: + +- masking tape to protect transducer heads from paint spray (often + circular regions around a transducer head) +- unpainted points on the model used for wind-on external calibration + by ``upsp-external-calibration`` +- fasteners used to add/remove portions of test model between runs +- oil stains from lubrication of test model articles + +While larger regions can be excluded manually from downstream analysis, +these smaller areas are much harder to manually exclude. In this case, +``psp_process`` can automatically “patch” over known fiducial points on +the model if they are provided in the input ``tgts`` file. The position +and diameter of the fiducial must be supplied; a patch is applied in the +image plane prior to projecting to the model grid, where the patched +pixels are replaced by a 3rd-order, 2D-interpolation of the patch +boundary pixels. For fiducials that are closely-spaced such that their +patches would overlap, one larger patch is applied to the set of +fiducials (to cover use cases such as a shock array of closely-spaced +transducer heads on the model in a streamwise line). Diagnostic images +written out by ``psp_process`` include an example of the patching output +for the first frame from each video file, and can be used to manually +tune the patch algorithm parameters for best performance on a +per-wind-tunnel-test basis. diff --git a/docs/sphinx/citing.rst b/docs/sphinx/citing.rst new file mode 100644 index 0000000..e31c1df --- /dev/null +++ b/docs/sphinx/citing.rst @@ -0,0 +1,25 @@ +================ +Citing this work +================ + +The following reference contains the original publication of the processing software algorithms and implementation: + +.. bibliography:: + :filter: False + + Powell-SciTech-2020 + +Additional work by the authors and collaborators related to uPSP research and development: + +.. bibliography:: + :filter: False + + Murakami-SciTech-2023 + Bremner-2022 + Li-SciTech-2022 + Tang-Aviation-2021 + Roozeboom-SciTech-2020 + Roozeboom-Aviation-2019 + Roozeboom-SciTech-2018 + Roozeboom-SciTech-2017 + Roozeboom-SciTech-2016 diff --git a/docs/sphinx/conf.py b/docs/sphinx/conf.py new file mode 100644 index 0000000..12c8508 --- /dev/null +++ b/docs/sphinx/conf.py @@ -0,0 +1,110 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys +sys.path.insert(0, os.path.abspath(os.path.join("..", "..", "python"))) + + +# -- Project information ----------------------------------------------------- + +project = 'uPSP' +copyright = '2021, uPSP Developers' +author = 'uPSP Developers' + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.autosectionlabel", + "sphinx.ext.intersphinx", + "sphinx.ext.napoleon", + "sphinx.ext.viewcode", + "sphinxcontrib.bibtex", +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +# html_theme = "sphinx_rtd_theme" +html_theme_options = "alabaster" +html_theme_options = { + "fixed_sidebar": True, + "page_width": "1200px", + "globaltoc_maxdepth": 2, +} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + + +# autodoc config +autodoc_default_options = { + "undoc-members": True, +} +autodoc_mock_imports = ["cv2", "upsp.raycast"] + +# just show the method/function name in signatures, not the full path +add_module_names = False + +numfig = True + +autosectionlabel_prefix_document = True + +latex_engine = "pdflatex" +latex_elements = { + "fontpkg": "", +} + +rst_prolog = """ +.. role:: python(code) + :language: python + :class: highlight +""" + +bibtex_bibfiles = ['refs.bib'] + +napoleon_preprocess_types = True +napoleon_type_aliases = { + "np.ndarray": ":class:`numpy.ndarray`", + "ndarray": ":class:`numpy.ndarray`", + "array_like": ":term:`array_like`", + "array-like": ":term:`array-like `", + "path-like": ":term:`path-like `", +} + +intersphinx_mapping = { + "python": ("https://docs.python.org/3", None), + "numpy": ("https://numpy.org/doc/stable/", None), + "scipy": ("https://docs.scipy.org/doc/scipy/", None), + "pandas": ("https://pandas.pydata.org/docs", None), + "matplotlib": ("https://matplotlib.org/stable/", None), +} + +# nitpicky = True diff --git a/docs/sphinx/dependencies.rst b/docs/sphinx/dependencies.rst new file mode 100644 index 0000000..76b2ee3 --- /dev/null +++ b/docs/sphinx/dependencies.rst @@ -0,0 +1,58 @@ +Third-Party Dependencies +======================== + +The majority of third-party dependencies are open-source and available +from public internet endpoints. :numref:`third party deps table` contains details for +each dependency including the version, license, and description of usage +by the uPSP software. + +The only current commercial dependency is the Message Passing Interface +(MPI) implementation used for operation of the uPSP software on the +National Advanced Supercomputing (NAS) cluster at NASA Ames. The +dependency is via shared library linking at build time and is required +for leveraging parallelization across NAS cluster nodes. Prior to +release of uPSP source code to non-NAS environments, this dependency +will be revisited to identify proper alternatives (there are open-source +MPT/MPI implementations available, however, they are not needed in the +NAS environment). + +.. _third party deps table: +.. csv-table:: Third-Party Dependencies + :header-rows: 1 + + Name,Version,License,Description of usage + Boost_,1.68.0,`Boost License `__,"C++ routines for efficient data structure manipulation (e.g., array iterators)" + CMake_,3.20.5,`3-clause BSD `__,Build system for uPSP project + Eigen_,3.3.9,`MPL2 `__,"Header-only software dependency, used for high-performance linear algebra and matrix mathematics" + gcc_,8.2,`GPLv3 `__,Compiler used to build C/C++ uPSP applications (usage is covered under GPLv3 GCC runtime exception) + GOMP_,1.0.0,`GPLv3 `__,"GNU implementation of the OpenMP specification, for parallelization of uPSP processing (usage is covered under the GPLv3 GCC runtime exception)" + GTest_,1.11.0,`3-clause BSD `__,Unit testing for C/C++ uPSP applications as part of the uPSP build system. + HDF5_,1.12.0,`HDF5 License `__,Some uPSP output data artifacts are encoded in the HDF5 format. + `JSON for Modern C++`_,3.9.1,`MIT `__,"Header-only software dependency, used for parsing JSON files." + kdtree_,0.5.7,`3-clause BSD `__,"C library for working with KD-trees, used as part of the camera-to-model data projection process." + Matplotlib_,3.3.2,`PSF-like `__,Plotting routines for Python uPSP applications. + Numpy_,1.19.2,`3-clause BSD `__,Scientific computing for Python uPSP applications. + OpenCV_,4.5.2,`3-clause BSD `__,Computer vision processing (image registration and p ost-processing). + OpenEXR_,2.5.0,`Modified BSD `__,The IlmBase library (component of OpenEXR) is used for high-performance raycasting as part of the camera-to-model data projection process. + pbrt-v3_,3,`2-clause BSD `__,High-performance raycasting as part of the camera-to-model data projection process. Several source code files from the pbrt project are copied + modified and distributed as part of the uPSP source code. + Python_,3.7.9,`PSF License `__,The Python interpreter is a necessary system dependency for running some uPSP pipeline applications. (The Python interpreter itself is not modified or redistributed as part of the release.) + Scipy_,1.6.0,`3-clause BSD `__,Scientific computing for Python uPSP applications. + `SGI MPT`_,2.17,Commercial License,uPSP software built for deployment on the NASA Advanced Supercomputing (NAS) facility leverages the NAS-provided commercial implementation of the MPI 1.1 specification for scaling of parallel processing across the NAS cluster nodes. The library is linked to uPSP applications as a shared library. + +.. _Boost: https://www.boost.org +.. _CMake: https://cmake.org +.. _Eigen: https://eigen.tuxfamily.org +.. _gcc: https://gcc.gnu.org +.. _GOMP: https://gcc.gnu.org/projects/gomp +.. _GTest: https://github.com/google/googletest +.. _HDF5: https://www.hdfgroup.org/solutions/hdf5 +.. _`JSON for Modern C++`: https://github.com/nlohmann/json +.. _kdtree: http://nuclear.mutantstargoat.com/sw/kdtree +.. _Matplotlib: https://matplotlib.org +.. _Numpy: https://numpy.org +.. _OpenCV: https://opencv.org +.. _OpenEXR: https://www.openexr.com/index.html +.. _pbrt-v3: https://github.com/mmp/pbrt-v3 +.. _Python: https://www.python.org +.. _Scipy: https://www.scipy.org +.. _`SGI MPT`: https://www.nas.nasa.gov/hecc/support/kb/sgi-mpt_89.html diff --git a/docs/sphinx/docutils.conf b/docs/sphinx/docutils.conf new file mode 100644 index 0000000..1bf4d83 --- /dev/null +++ b/docs/sphinx/docutils.conf @@ -0,0 +1,2 @@ +[restructuredtext parser] +syntax_highlight = short diff --git a/docs/sphinx/file-formats.rst b/docs/sphinx/file-formats.rst new file mode 100644 index 0000000..c0a83aa --- /dev/null +++ b/docs/sphinx/file-formats.rst @@ -0,0 +1,929 @@ +============ +File Formats +============ + +The following file formats are used by the uPSP processing software. + +.. _sec-video-file: + +High-speed camera video formats +=============================== + +The uPSP software supports the following vendor-specific files: + +- CINE files (from Phantom Vision cameras) +- MRAW/CIH files (from Photron cameras) + +In particular, the software is tested primarily with monochannel, +“12-bit packed” images. Other packing strategies are supported but are +less commonly used. + +.. _sec-grid-file: + +Surface grid definition +======================= + +The uPSP software requires an accurate definition of the wind tunnel +model wetted surfaces. It currently supports two file formats, +auto-detected based on file extension: + +- ``*.g``, ``*.x``, ``*.grid``, ``*.grd``: PLOT3D structured grid +- ``*.tri``: Cart3D unstructured grid + +The following subsections provide more detail for each grid file format. + +.. _sec-plot3d: + +PLOT3D file formats +------------------- + +`PLOT3D `_ is a computer +graphics program (written at NASA) to visualize the +grids and solutions of computational fluid dynamics. +The program defined several file formats that are now industry standard; +in particular, refer to +`PLOT3D User Manual, Chapter 8: Data File Formats `_. +The uPSP software processes the following subset of PLOT3D file formats: + +- 3D Grid XYZ file (``*.grd``, ``*.x``, ``*.g``) (PLOT3D User Manual, + Section 8.2) + + - Unformatted FORTRAN (binary, and contains leading- and trailing- + 32-bit record separators) + - Multi-grid + - Single precision + - No IBLANKS + +- 3D Function file (``*.p3d``, ``*.f``) (PLOT3D User Manual, Section + 8.4) + + - Unformatted FORTRAN (binary, and contains leading- and trailing- + 32-bit record separators) + - Multi-grid + - Single precision + - Assumes only one scalar variable per vertex (in user manual, + ``NVAR=1``) + +Note that the uPSP software does *not* ingest solution files (``*.q``), +which normally contain values for the full flow variable solution at +each grid point. Solution files are more commonly produced by CFD +solvers and are occasionally confused with function files. + +.. _sec-cart3d: + +Cart3D file formats +------------------- + +`Cart3D `_ +is a high-fidelity inviscid analysis package (written at NASA) for conceptual and preliminary +aerodynamic design. The package defined several `grid file formats `_ +that are now industry standard. The uPSP software processes the following subset of Cart3D file formats: + +- Surface triangulations (``*.tri``) for 3D surface geometry definition +- Annotated triangulations (``*.triq``) for scalar values defined over + a surface triangulation + +.. _sec-tgts-file: + +"Targets" file (``*.tgts``) +=========================== + +In addition to a specification of the model surface geometry, the uPSP +software must be provided a file that specifies: + +- The 3D position and normal for visible key points on the model surface, known + as targets + + - These key points must be dark regions on the model surface where + paint was either not applied to purposefully darked with a stain + +- Circular surface regions (position + diameter) where paint is + not present or is damaged + +The targets locations must be outside the model grid by a distance of +1e-5 +/- 5e-6 inches. + +The file format is commonly called the “targets” file and is defined by +the DOTS application, a steady-state PSP solver application used at +NASA, but should be modified to ensure the targets are consistent with +the grid file. An example is included below. + +Targets File Example: + +.. code:: + + # x y z normal x normal y normal z size i j k name + 1 67.02449799 3.17536973 -2.70057420 0.000000 0.762017 -0.647557 0.433 596 21 55 st6 + 2 66.52809906 2.83113538 3.05953808 0.000000 0.678209 0.734869 0.433 596 85 50 st7 + 3 79.28500366 7.62883166 -1.21192827 0.000420 0.742280 -0.670089 0.433 2261 160 26 st8 + 4 79.03060150 7.57744602 1.26724449 -0.000430 0.718970 0.695040 0.433 2261 20 23 st9 + 5 81.26619721 2.67079590 3.20469503 0.000000 0.641469 0.767148 0.433 716 29 25 st10 + 6 95.46779633 7.70066046 1.12676893 0.000000 0.786289 0.617859 0.433 2260 17 30 st11 + 7 98.17250061 2.85770474 3.03950497 0.000000 0.677413 0.735603 0.433 716 189 27 st12 + 8 110.63110352 7.77465312 1.02533593 0.000000 0.826620 0.562760 0.433 2260 15 224 st13 + + +.. _sec-steady-file: + +Steady-state surface pressure file +================================== + +The uPSP software requires a reference, steady-state surface pressure at +each grid point in order to compute the unsteady fluctuating pressure. +It supports the following file formats for ingesting steady-state +pressure data, auto-detected by file extension. It is also dependent on +the format used for the model grid file. + +- PLOT3D function file (``*.f``, ``*.p3d``) if the model grid is PLOT3D + structured +- Cart3D annotated triangulation (``*.triq``) if the model grid is + Cart3D unstructured + +See :ref:`file-formats:PLOT3D file formats` and :ref:`file-formats:Cart3D file formats` +for more details. In both formats, the scalar values +should be provided as values of the coefficient of pressure (:math:`C_p`). + +.. _sec-wtd-file: + +Wind Tunnel Data (``*.wtd``) +============================ + +During operations at the NASA Ames UPWT Complex 11-ft test section, +time-averaged values for positioning of the model in the test +section and current flow conditions are provided +by the wind tunnel data systems via a simple, human-readable +text file simply called the Wind Tunnel Data (``*.wtd``) file. + +An example WTD file is shown below. Tunnel data system values are written +tab-delimited with their name in the corresponding header column. + +.. code:: + + # MACH ALPHA BETA PHI STRUTZ_UNC + 0.838751 -0.041887 0.011826 -90.000000 3.012192 + +There are several uPSP applications that make use of data from +this file: + +- The wind-on external calibration step requires an initial guess of + the model positioning in the test section +- The "intensity-to-pressure" paint calibration process requires + the steady-state flow conditions for empirical estimates of the + model surface temperature in cases where a per-grid-point temperature + input file is not provided + +The uPSP applications make use of the following columns from the +WTD file (which may have many more columns that are ignored by the uPSP code): + +- ``STRUTZ_UNC``: (Uncorrected) "strutz" position, or current Z-coordinate of the model strut +- ``ALPHA``: (Corrected) Model pitch angle +- ``BETA``: (Corrected) Model sideslip angle +- ``PHI``: (Corrected) Model roll angle + +The model angles are "corrected" by the wind tunnel data system to +account for additional bending due to deflection of the sting and +model mounting system. All coordinates and angles are consistent with definitions +published by NASA for the 11-ft test section. Technical details about the NASA +Ames 11-ft test section, including coordinate system definitions and +naming conventions, are published online by NASA (see +`Unitary Plan Wind Tunnel 11-by 11-foot TWT Test Section `_). + +.. _sec-upsp-gain-file: + +Unsteady PSP gain calibration +============================= + +The uPSP software requires a pre-computed calibration for converting +intensity ratios from camera pixel values into physical units of +pressure. + +The calibration has 6 coefficients (``a``, ``b``, ``c``, ``d``, ``e``, +``f``) that should be provided in a plain text file, example as follows: + +.. code:: + + a = 1.1 + b = 2.2 + c = 3.3 + d = 4.4 + e = 5.5 + f = 6.6 + +See :ref:`swdd:Phase 2 processing` for their usage in a polynomial +relation between pressure, temperature, and the unsteady gain value. + +.. _sec-cam-cal-file: + +Camera-to-tunnel calibration +============================ + +The camera-to-tunnel calibration file contains the intrinsic camera +parameters as well as the extrinsic camera parameters relative to the +tunnel origin. It is a JSON file with the following elements: + +- ``uPSP_cameraMatrix``: Camera Matrix formatted as ``[[f, 0, dcx], [0, f, + dcy], [0, 0, 1]]``, where f is the focal length (in pixels), and (dcx, + dcy) is the vector in pixel space from the image center to the + principal point. +- ``distCoeffs``: OpenCV 5-parameter lens distortion coefficients formatted + as ``[k1, k2, p1, p2, k3]`` +- ``rmat``: rotation matrix from camera to tunnel +- ``tvec``: translation vector from camera to tunnel + +Optional: + +- ``sensor_resolution``: Camera sensor resolution +- ``sensor_size``: Camera sensor physical size +- ``Updated``: Date of last update to this file + +.. _sec-input-deck: + +``psp_process`` configuration (``*.inp``) +========================================= + +The input deck file was designed to coordinate most of the inputs and +options needed for ``psp_process``. It is also a good reference for +which files influenced the final processed results. Descriptions of all +the variables included in the input deck are included in :numref:`input-deck` + +.. code:: + + @general + test = my-test-event-name + run = 1234 + sequence = 56 + tunnel = ames_unitary + @vars + dir = /nobackup/upsp/test_name + @all + sds = $dir/inputs/123456.wtd + grid = $dir/inputs/test-subject.grid + targets = $dir/inputs/test-subject.tgts + @camera + number = 1 + cine = $dir/inputs/12345601.cine + calibration = $dir/inputs/cam01-to-model.json + aedc = false + @camera + number = 2 + filename = $dir/inputs/12345602.mraw + calibration = $dir/inputs/cam02-to-model.json + aedc = false + @options + target_patcher = polynomial + registration = pixel + overlap = best_view + filter = gaussian + filter_size = 3 + oblique_angle = 70 + number_frames = 2000 + @output + dir = $dir/outputs + + +.. _input-deck: +.. table:: ``psp_process`` input deck parameter descriptions. + + .. list-table:: + :widths: 5 10 20 5 60 + :header-rows: 1 + + * - Section + - Variable + - Description + - Required? + - How it is used + + * - General + - + - + - + - + + * - + - test + - test id number + - yes + - included in output HDF5 files + + * - + - run + - run number + - yes + - included in output HDF5 files + + * - + - sequence + - sequence number + - yes + - included in output HDF5 files + + * - + - tunnel + - tunnel identifier + - yes + - used for determining which tunnel transformations and input files to expect, + only currently support ``ames_unitary`` + + * - Vars + - + - allows variables to be set for use within the file + - no + - any variable can be used anywhere else in the file when preceeded with ``$``, it will + be replaced with the value when processed + + * - All + - + - + - + - + + * - + - sds + - wind tunnel data (WTD) file + - yes + - many variables are included in the output HDF5 files; used to determine the + orientation of the model for calibration; used as part of converting + camera intensity to pressure + + * - + - grid + - grid file + - yes + - will be the basis of the projection from the image plane into space, data will be + stored, when available, at each grid node + + * - + - targets + - targets file + - yes + - targets used to correct the calibration for this data point; targets and + fiducials: patched over by the target patcher + + * - + - normals + - grid vertex normals override + - no + - allows for individually setting the normal direction for individual grid nodes, used as part + of projection, useful for non-watertight structured grids + + * - Camera + - + - + - + - need a block per camera that will be processed + + * - + - number + - camera id number + - yes + - used to match cine files to the correct camera calibration, should not have duplicate camera numbers + + * - + - cine + - cine file + - yes + - path to the + camera video file + that will be + processed + (deprecated; + prefer “filename” + key instead) + + * - + - aedc + - aedc cine file type flag + - no + - aedc format is + different than other cine file formats, so it is used to read the cine + file; default is false + + * - + - filename + - video file + - yes + - path to the + camera video file. Supported extensions: \*.mraw, \*.cine. For + \*.mraw, the \*.cih header file must be a sibling file of the \*.mraw + file with the same basename, e.g., ``video-01.mraw`` and + ``video-01.cih``. + + * - + - calibration + - model-to-camera external calibration file + - yes + - path to external + camera calibration file (output from ``upsp-extern al-calibration``; + calibration of camera frame relative to position of wind tunnel model + in the first frame of the camera video file). + + * - Options + - + - + - + - + + * - + - target_patcher + - type of target patching + - no + - decide what type of target patching is implemented, supports either ``polynomial`` or + ``none``; default is ``none`` + + * - + - registration + - image registration type + - no + - decide what type of image registration to perform, supports either ``pixel`` or + ``none``; default is ``none`` + + * - + - overlap + - multi-view handling + - no + - specify how to handle points that are visible from multiple cameras, supports either + ``best_view`` or ``average_view``; default is ``average_view`` + + * - + - filter + - image plane filtering + - no + - decide what type of filtering to apply to each image prior to projection, supports + either ``gaussian`` or ``none``; default is ``none`` + + * - + - filter_size + - size of the filter + - yes + - decide how large the filter will be in pixels, must be odd + + * - + - oblique_angle + - minimum projection angle + - no + - minimum angle between grid surface plane and camera ray to be considered visible by + the camera; default 70 (degrees) + + * - + - number_frames + - number of frames to process + - yes + - number of camera frames to process (-1 = all frames) + + * - Output + - + - + - + - + + * - + - dir + - output directory + - yes + - destination directory for output files + + +.. _sec-excal-file: + +External camera calibration configuration parameters +==================================================== + +The external calibration application requires a set of configuration +parameters. These parameters are mainly static parameters related to the +tunnel, model setup in the tunnel, or hyper parameters related to the +external calibration process. This input is stored as a JSON with the +following elements: + +- ``oblique_angle``: Oblique viewing angle for target visibility checks +- ``tunnel-cor_to_tgts_tvec``: Tunnel center of rotation to targets frame + translation vector +- ``tunnel-cor_to_tgts_rmat``: Tunnel center of rotation to targets frame + rotation matrix +- ``tunnel-cor_to_tunnel-origin_tvec``: Tunnel center of rotation to tunnel + origin translation vector +- ``tunnel-cor_to_tunnel-origin_rmat``: Tunnel center of rotation to tunnel + origin rotation matrix +- ``dot_blob_parameters``: Blob detection parameters to find sharpie + targets +- ``dot_pad``: Sharpie target padding distance for sub-pixel localization +- ``kulite_pad``: Kulite target padding distance for sub-pixel localization +- ``max_dist``: Maximum matching distance between a wind-off target + position and a detected target +- ``min_dist``: Minimum distance between two targets before they become too + close and ambiguous + +Optional: + +- ``Updated``: Date of last update to this file + +Batch processing configuration +============================== + +Datapoint index +--------------- + +Example: + +.. code:: json + + { + "__meta__": { + "config_name": "", + "test_name": "" + } + "": { + "camera_tunnel_calibration_dir": "", + "camera_video_dir": "", + "grid_file": "", + "kulites_files_path": "", + "normals_file": "", + "paint_calibration_file": "", + "steady_psp_file": "", + "targets_file": "", + "wtd_file": "" + } + } + +- Each input file is described in more detail in other sections of this chapter +- In the case of ``*.mraw``-formatted camera video files, it is assumed + that the corresponding ``*.cih`` video header file is a sibling of + the ``*.mraw`` file in the same folder, with the same name (e.g., + ``12345601.mraw`` and ``12345601.cih``). + +Processing pipeline configuration parameters +-------------------------------------------- + +The processing parameters file is a single location for the user to +specify parameter values for all applications in the processing +pipeline. + +The file is structured to allow the user to specify a set of default +values as well as one or more “overlays” to customize parameter values +for individual datapoints or for sets of datapoints that share common +characteristics. Each overlay is applied to a datapoint based on whether +one or more key-value pairs in its entry in the datapoint index (see +:ref:`file-formats:Datapoint index`) matches a series of regular +expressions (specified using Python-format syntax, see +`here `__). + +Example: + +.. code:: json + + { + "__meta__": { + "name": "" + }, + "processing": { + "defaults": { + "psp_process": { + "cutoff_x_max": 120, + "filter": "none", + "filter_size": 1, + "number_frames": 20, + "oblique_angle": 70, + "registration": "pixel", + "target_patcher": "polynomial" + }, + "external-calibration": { + "Updated": "07/27/2021", + "dot_blob_parameters": [ + ["filterByColor", 1], + ["filterByArea", 1], + ["filterByCircularity", 1], + ["filterByInertia", 1], + ["filterByConvexity", 1], + ["thresholdStep", 1], + ["minThreshold", 13], + ["maxThreshold", 40], + ["minRepeatability", 4], + ["minDistBetweenBlobs", 0], + ["blobColor", 0], + ["minArea", 4], + ["maxArea", 24], + ["minCircularity", 0.72], + ["maxCircularity", 0.95], + ["minInertiaRatio", 0.25], + ["maxInertiaRatio", 1.01], + ["minConvexity", 0.94], + ["maxConvexity", 1.01] + ], + "dot_pad": 4, + "image_dims": [512, 1024, 1], + "kulite_pad": 3, + "max_dist": 6, + "min_dist": 10, + "model_length": 87.1388, + "oblique_angle": 70, + "sensor_resolution": [800, 1280, 1], + "sensor_size": [0.8818898, 1.4110236], + "sting_offset": [-195.5125, 0, 0], + "tgts_transformation_rmat": [ + [1, 0, 0], + [0, 1, 0], + [0, 0, 1] + ], + "tgts_transformation_tvec": [-32.8612, 0, 0], + "tunnel-cor_to_tgts_tvec": [-195.5125, 0, 0] + } + }, + "test-model-geometry-2": { + "external-calibration": { + "model_length": 123.456 + } + }, + "test-model-geometry-3": { + "external-calibration": { + "model_length": 150.456 + } + }, + "__overlays__": [ + ["defaults", {".*": ".*"}], + ["test-model-geometry-2", {"grid_file": ".*test-model-geometry-2.*"}] + ["test-model-geometry-3", {"grid_file": ".*test-model-geometry-3.*"}] + ] + } + } + +Each child of ``"processing"`` NOT named ``__overlays__`` specifies +some/all parameter values for some/all applications in the pipeline, and +is referred to here as a “parameter set”. Each parameter set is given a +name, for example, ``"defaults"`` or ``"test-model-geometry-3"`` in the above file. +Each parameter set need not specify every available application +parameter; in the example above, ``"defaults"`` contains a full +specification, whereas ``"test-model-geometry-3"`` contains a specific value just +to override the ``"model_length"`` parameter of the ``"external-calibration"`` +pipeline application. The ``"test-model-geometry-3"`` overlay is applied to all +datapoints whose ``"grid_file"`` matches the regular expression :python:`r".*test-model-geometry-2.*"` + +In general, all parameter names map to corresponding parameters supplied +directly to each pipeline application when running them individually/manually +(see :ref:`swdd:Pipeline Application Design Details`). + +The ``__overlays__`` section specifies how to use the parameter sets to +configure each datapoint. The user must provide a list of overlay +entries, each in the format ``[, ]``. Each datapoint +matching the contents of ```` will use the parameter values +given by the parameter set named ````. Overlays are applied in the +order listed, so in the example above, the ``config123`` parameter set +will override any values already specified in ``defaults``. The usage of +```` for matching a datapoint is as follows: + +- ```` is a dictionary where each key ``k`` and each value + ``v`` are regular expression strings (Python-format) +- For each entry in ````, and for each input in the index + JSON (``grid_file``, ``targets_file``, etc.) + + - If the input key matches the regular expression given by ``k``, + then + - The input value is tested against the regular expression given by + ``v``. + +- If all tests pass for all entries in ````, then the + datapoint “matches.” + +Portable Batch Scheduler (PBS) job configuration parameters +----------------------------------------------------------- + +Example: + +.. code:: json + + { + "__meta__": { + "name": "" + }, + "nas": { + "__defaults__": { + "charge_group": "", + "node_model": "", + "queue": "", + "number_nodes": "", + }, + "external-calibration": { + "launcher": "parallel", + "wall_time": "" + }, + "extract-first-frame": { + "launcher": "parallel", + "wall_time": "" + }, + "psp_process": { + "launcher": "serial", + "number_nodes": "", + "wall_time": "" + }, + "render-images": { + "launcher": "parallel", + "wall_time": "" + } + } + } + +Each step in the pipeline should be assigned a full set of NAS job +parameters. A set of ``__defaults__`` will apply to all steps, and then +optional overrides can be provided in a section named for each step. The +job parameters are used by ``qsub-step`` to launch step jobs on the NAS +cluster nodes, and are tuned primarily by the number of vertices in the +grid file and the number of frames in each camera video file. See +:ref:`quick-start:Guidelines for setting NAS PBS job parameters` for more details. + +Plotting configuration parameters +--------------------------------- + +Example: + +.. code:: json + + { + "__meta__": { + "name": "" + }, + "plotting": { + "render-images": { + "scalars": { + "steady_state": { + "display_name": "Steady State", + "contours": [-1.0, -0.8, -0.6, -0.4, -0.2, 0.0, 0.2, 0.4, 0.6, 0.8, 1.0] + }, + "rms": { + "display_name": "delta Cp RMS", + "contours": [0, 0.005, 0.01, 0.015, 0.02, 0.025, 0.03] + } + }, + "grids": { + "config111": { + "views": { + "left-side": { + "x": 123.4, + "y": 123.4, + "z": 123.4, + "psi": 90.0, + "theta": 0.0, + "width": 120.0, + "imagewidth": 1280 + }, + }, + "exclude_zones": [101, 120] + } + } + }, + "generate-miniwall": { + "variables": [ + {"name": "mach", "format": "%4.3f", "values": [0.8, 0.85, 0.9]}, + {"name": "alpha", "format": "%4.2f", "values": [0.0, 5.0, 10.0]}, + {"name": "beta", "format": "%4.2f", "values": [0.0, 5.0, 10.0]} + ], + "selectors": ["alpha", "beta"], + "layout": {"cols": ["mach"], "rows": ["alpha", "beta"]}, + "site": { + "page_name": "My MiniWall", + "images_dir": "images", + "model_config_prefix": "PREFIX" + }, + "run_log": { + "model_config": "111", + "path": "my-run-log.txt" + }, + "image_discovery": { + "ext": "png", + "patterns": { + "datapoint": "[0-9]{6}", + "panel_name": "[0-9][0-9]" + }, + "cache": "lib/image_discovery.json" + } + } + } + } + +All parameters map to corresponding elements of the ``upsp-plotting`` +configuration file (see :ref:`swdd:Pipeline Application Design Details`). + +Output files from ``psp_process`` +================================= + +Output data from ``psp_process`` consists of :math:`\Delta C_p` +measurements versus time for all nodes of the user-provided wind tunnel +model grid. The measurements are provided either in raw binary format or +bundled into an HDF5 file. In addition, metadata containing the model +grid definition, wind tunnel conditions, and camera settings are bundled +with the measurements in the HDF5 file. + +.. _sec-pressure-transpose-file: + +Pressure-time history raw binary outputs +---------------------------------------- + +The ``psp_process`` application will produce two (often large) binary +files called ``pressure`` and ``pressure_transpose``. If using the batch +processing tools (described in :ref:`file-formats:Batch processing configuration`), +these files will be available in the ``output/10_other/additional_output`` subdirectory +for each datapoint; otherwise, they will be available in the directory +supplied to ``psp_process`` using the ``--add_out_dir`` command line +option. Both files contain the same data, however, the formatting +differs to facilitate downstream use cases. + +The ``pressure`` binary file is formatted as follows. For a given +datapoint, the primary output data from ``psp_process`` is the +pressure-time history, defined as a matrix :math:`P` of size +:math:`[N_f \times N_m]` where :math:`N_f` is the number of frames from +a uPSP camera (all cameras are synchronized, so this corresponds to the +number of time steps) and :math:`N_m` is the number of grid nodes. If we +let :math:`i = 1, 2, \ldots, N_f`, and :math:`j = 1, 2, \ldots, N_m`, +then element :math:`P_{i, j}` corresponds to the unsteady pressure +(:math:`\Delta C_p`) measured during the :math:`i`\ ’th time step at the +location of the :math:`j`\ ’th model grid node (the ordering of grid +nodes corresponds one-to-one with the ordering in the user-supplied +model grid definition file; see :ref:`file-formats:Surface grid definition`). +Equivalently, rows of :math:`P` correspond to snapshots of the pressure +distribution over the entire grid at each time step. The ``pressure`` +binary file contains the matrix :math:`P` written to disk in row-major +format, where each value is a 32-bit, little endian, IEEE floating point +number. This corresponds to the following sequence of matrix elements: + +.. math:: + + P_{1,1}, P_{1,2}, ..., P_{1,N_m}, P_{2,1}, P_{2,2}, ..., P_{2,N_m}, ..., P_{N_f,1}, P_{N_f,2}, ..., P_{N_f,N_m} + +Because this file can be quite large (for example, a grid with 1 million +nodes and a datapoint with 50,000 time points will have a total of 50 +billion floating point values, corresponding to approximately 186 GB), +performance of file read operations can become an issue during +post-processing. It is advantageous to read contiguous values from the +file, meaning the ``pressure`` binary file is best suited for analyses +of the *entire* model grid for a *subset* of time history. + +To facilitate analyses of the *entire* time history for a *subset* of +model grid nodes, a separate ``pressure_transpose`` file is also +generated. It contains the values of the *transpose* of :math:`P` +written to disk in a similar row-major format. This corresponds to the +following sequence of matrix elements: + +.. math:: + + P_{1,1}, P_{2,1}, ..., P_{N_f,1}, P_{1,2}, P_{2,2}, ..., P_{N_f,2}, ..., P_{1,N_m}, P_{2,N_m}, ..., P_{N_f,N_m} + +In the ``pressure_transpose`` file, the time history for each individual +grid node is contiguous in memory. + +The following Python code snippet demonstrates how the +``pressure_transpose`` flat file may be used to obtain the time history +for a single model grid node: + +.. code:: python + + import numpy as np + import os + + def read_pressure_transpose(filename, number_grid_nodes, node_index): + """ Returns a 1-D numpy array containing model grid node pressure time history + - filename: path to pressure_transpose file + - number_grid_nodes: total number of model grid nodes + - node_index: index of grid node (zero-based) in model grid file + """ + filesize_bytes = os.path.getsize(filename) + itemsize_bytes = 4 # 32-bit floating point values + number_frames = int((filesize_bytes / itemsize_bytes) / number_grid_nodes) + assert(number_frames * number_grid_nodes * itemsize_bytes == filesize_bytes) + with open(filename, 'rb') as fp: + offset = node_index * number_frames + fp.seek(offset, 0) + return np.fromfile(fp, dtype=np.float32, count=number_frames) + +HDF5-formatted files +-------------------- + +HDF5-formatted files are also provided containing the pressure-time +history solution matrix and associated metadata. + +Diagnostics and quality checks +------------------------------ + +There are a number of additional outputs that are useful for checking +behavior of the processing code and to check quality of input data +files. By default, outputs are stored in the output directory specified +in the input deck file; this can be overridden using the +``-add_out_dir`` command line option. :numref:`psp-process-quality-checks` +describes each output file in more detail. + +.. _psp-process-quality-checks: +.. list-table:: ``psp_process`` output files for diagnostics and quality checks. The prefix ```` refers to an image from camera ``XX``. + :widths: 20 80 + :header-rows: 1 + + * - File name + - Description + * - ``-8bit-raw.png`` + - first frame, scaled to 8-bit + * - ``-raw.exr`` + - first frame, converted to high-dynamic-range, 32-bit, OpenEXR format. High-fidelity representation of the first frame from the camera. Pixel values are equal to those in the source image — all supported source video files have less than 32 bits per pixel, so 32-bit OpenEXR provides a way to preserve fidelity across vendor-specific input video files. + * - ``-8bit-projected-fiducials.png`` + - first frame, scaled to 8-bit, with fiducial positions from the ``tgts`` file projected into the frame accounting for occlusion and obliqueness checks, using the input external calibration parameters + * - ``-8bit-fiducial-clusters.png`` + - first frame, scaled to 8-bit, with fiducials colored according to their cluster ID (only clusters with >1 fiducial are assigned colors) + * - ``-8bit-cluster-boundaries.png`` + - first frame, scaled to 8-bit, showing boundaries drawn around fiducial clusters. Pixels inside the drawn boundaries will be “patched” by replacing their values with a polynomial interpolant (prior to projection to the 3D grid) + * - ``-nodecount.png`` + - first frame, colormapped to show # grid nodes mapped to each pixel. Ideally, if no image filter is used, then there should be an approximate one-to-one map of grid nodes to pixel; otherwise, an image filter can be used to average neighboring pixels, so that pixels with no grid node will have their value averaged into a neighboring value that does map to a grid node. If there are many more grid nodes than pixels, then it may indicate the grid resolution is over-specified compared to the resolution of the camera. diff --git a/docs/sphinx/index.rst b/docs/sphinx/index.rst new file mode 100644 index 0000000..e3e26e6 --- /dev/null +++ b/docs/sphinx/index.rst @@ -0,0 +1,40 @@ +================================================================= +Unsteady Pressure-Sensitive Paint (uPSP) Data Processing Software +================================================================= + +The ``upsp`` package consists of a suite of applications for processing +high-speed video files from fast-response, pressure-sensitive paint +camera systems. It also provides a Python API for developers to +extend and modify functionality. The software is maintained by NASA +and used as part of several uPSP systems deployed at its small- and large-scale wind tunnel +test facilities. + +:numref:`sls-screenshot` shows one frame of a video animation of the pressure-time history produced +by the uPSP software for a test event at the NASA Ames 11-ft transonic wind tunnel +of the Space Launch System (SLS) Block 1B crew configuration (`source `_). + +.. _sls-screenshot: +.. figure:: _static/sls-pressure-time-history-screenshot.png + + One frame of a video animation of time-varying surface pressure measured by uPSP. + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + installation + quick-start + applications + file-formats + terminology + dependencies + swdd + known-issues + citing + _apidoc/upsp + +.. rubric:: Indices and tables: + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/sphinx/installation.rst b/docs/sphinx/installation.rst new file mode 100644 index 0000000..060711c --- /dev/null +++ b/docs/sphinx/installation.rst @@ -0,0 +1,64 @@ +============ +Installation +============ + +The project does not currently provide pre-built binary releases. Users must +build the project from source on their local system. The project is built as +a Python package with native applications and extension modules written in C/C++. + +The following instructions will help the user build and install the package into +their local Python environment as a ``pip`` package. The package build system +leverages ``scikit-build`` and CMake to compile native C/C++ elements and deploy +them alongside pure Python modules in the final package. + +Prerequisites +============= + +- The project is currently tested for use on Linux x86-64 target systems. +- See :ref:`dependencies:Third-Party Dependencies` for a list of third-party libraries that must be installed + on the local system in order to build the project. They can be managed using the + system package manager (``apt`` on Ubuntu, ``yum`` on CentOS, etc.) or with a C/C++ + package management framework---the package maintainers at NASA develop and test using + `vcpkg `_. If using ``vcpkg``, the dependencies will likely not be in a default system folder and + so the following environment variables should be set to point the build system at the + CMake toolchain file provided by your ``vcpkg`` install: + + .. code:: bash + + #!/bin/bash + export SKBUILD_CONFIGURE_OPTIONS=" -DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" + +- Python must be installed on the local system (3.7+), ideally inside a virtual environment. +- We require ``pip>=22.3.1``. To upgrade, you can run ``pip install --upgrade pip``. + +Obtaining source code +===================== + +You can clone the repository from (TODO), or download the latest source code archive from (TODO). + +Build and install +================= + +Navigate to the source code folder tree and run + +.. code:: bash + + pip install -v . + +This will build the ``upsp`` package and install it in your Python environment. Several executable +entrypoints will be available in your local Python environment ``bin`` folder after install. + +To check successful install, you can run ``psp_process -h`` to display its application usage, and run +``python -c "import upsp.raycast; import pydoc; print(pydoc.render_doc(upsp.raycast.BVH))"`` to display +the Python API usage statement for one of the ``upsp`` C/C++ Python extension modules. + +Unit Tests +========== + +Unit tests for C/C++ modules are written with Google Test and are built as part of the packaging process. + +- After running ``pip install``, you should have a local folder called + ``_skbuild`` that caches the results of the build, for example, + ``_skbuild/linux-x86_64-3.9/cmake-build`` +- Navigate to ``cpp/test`` and then run ``../../_skbuild/linux-x86_64-3.9/cmake-build/run_tests`` + (Some unit tests rely on relative paths to test data located in ``cpp/test``). diff --git a/docs/sphinx/known-issues.rst b/docs/sphinx/known-issues.rst new file mode 100644 index 0000000..bf37f92 --- /dev/null +++ b/docs/sphinx/known-issues.rst @@ -0,0 +1,30 @@ +============ +Known Issues +============ + +- Processing not robust to cases where targets defined in the + ``*.tgts`` file are under the surface of the model grid + + - Some targets can have 3D locations that are defined such that they + lie under the surface of the wind tunnel model grid. Thus, these + targets will be rejected in phase 0 processing and will not be + used as part of the registration process. + - The current workaround is to perform preprocessing of the + ``*.tgts`` file to ensure it is consistent with the model grid + file (for instance, the locations can be offset in the direction + of the nearest model grid surface normal by a small distance until + they lie outside the grid surface) + +- Image registration can cause poor performance when wind tunnel model + has a significant amount of motion + + - The pixel-to-grid projection in Phase 0 is computed based on the + first camera frames, and is then re-used for all camera frames; + subsequent frames are “warped” to align with the first camera + frame prior to projection, but tests for occlusion and obliqueness + of model grid nodes are left unmodified + - However, at the edges of the model visible to a given camera, when + the model is moving significantly, parts of the model may move in- + and out- of view of the camera + - These model edges may then have degraded uPSP measurement accuracy + when the model has lots of motion (e.g., at high Mach number). diff --git a/docs/sphinx/make.bat b/docs/sphinx/make.bat new file mode 100644 index 0000000..153be5e --- /dev/null +++ b/docs/sphinx/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/sphinx/quick-start.rst b/docs/sphinx/quick-start.rst new file mode 100644 index 0000000..8bafb41 --- /dev/null +++ b/docs/sphinx/quick-start.rst @@ -0,0 +1,307 @@ +=========== +Quick Start +=========== + +In general, there are two ways to run the uPSP processing applications: + +- :ref:`Processing one test condition `: Useful for initial checkout of raw + data from a wind tunnel test setup. The user manually runs each step + in the pipeline. +- :ref:`Batch processing multiple test conditions `: Useful after initial + checkout. A framework auto-generates scripts that will invoke each + pipeline step for each wind tunnel test condition. The user controls + the pipeline via top-level “launcher” scripts. + +.. _sec-single-processing: + +Producing surface pressure time history for one test condition +============================================================== + +The following instructions allow a user to process a single test +conditions with the uPSP software. The steps will use bracketed +shorthand like ```` to refer to the path to input files on +disk (for more details concerning file formats, see :doc:`file-formats`). + +#. Acquire the following input files: + + - ````: High-speed video of your test subject (:ref:`sec-video-file`) + - ````: Test subject 3D model (:ref:`sec-grid-file`) + - ````: Registration targets and fiducials (:ref:`sec-tgts-file`) + - ````: Steady-state surface pressure (:ref:`sec-steady-file`) + - ````: Time-averaged wind tunnel conditions (:ref:`sec-wtd-file`) + - ````: Unsteady gain paint calibration coefficients (:ref:`sec-upsp-gain-file`) + - ````: Camera calibration parameters (one file per camera) (:ref:`sec-cam-cal-file`) + +#. Create the following processing configuration files: + + - ````: Configuration for ``upsp-external-calibration`` (:ref:`sec-excal-file`) + - ````: Input deck for ``psp_process`` (:ref:`sec-input-deck`) + +#. Run ``upsp-extract-frames`` to extract the first frame from the camera video file as a PNG. + + .. code:: bash + + #!/bin/bash + upsp-extract-frames \ + -input= \ + -output=first-frame.png \ + -start=1 \ + -count=1 + + Run ``upsp-extract-frames -h`` for more usage details. + + The output file ``first-frame.00001.png`` contains the first frame + from the input ````, scaled to 8-bit depth. + +#. Run ``upsp-external-calibration`` to compute the external camera + calibration relative to the model position in the first frame (this + step accounts for “wind-on” deflection of the model position that + is not accounted for in the time-averaged model position reported by + the wind tunnel data systems): + + .. code:: bash + + #!/bin/bash + upsp-external-calibration \ + --tgts \ + --grd \ + --wtd \ + --cfg \ + --cal_dir \ + --out_dir . \ + --img first-frame.00001.png + + ```` refers to the name of the directory containing + ````. Run ``upsp-external-calibration -h`` for more usage + details. + + The output file ``cam01-to-model.json`` contains the external + calibration parameters that will be fed to ``psp_proces`` in the next + step. + +#. Run ``psp_process`` to produce a (usually quite large) time history of + the surface pressure at each point on the model grid. This can take + significant time and memory if run on a personal computer; we recommend + instead that the application be run in parallel on a compute cluster. + The application is best run so that one MPI rank runs on each compute node, + and then thread-level parallelism is leveraged by each MPI rank to scale + across cores on a given node. + + An example PBS job script for running ``psp_process`` on the Pleaides cluster at the + NASA Advanced Supercomputing (NAS) Division is shown below. For more details + about PBS syntax and uPSP-specific rules of thumb for sizing the job allocation, + see :ref:`sec-nas-parameters`. + + .. code:: bash + + #!/bin/bash + #PBS -q normal + #PBS -l select=40:model=ivy + #PBS -l walltime=00:20:00 + export MPI_DSM_DISTRIBUTE=off + export OMP_STACKSIZE=250M + export OMP_NUM_THREADS=16 + source /usr/local/lib/global.profile + module purge + module load mpi-hpe/mpt.2.25 + mpiexec psp_process \ + -input_file= \ + -h5_out=/output.h5 \ + -paint_cal= \ + -steady_p3d= + + ```` refers to the value of the ``@output/dir`` variable + specified in the ````. Run ``psp_process -h`` for more + usage details. + + The output ``pressure_transpose`` file contains the surface pressure + time history for each node on the model grid (see :ref:`sec-pressure-transpose-file`). + Several diagnostic images are printed to verify the external calibration and (optional) + fiducial patches align well with the position of the model in the + first video frame. + +#. (Optional) post-processing steps + + - Run ``add_field`` to add the ``pressure_transpose`` data into the + HDF5 file produced by ``psp_process``. For some of its command + line arguments, ``add_field`` must be provided with the number of + vertices in the 3D model and the number of frames that were + processed. Example usage below shows how to obtain these values + from inspecting files output by ``psp_process`` in the BASH script + language. ```` should be replaced with the same directory + as ```` in the previous step (the directory containing + outputs from ``psp_process``). + + .. code:: bash + + #!/bin/bash + + # Set output_dir to the folder containing outputs from `psp_process` + # in previous step. + output_dir= + trans_h5_file=$output_dir/output.h5 + + # Inspect the 'X' file, which is a flat binary dump of the + # X-coordinates of the input wind tunnel model grid vertices. + # The number of coordinates in the file gives the size of the model. + data_item_size=4 # coordinates stored as 4-byte float's + model_size="$(expr $(stat --printf="%s" $output_dir/X) '/' $data_item_size)" + trans_flat_file="$(find $output_dir -type f -name 'pressure_transpose')" + trans_flat_file_size="$(stat --printf="%s" $trans_flat_file)" + trans_flat_file_number_frames="$(expr $trans_flat_file_size '/' '(' $model_size '*' $data_item_size ')')" + + echo ">>> number of model nodes: $model_size" + echo ">>> data item size: $data_item_size" + echo ">>> time history flat file: '$trans_flat_file'" + echo ">>> time history flat file size: $trans_flat_file_size" + echo ">>> time history flat file number of frames: $trans_flat_file_number_frames" + echo ">>> adding flat-file data to '$trans_h5_file'" + + ex="add_field $trans_h5_file frames $trans_flat_file $trans_flat_file_number_frames" + echo ">>> running: '$ex'" + t_start="$(date +%s.%N)" + echo ">>> started at: $(date)" + $ex + t_end="$(date +%s.%N)" + echo ">>> elapsed time: $(python -c "print('%4.1f' % ($t_end - $t_start))") seconds" + + echo ">>> run-add-field DONE." + +.. _sec-batch-processing: + +Batch processing multiple test conditions +========================================= + +The following instructions allow a user to batch process one or more +test conditions from a wind tunel test with the uPSP software. + +Batch processing is configured by ``upsp-make-processing-tree``, a tool +that auto-generates a file tree and associated command-line scripts that +the user can then run to execute each step in the uPSP pipeline for one +or more datapoints. The configuration process is illustrated in +:numref:`flowchart-batch-processing` and consists of the following steps: + +#. The user locates raw data files from a wind tunnel test on disk +#. The user prepares four Javascript Object Notation (JSON) configuration files: + + - A **datapoint index**, listing the path to each raw input file for + each datapoint + + - This often consists of writing test-specific scripts/tools to + grok each input file on disk + + - A **processing parameters file**, containing parameter settings + for each step in the pipeline + - A **PBS job parameters file**, containing PBS scheduler + settings (group ID, reservation wall time, number of nodes, etc.) + - A **plotting parameters file**, containing parameters for plotting + steps in the pipeline + +#. The user runs ``upsp-make-processing-tree`` and provides it with each + configuration file. The script will autogenerate a file tree on disk + to store all artifacts for batch processing + +Once the processing tree is generated and saved to disk, the user +can navigate to the ``03_launchers`` subfolder and trigger each step +in the pipeline as follows: + +#. Each step in the pipeline is launched using a script named + ``step+<+optional-subtask-name>``. + + - They should be run in the order given here (some steps use outputs + from previous steps): + + 1. ``step+extract-first-frame``: extract the first frame from each + camera video file. + 2. ``step+external-calibration``: run the wind-on, first-frame + external calibration for each camera. + 3. ``step+psp_process+psp-process``: run ``psp_process`` - + image-to-grid projection and calibration to units of pressure. + 4. ``step+psp_process+add-field``: post-process ``psp_process`` + outputs; add largest pressure-time history dataset into the + HDF5 output file. + + - Each step launcher script can be invoked as follows: + + - ``./ ...`` + to process a specific subset of datapoints. By default, all + datapoints are processed. + - ``./qsub-step ...`` + to launch the step on the cluster as one or more jobs (uses + ``qsub``). The jobs can then be monitored using + ``qstat -u $USER``. The jobs reservations are configured using + the PBS job parameters supplied in the PBS job parameters JSON file. + +#. Once complete, data products for each datapoint will be available + under ``04_products/00_data//``. + +The JSON file format was chosen for batch processing configuration files +due to its ubiquitous usage in industry and broad +cross-platform/cross-language support. Users should be familiar with +plain-text editing of JSON files and can reference the official JSON +syntax `here `__. + +.. _flowchart-batch-processing: +.. figure:: _static/flowchart-batch-processing.png + :alt: uPSP NAS batch processing flowchart. + :name: fig:flowchart + :width: 100.0% + + uPSP NAS batch processing flowchart. + +.. _sec-nas-parameters: + +Guidelines for setting NAS PBS job parameters +============================================= + +For complete details and tutorials, see the HECC wiki, `“Running Jobs with +PBS” `__. + +Specific to the current implementation of the uPSP processing code, the following +is rationale for practical "rules of thumb" for scaling the size of the PBS job to +your input data size. Trial-and-error may be required to define these parameters +correctly after initial best-guesses. + +Given the following variable definitions: + +- :math:`N_R``: Number of MPI ranks +- :math:`N_C`: Number of cameras +- :math:`F_R`: Camera resolution (pixels) +- :math:`N_M`: Number of 3D model grid nodes +- :math:`N_T`: Number of time samples + +Then, tl;dr a rule-of-thumb is to ensure each MPI rank has access to at least +:math:`M_T = M_C + M_1 + M_2` bytes of local memory, where + +- :math:`M_C = O\left(K_C (N_T N_C F_R)/N_R\right)` accounts for storage of camera frames +- :math:`M_1 = O\left(K_1 (N_T N_M)/N_R\right)` accounts for storage of the 3D-projected data (pixel counts) +- :math:`M_2 = O\left(K_2 (N_T N_M)/N_R\right)` accounts for storage of the calibrated, 3D-projected data (physical pressure units) + +and :math:`K_C = 2`, :math:`K_1 = 8`, and :math:`K_2 = 8` are reasonable constant "fudge factors" accounting for +variability in camera bit depth and intermediate storage of elements of the solution in memory. + +A practical application of this rule of thumb is as follows: + +- At NASA, for one example test condition, we collected 60K frames from 4x cameras, each approximately 1MP resolution +- The wind tunnel 3D model had approximately 1M vertices +- So: + + - :math:`M_C \approx 2 \cdot 60K \cdot 4 \cdot 1M / N_R \approx 5E11 / N_R` + - :math:`M_1 \approx 8 \cdot 60K \cdot 1M / N_R \approx 5E11 / N_R` + - :math:`M_2 \approx 8 \cdot 60K \cdot 1M / N_R \approx 5E11 / N_R` + - :math:`M_T \approx 1.5E12 / N_R` (bytes) + +- We can use 40 MPI ranks, one per compute node, so the memory requirement per + compute node is approximately 1.5E12 / 40 = 3.75E10 bytes, or 37.5GB. The NAS + Ivy nodes each have `64GB of memory available `_, + so we can fit our job into a PBS session with 40 Ivy nodes and 40 MPI ranks. + From practical experience, we know this job takes less than 10 minutes of wall clock time, + so we can use the following PBS directive to allocate the job: + + .. code:: bash + + #PBS -lselect=40:model=ivy:walltime:00:10:00 + + +Rationale for the rule of thumb is based on complexity analysis of the current algorithm implementation, +described in more detail in :doc:`swdd`. diff --git a/docs/sphinx/refs.bib b/docs/sphinx/refs.bib new file mode 100644 index 0000000..ddc4d97 --- /dev/null +++ b/docs/sphinx/refs.bib @@ -0,0 +1,254 @@ +%% This BibTeX bibliography file was created using BibDesk. +%% https://bibdesk.sourceforge.io/ + +%% Created for Marc Shaw-Lecerf at 2022-12-09 09:14:51 -0500 + + +%% Saved with string encoding Unicode (UTF-8) + + +@comment{jabref-meta: databaseType:bibtex;} + + + +@inproceedings{Bremner-2022, + author = {Paul Bremner and Marc Shaw-Lecerf and Nettie Roozeboom and Jie Li}, + booktitle = {Spacecraft and Launch Vehicle Dynamic Environments Workshop, Virtual Event}, + date-added = {2022-12-09 08:44:36 -0500}, + date-modified = {2022-12-09 09:14:44 -0500}, + month = {June}, + title = {Noise Reduction and Calibration of Unsteady Pressure-Sensitive Paint for High Resolution Measurement of Aero-acoustic Loads}, + year = {{2022}}} + +@inproceedings{Panda-CEAS-2016, + author = {Jayanta Panda and Nettie H. Roozeboom and James C. Ross}, + booktitle = {22nd AIAA/CEAS Aeroacoustics Conference}, + date-added = {2022-10-28 14:49:45 -0400}, + date-modified = {2022-10-28 14:52:23 -0400}, + doi = {10.2514/6.2016-3007}, + eprint = {https://arc.aiaa.org/doi/pdf/10.2514/6.2016-3007}, + month = {May}, + number = {AIAA 2016-3007}, + publisher = {American Institute of Aeronautics and Astronautics}, + title = {Wavenumber-Frequency Spectra of Pressure Fluctuations Measured via Fast Response Pressure Sensitive Paint}, + url = {https://arc.aiaa.org/doi/abs/10.2514/6.2016-3007}, + year = {2016}, + bdsk-url-1 = {https://arc.aiaa.org/doi/abs/10.2514/6.2016-3007}, + bdsk-url-2 = {https://doi.org/10.2514/6.2016-3007}} + +@inproceedings{Murakami-SciTech-2023, + author = {David Murakami and Marc Shaw-Lecerf and E. Lara Lash and Kenneth Lyons and Nettie Roozeboom}, + booktitle = {{AIAA} {SCITECH} 2023 Forum}, + date-added = {2022-10-28 13:26:20 -0400}, + date-modified = {2022-10-28 13:28:29 -0400}, + month = {jan}, + publisher = {American Institute of Aeronautics and Astronautics}, + title = {Implementation of the Lifetime Method in Unsteady Pressure-Sensitive Paint Measurements}, + year = {2023 (unpublished)}, + bdsk-url-1 = {https://doi.org/10.2514/6.2022-0141}} + +@inproceedings{Steva-Aviation-2019, + author = {Thomas B. Steva and Victoria J. Pollard and Andrew Herron and William A. Crosby}, + booktitle = {{AIAA} Aviation 2019 Forum}, + date-added = {2022-10-18 15:04:38 -0400}, + date-modified = {2022-10-18 15:08:29 -0400}, + doi = {10.2514/6.2019-3303}, + eprint = {https://arc.aiaa.org/doi/pdf/10.2514/6.2019-3303}, + month = {June}, + publisher = {American Institute of Aeronautics and Astronautics}, + title = {Space Launch System Aeroacoustic Wind Tunnel Test Results}, + url = {https://arc.aiaa.org/doi/abs/10.2514/6.2019-3303}, + year = {2019}, + bdsk-url-1 = {https://arc.aiaa.org/doi/abs/10.2514/6.2019-3303}, + bdsk-url-2 = {https://doi.org/10.2514/6.2019-3303}} + +@article{Corcos-1963, + author = {Corcos,G. M.}, + date-added = {2022-10-18 14:18:48 -0400}, + date-modified = {2022-10-18 14:19:22 -0400}, + doi = {10.1121/1.1918431}, + eprint = {https://doi.org/10.1121/1.1918431}, + journal = {The Journal of the Acoustical Society of America}, + number = {2}, + pages = {192-199}, + title = {Resolution of Pressure in Turbulence}, + url = {https://doi.org/10.1121/1.1918431}, + volume = {35}, + year = {1963}, + bdsk-url-1 = {https://doi.org/10.1121/1.1918431}} + +@incollection{Blake-2017-Vol2-Ch2, + abstract = {The flow-induced pressures generated by turbulent boundary layers cause vibration and, ultimately, sound. The convected nature of the pressures has led to considerable sophistication in the mathematical modeling of boundary-layer-induced vibration and sound. The work in this general area has been applied, and is of direct importance, to the prediction and reduction of cabin noise in aircraft fuselages and automobile interiors, the vibration of reentry vehicles, sound in sonar domes, and vibrations and sound generated in pipes and ducts. This chapter will develop various ways of describing the forcing function, discussing the nature of the cross-spectral and wave vector properties of surface pressures generated by turbulent flow over smooth or rough surfaces that relate to the production of sound and vibration.}, + author = {William K. Blake}, + booktitle = {Mechanics of Flow-Induced Sound and Vibration, Volume 2 (Second Edition)}, + date-added = {2022-10-18 14:10:27 -0400}, + date-modified = {2022-10-18 14:10:55 -0400}, + doi = {https://doi.org/10.1016/B978-0-12-809274-3.00002-7}, + edition = {Second Edition}, + editor = {William K. Blake}, + isbn = {978-0-12-809274-3}, + keywords = {Autospectrum, Corcos spectrum, equilibrium boundary layers, frequency-space cross correlations, self-serving boundary layers, space--time correlation, turbulent boundary layers, wall pressure, wave number spectra}, + pages = {81-177}, + publisher = {Academic Press}, + title = {Chapter 2 - Essentials of Turbulent Wall Pressure Fluctuations}, + url = {https://www.sciencedirect.com/science/article/pii/B9780128092743000027}, + year = {2017}, + bdsk-url-1 = {https://www.sciencedirect.com/science/article/pii/B9780128092743000027}, + bdsk-url-2 = {https://doi.org/10.1016/B978-0-12-809274-3.00002-7}} + +@inproceedings{Li-SciTech-2022, + author = {Jie Li and Lara Lash and Nettie Roozeboom and Theodore J. Garbeff and Christopher Henze and David Murakami and Nathanial Smith and Jennifer Baerny and Lawrence Hand and Marc Shaw-Lecerf and Paul Stremel and Lucy Tang}, + booktitle = {{AIAA} {SCITECH} 2022 Forum}, + doi = {10.2514/6.2022-0141}, + month = {jan}, + publisher = {American Institute of Aeronautics and Astronautics}, + title = {Dynamic Mode Decomposition of Unsteady Pressure-Sensitive Paint Measurements for the {NASA} Unitary Plan Wind Tunnel Tests}, + year = {2022}, + bdsk-url-1 = {https://doi.org/10.2514/6.2022-0141}} + +@inproceedings{Soranna-SciTech-2021, + author = {Francesco Soranna and Martin K. Sekula and Patrick S. Heaney and James M. Ramey and David J. Piatak}, + booktitle = {{AIAA} Scitech 2021 Forum}, + doi = {10.2514/6.2021-1834}, + month = {jan}, + publisher = {American Institute of Aeronautics and Astronautics}, + title = {Coherence Analysis of the Space Launch System using Unsteady Pressure Sensitive Paint}, + year = {2021}, + bdsk-url-1 = {https://doi.org/10.2514/6.2021-1834}} + +@inproceedings{Roozeboom-SciTech-2016, + author = {Nettie Roozeboom and Scott M. Murman and Laslo Diosady and Nathan J. Burnside and James C. Ross}, + booktitle = {54th {AIAA} Aerospace Sciences Meeting}, + doi = {10.2514/6.2016-2017}, + month = {jan}, + publisher = {American Institute of Aeronautics and Astronautics}, + title = {Unsteady {PSP} Measurements on a Rectangular Cube}, + year = {2016}, + bdsk-url-1 = {https://doi.org/10.2514/6.2016-2017}} + +@inproceedings{Roozeboom-SciTech-2017, + author = {Nettie Roozeboom and Jennifer K. Baerny}, + booktitle = {55th {AIAA} Aerospace Sciences Meeting}, + doi = {10.2514/6.2017-1055}, + month = {jan}, + publisher = {American Institute of Aeronautics and Astronautics}, + title = {Customer Guide to Pressure-Sensitive Paint Testing at {NASA} Ames Unitary Plan Wind Tunnels}, + year = {2017}, + bdsk-url-1 = {https://doi.org/10.2514/6.2017-1055}} + +@inproceedings{Sellers-SciTech-2017, + author = {Marvin E. Sellers and Michael A. Nelson and Nathan J. Burnside and Nettie Roozeboom}, + booktitle = {55th {AIAA} Aerospace Sciences Meeting}, + doi = {10.2514/6.2017-1402}, + month = {jan}, + publisher = {American Institute of Aeronautics and Astronautics}, + title = {Evaluation of Unsteady Pressure Sensitive Paint Use for Space Launch Vehicle Buffet Determination}, + year = {2017}, + bdsk-url-1 = {https://doi.org/10.2514/6.2017-1402}} + +@inproceedings{Panda-SciTech-2017, + author = {Jayanta Panda and Nettie Roozeboom and James C. Ross}, + booktitle = {55th {AIAA} Aerospace Sciences Meeting}, + doi = {10.2514/6.2017-1406}, + month = {jan}, + publisher = {American Institute of Aeronautics and Astronautics}, + title = {Wavenumber-Frequency Spectra of Pressure Fluctuations on a Generic Space Vehicle Measured via Fast-Response Pressure-Sensitive Paint}, + year = {2017}, + bdsk-url-1 = {https://doi.org/10.2514/6.2017-1406}} + +@inproceedings{Roozeboom-SciTech-2018, + author = {Nettie Roozeboom and Christina Ngo and Jessica M. Powell and Jennifer Baerny and David Murakami and James C. Ross and Scott Murman}, + booktitle = {2018 {AIAA} Aerospace Sciences Meeting}, + doi = {10.2514/6.2018-1031}, + month = {jan}, + publisher = {American Institute of Aeronautics and Astronautics}, + title = {Data Processing Methods for Unsteady Pressure-Sensitive Paint Application}, + year = {2018}, + bdsk-url-1 = {https://doi.org/10.2514/6.2018-1031}} + +@inproceedings{Roozeboom-Aviation-2019, + author = {Nettie Roozeboom and Jessica Powell and Jennifer Baerny and David Murakami and Christina Ngo and Theodore J. Garbeff and James C. Ross and Ross Flach}, + booktitle = {{AIAA} Aviation 2019 Forum}, + doi = {10.2514/6.2019-3502}, + month = {jun}, + publisher = {American Institute of Aeronautics and Astronautics}, + title = {Development of Unsteady Pressure-Sensitive Paint Application on {NASA} Space Launch System}, + year = {2019}, + bdsk-url-1 = {https://doi.org/10.2514/6.2019-3502}} + +@inproceedings{Powell-SciTech-2020, + author = {Jessica M. Powell and Scott M. Murman and Christina Ngo and Nettie Roozeboom and David D. Murakami and Jennifer K. Baerny and Ji Lie}, + booktitle = {{AIAA} Scitech 2020 Forum}, + doi = {10.2514/6.2020-0292}, + month = {jan}, + publisher = {American Institute of Aeronautics and Astronautics}, + title = {Development of Unsteady-{PSP} Data Processing and Analysis Tools for the {NASA} Ames Unitary 11ft Wind Tunnel}, + year = {2020}, + bdsk-url-1 = {https://doi.org/10.2514/6.2020-0292}} + +@inproceedings{Roozeboom-SciTech-2020, + author = {Nettie Roozeboom and Jennifer K. Baerny and David D. Murakami and Christina Ngo and Jessica M. Powell}, + booktitle = {{AIAA} Scitech 2020 Forum}, + doi = {10.2514/6.2020-0516}, + month = {jan}, + publisher = {American Institute of Aeronautics and Astronautics}, + title = {Recent Developments in {NASA}{\textquotesingle}s Unsteady Pressure-Sensitive Paint Capability}, + year = {2020}, + bdsk-url-1 = {https://doi.org/10.2514/6.2020-0516}} + +@inproceedings{Soranna-Aviation-2020, + author = {Francesco Soranna and Patrick S. Heaney and Martin K. Sekula and David J. Piatak and James M. Ramey and Nettie Roozeboom and David D. Murakami and Jennifer K. Baerny and Jie Li and Paul M. Stremel and Jessica M. Powell}, + booktitle = {{AIAA} {AVIATION} 2020 {FORUM}}, + doi = {10.2514/6.2020-2685}, + month = {jun}, + publisher = {American Institute of Aeronautics and Astronautics}, + title = {Analysis of Buffet Forcing Functions Estimated from Unsteady Pressure Sensitive Paint}, + year = {2020}, + bdsk-url-1 = {https://doi.org/10.2514/6.2020-2685}} + +@inproceedings{Tang-Aviation-2021, + author = {Lucy Tang and Lawrence Hand and David Murakami and Nettie Roozeboom and Marc Shaw-Lecerf}, + booktitle = {{AIAA} {AVIATION} 2021 {FORUM}}, + doi = {10.2514/6.2021-2579}, + month = {jul}, + publisher = {American Institute of Aeronautics and Astronautics}, + title = {{U}nsteady {P}ressure-{S}ensitive-{P}aint {S}hot {N}oise {R}eduction}, + year = {2021}, + bdsk-url-1 = {https://doi.org/10.2514/6.2021-2579}} + +@inproceedings{Panda-Aviation-2021, + author = {Jayanta Panda}, + booktitle = {{AIAA} {AVIATION} 2021 {FORUM}}, + doi = {10.2514/6.2021-2922}, + month = {jul}, + publisher = {American Institute of Aeronautics and Astronautics}, + title = {Effect of Electronic Shot Noise on Dynamic Measurements Using Optical Techniques: Examples from Rayleigh scattering and Unsteady {PSP}}, + year = {2021}, + bdsk-url-1 = {https://doi.org/10.2514/6.2021-2922}} + +@inproceedings{Sellers-SciTech-2016, + author = {Marvin Sellers and Michael Nelson and Jim W. Crafton}, + booktitle = {54th {AIAA} Aerospace Sciences Meeting}, + doi = {10.2514/6.2016-1146}, + month = {jan}, + publisher = {American Institute of Aeronautics and Astronautics}, + title = {Dynamic Pressure-Sensitive Paint Demonstration in {AEDC} Propulsion Wind Tunnel 16T}, + year = {2016}, + bdsk-url-1 = {https://doi.org/10.2514/6.2016-1146}} + +@incollection{Blake-2017-Vol2-Ch3, + abstract = {Phased arrays of pressure transducers have found broad application in both airborne and underwater acoustic technology. This chapter examines several examples of the responses of transducers, transducer arrays, and elastic structures to wave number-distributed turbulent boundary layer wall pressures and diffuse sound. The responses of planar surfaces in air with minimum fluid loading, fluid-loaded panels, and surfaces made of elastomeric materials are all discussed. This chapter also considers both rough and smooth wall boundary layers and incident acoustic fields as notionally occurring with an elastic panel in the wall of a reverberant chamber.}, + author = {William K. Blake}, + booktitle = {Mechanics of Flow-Induced Sound and Vibration, Volume 2 (Second Edition)}, + doi = {https://doi.org/10.1016/B978-0-12-809274-3.00003-9}, + edition = {Second Edition}, + editor = {William K. Blake}, + isbn = {978-0-12-809274-3}, + keywords = {Flexible panels, flow-excited structural vibration, hydrodynamic coincidence, modal vibration, response function, transducers, transducer arrays}, + pages = {179-296}, + publisher = {Academic Press}, + title = {Chapter 3 - Response of Arrays and Structures to Turbulent Wall Flow and Random Sound}, + url = {https://www.sciencedirect.com/science/article/pii/B9780128092743000039}, + year = {2017}, + bdsk-url-1 = {https://www.sciencedirect.com/science/article/pii/B9780128092743000039}, + bdsk-url-2 = {https://doi.org/10.1016/B978-0-12-809274-3.00003-9}} diff --git a/docs/sphinx/swdd.rst b/docs/sphinx/swdd.rst new file mode 100644 index 0000000..79ed012 --- /dev/null +++ b/docs/sphinx/swdd.rst @@ -0,0 +1,636 @@ +=============== +Software Design +=============== + +The following sections describe the overall software +design and architecture, including design rationale. +We especially highlight impacts on computational performance and complexity. + +Software Architecture +===================== + +The primary function of the unsteady Pressure-Sensitive Paint (uPSP) +software is to process raw wind tunnel test data to produce time +histories of flowfield pressure measurements on the surface of a wind +tunnel model. This function is implemented using a set of software +applications that can be run from a UNIX command-line interface (CLI): + +1. ``upsp-extract-frames``: Dumps individual frames from, or transcode + video format of, high-speed camera video files. +2. ``upsp-external-calibration``: Computes external camera calibration + relative to the model position and orientation as viewed in a single + camera frame. The first frame from each video file (dumped using + ``upsp-extract-frames``) is used to calibrate each camera’s + positioning relative to the model at the start time of the test + condition +3. ``psp_process``: Projects image pixel values from each video frame + onto a 3D grid representation of the wind tunnel test model + surface, and convert into surface pressure values + +The computationally-intensive processing is primarily within +``psp_process``, which is a monolithic, highly-parallelized C++ +application (``psp_process``). + +In addition to the primary processing applications, a Python-based +preprocessing application, ``upsp-make-processing-tree``, is provided to +allow users to configure inputs to ``psp_process`` for batch processing +in the NAS environment. + +Pipeline Application Design Details +=================================== + +The following section describes each pipeline application in more +detail, including: + +- **Functional flow**: Flow of data into, throughout, and out of the + application +- **Algorithms**: Algorithms used by the application +- **Implementation details**: Details related to requirements imposed + on target compute systems — memory management, disk usage, + parallelization considerations +- **Design rationale**: Context for architectural/design decisions + related to the application + +``upsp-extract-frames`` +----------------------- + +The ``upsp-extract-frames`` application helps extract individual frames +from supported high-speed camera video files and save them in more portable +image file format(s). It can also encode segments of the video file into more +portable video format(s). It makes use of the OpenCV ``VideoWriter`` and ``imsave`` +API elements, meaning it can encode images and video in formats supported by the +OpenCV installation. + +The application's primary use in the processing pipeline is to extract the +first frame from the video file, which is then used by ``upsp-external-calibration``. + +``upsp-external-calibration`` +----------------------------- + +The external calibration pipeline implements a coarse and refined stage to +iteratively improve the calibration quality. For both stages, the goal +is to match 3D target positions with image locations and optimize the +extrinsic parameters such that the re-projection error is minimized. The +coarse stage takes an initial guess for the position and orientation +(pose) based on the wind-off model position and updates the guess. The +refined stage uses the coarse solution and further refines it. +Typically, the initial guess has a re-projection error > 5 pixels, the +coarse solution has an error of ~1 pixels, and the refined solution is < +1 pixel. + +Coarse Stage +^^^^^^^^^^^^ + +The coarse stage begins with the inputs described in the uPSP User +Manual. The process is outlined in :numref:`flowchart-external-calibrate-coarse`. +The first steps of the coarse stage are to get the wind-off visible targets, and detect image targets. +Targets must be detected since the re-projection error from the wind-off +external calibration guess is typically high, and the expected image +positions can be far enough that the expected location is not within the +associated target. This can cause ambiguity with matching, or errors in +matching to image noise. + +.. _flowchart-external-calibrate-coarse: +.. figure:: _static/flowchart-external-calibrate-coarse.png + :width: 100% + + External Calibrate (Coarse) Functional Flowchart. + +To get the wind-off visible targets, the wind-off camera-to-model +extrinsics must be found. The wind-off camera-to-model extrinsics are +obtained by generating a model-to-tunnel transform, and combing it with +the camera-to-tunnel transform found in the camera-to-tunnel calibration +file. The model-to-tunnel transform is generated from the WTD file and +tunnel/model properties in the external camera calibration parameters. +With the wind-off extrinsics known, the Bounding Volume Hierarchy (BVH) +visibility checker module is used to find the wind-off visible targets. +See :ref:`swdd:BVH Visibility Checker` for details on how the visibility check is +performed. + +The detected image targets are found using `OpenCV’s blob +detection `__. +The parameters for blob detection are saved in the external camera +calibration parameters. The image is first pre-processed by scaling the +image intensity between 0 and the largest inlier pixel intensity. +Details on the pre-processing algorithm are available in :ref:`swdd:Image Pre-Processing`. + +The wind-off visible targets, and the detected image targets can then be +matched and filtered. Details on the matching and filtering process are +available in :ref:`swdd:Matching and Filtering`. + +Once matched and filtered, the remaining matches have the detected image +target sub-pixel localized. Details on the sub-pixel localization +algorithm are available in :ref:`swdd:Sub-Pixel Localization`. The resulting +targets and sub-pixel image locations are processed with OpenCV’s +PnPRansac to determine the extrinsic parameters that minimize the +re-projection error. These first stage extrinsic parameters are known as +the coarse external calibration. + +Refined Stage +^^^^^^^^^^^^^ + +The refined stage begins with the same inputs, but the added benefit of +having the coarse external calibration. The process is outlined in +:numref:`flowchart-external-calibrate-refined`. The refined stage has the +same general steps as the coarse stage: get the visible targets and +image locations, match and filter them, sub-pixel localize, and +PnPRansac. However, instead of using blob detection, projection is used +for the image locations. Projection is used here since the re-projection +error from the coarse external calibration is typically small, and the +projected locations almost always lie within the associated target. + +.. _flowchart-external-calibrate-refined: +.. figure:: _static/flowchart-external-calibrate-refined.png + :width: 100% + + External Calibrate (Refined) Functional Flowchart. + +The first steps of the refined stage are to find the visible targets, +and the projected locations of those visible targets based on the coarse +external calibration. The same BVH as used in the coarse stage is used +in the refined stage. The matching stage is trivial since the projected +locations are generated 1:1 from the visible targets. Projection is done +using `OpenCV’s +projectPoints `__. + +With the visible targets and their projected locations, the same +filtering process used in the coarse stage is used here. The image +locations are then sub-pixel localized, and `OpenCV’s +PnPRansac `__ +is used on the visible targets and sub-pixel localized image locations. +The second stage external calibration parameters are known as the +refined external calibration, and are written to the external +calibration output file to be used in ``psp_process``. + +.. _algorithms-1: + +Algorithms +~~~~~~~~~~ + +The algorithms used in the external calibration are the image +pre-processing, BVH visibility checker, matching and filtering, and +sub-pixel localization. + +Image Pre-Processing +^^^^^^^^^^^^^^^^^^^^ + +The image pre-processing is used in an attempt to normalize the pixel +intensity of the model across all data points. Due to variations in wind +speed, and degradation of the paint due to UV exposure, the model can be +significantly brighter or darker day to day or tunnel condition to +tunnel condition. To normalize this, the image intensity is scaled from +0 to the largest inlier pixel intensity. This is done by converting the +image to a floating point ``numpy.ndarray`` (rather than a ``uint8`` or ``uint16``), +dividing by the largest inlier pixel intensity, clipping the maximum +value to 1, then multiplying by 255 (or 4095 if using 16-bit). + +The largest inlier pixel intensity is defined as the largest value in +the sorted list of pixel intensities where a substantially far pixel +must is at least 90% the intensity as the current pixel. Substantially +far is defined as 0.001 \* the current pixel’s index. + +It is easier to see in the code: + +:: + + i = len(img_flat_sorted) - 1 + while (0.9 * img_flat_sorted[i] > img_flat_sorted[int(np.rint(i * 0.999))]): + i -= 1 + max_val = img_flat_sorted[i] + +So in a 1024 x 512 image with 524,288 pixels, in order for the brightest +pixel to be considered an inlier (position 524,287 in the sorted list), +the value at position 523,763 must be at least 90% of its intensity. If +it is not, the second brightest pixel is checked, and this continues +down until an inlier is found. + +This intensity check relative to other pixel intensities ensures that a +small number of very bright pixels do not cause the scaling of the image +to be controlled by that small group. Should a small number of pixels be +very high due to saturation from glare, or hot pixels due to sensor +error, those will be deemed outliers and have their intensity clipped to +256 (or 4095). + +BVH Visibility Checker +^^^^^^^^^^^^^^^^^^^^^^ + +The Bounding Volume Hierarchy (BVH) checks the visibility of a point +with known position and normal vector. This can be a grid node, target, +or other point of interest so long as it has a position and normal +vector. + +To determine if a point is visible, the BVH visibility checker first +checks if that point has an oblique viewing angle greater than that +specified in the external calibration parameters. Typically, a value of +70° is used for the maximum allowable oblique viewing angle since points +with oblique viewing angles greater than that become difficult to view +due to perspective distortion. Points that pass the check are then +passed to the BVH to check for occlusions. + +The oblique viewing angle is defined as the angle between the point’s +normal vector, and the vector from the point to the camera. If the point +fails that check, it is immediately deemed not visible. In reality, it +may be visible if the oblique viewing angle is between 70°, and 90°. +However, above 70° and the point experiences significant perspective +distortion. For grid nodes, this means poor pixel intensity association +and thus a pressure with large uncertainty. For targets, this means a +large sub-pixel localization error. This operation is similar to +back-face culling, and would be identical the back-face culling if the +oblique viewing angle was set to 90°. Just as with back-face culling, +the oblique viewing angle check is significantly less expensive than the +occlusion checks. Therefore, oblique viewing angle is checked first +since any points removed will not have to undergo the expensive +occlusion checks. With 70°, on average about 60% of the points will be +rejected. + +Points that pass the oblique viewing angle check are passed to the +bounding volume hierarchy to check for occlusions. The BVH is a +recursive data structure that can efficiently check for the intersection +between a mesh and a ray (O(logn) where n is the number of mesh nodes). +The mesh in this case is the model grid, and the ray is the ray between +the point and the camera. The ray origin is actually taken to be the +point location, plus a small distance (1e-4”) along the point’s normal +vector rather than the point’s location directly. This ensures that if +the point is exactly on the model surface (or even inside the model by a +small amount), it is not wrongfully deemed occluded. + +Points that pass both the oblique viewing angle check and the occlusion +check are deemd visible. Note, for target the point is typically taken +to be the center location. This assumes that the if the center of the +target is visible, then all of the target is visible. For mesh nodes, +usually all vertices of the node are checked. If all are visible, it is +assumed that the entire 2D surface of the node is visible. These are +reasonable assumptions since the targets, and especially the mesh nodes, +are relatively small. So if the center is visible it is very likely that +the entire target/node is visible. + +Matching and Filtering +^^^^^^^^^^^^^^^^^^^^^^ + +The matching process matches 3D targets to detected image locations. To +do this, the 3D targets are projected into the image. Each projected +location is then matched to the nearest detected image target. Once all +visible targets are matched, any matches that are not one-to-one +(detected image targets matched to multiple projected locations) are +thrown out. Additionally, matches are thrown out if the pixel distance +between the projected location and detection image target location is +over the max_dist threshold (specified in the external camera +calibration parmeters). Matches are further filtered if any 2 image +locations are closer than the min_dist threshold (specified in the +external camera calibration parmeters). + +.. raw:: html + + + +Sub-Pixel Localization +^^^^^^^^^^^^^^^^^^^^^^ + +The sub-pixel localization fits a `2D +Super-Gaussian `__ +distribution to a cropped region around an image target. The idea being +to improve a rough localization found with blob detection or 3D +projection. A 2D Super-Gaussian approximate the form of an image target, +and therefore the 2D mean location can be taken to be the target’s +center location. For (ideal) sharpie targets with diameter ~4-5 pixels, +the median error is ~0.05 pixels, and is within 0.265 pixels 99.9% of +the time. + +The algorithm is performed by defining a 2D Super Gaussian function, +then optimizing the gaussian function parameters to the cropped image +region using `Scipy’s +curvefit `__ +module. + +Design Rationale +~~~~~~~~~~~~~~~~ + +Ideally, the external calibration routine would begin with a very close +initial guess. Close here can refer to a max re-projection error of ~1 +pixel. With a low re-projection error, the targets could be projected +into the image, then the region around the projected location could be +passed through the sub-pixel localization algorithm. The targets and +sub-pixel localized image locations could then be passes to OpenCV’s +PnPRansac. This would be akin to performing just the refined external +calibration stage from the initial guess. However, using the wind-off +pose yields a max re-projection error > 3 pixels and > 5 pixels in some +cases. + +The radius of the sharpie targets is ~2.5 pixels, so in many cases the +projected locations are not even inside the sharpie target. In order for +the cropped region of the image passed to the sub-pixel localization to +contain the entire target, the radius of the cropped region would have +to be ~9 pixels (5 pixels to contain the center of the sharpie target +plus 2.5 pixels to contain the whole target, plus 1.5 pixels to have a +pixel between the edge and the target). A region that large is likely to +pick up noise, image background, or other features that would cause the +sub-pixel localization routine to produce bad results. Therefore, a two- +stage, iterative method is used instead. + +The first stage (coarse optimization) uses blob detection to find the +target image locations rather than projection. The projected locations +will still be close to the detected locations to likely be correct and +unambiguous. The goal of the first stage is to set up the ‘ideal’ +initial guess previously mentioned. The blob detection typically has a +recall of ~90%, and a precision of ~30%. This means ~90% of the targets +are found, and for every target found there are ~2 false positives. +While this seems high, most of the false positives are in the background +or far from the sharpie targets. This means that most false positives do +not interfere with the matching to real image targets. After the +matching, typically ~60% of the sharpie targets are correctly matched. + +The second stage (refined optimization) uses the external calibration +from the first stage, and uses projection as if it was the ‘ideal’ case. +Typically, the second stage makes use of > 95% of the sharpie targets. +Some filtering is still implemented for situations where the image is +particularly dark, or makes particularly bad use of the camera’s dynamic +range. Monte Carlo simulations of the external calibration routine have +shown that the use of a second stage typically cuts the external +calibration uncertainty in half. Since the second stage is not +particularly expensive, the trade-off of additional processing time is +well worth it. + +It was decided to not combine sharpie targets and unpainted Kulites as +targets since the addition of Kulites did not significantly reduce +uncertainty, significantly increases computation time, and opens the +door for significant errors. The Kulites have worse sub-pixel +localization error than the sharpie targets since they roughly 1/4 the +pixel area. Therefore, even with many of them, combining ~50 Kulites +with ~20 sharpie targets only decreases the uncertainty by ~6% +(according to the Monte Carlo simultations). However, computation time +scales roughly linearly (or can be slightly super-linear due to PnP +RANSAC) and so it roughly triples the refined stage’s computation time. +Additionally, it is common for the sub-pixel localization routine to +optimize on the wrong image feature since noise or a scratch on the +model only has to be ~2 pixels in diameter to be the same size as the +Kulite. With all this considered, when sharpie targets are present it is +highly recommended to only use sharpie targets. + +``psp_process`` +--------------- + +:numref:`flowchart-psp-process` presents the functional data flow for the +``psp_process`` application. The processing is divided into three “phases”: + +- Phase 0: initialization; camera calibration. +- Phase 1: camera image pixel values projected onto 3D wind tunnel + model grid; conversion from image pixel values into intensity ratios; + image registration. +- Phase 2: conversion from intensity ratios to pressure ratios; write + final unsteady pressure ratio time-histories to file. + +.. _flowchart-psp-process: +.. figure:: _static/flowchart-psp-process.png + :width: 100% + + ``psp_process`` functional flowchart. + +Phase 1 processing +^^^^^^^^^^^^^^^^^^ + +Phase 1 processing maps the camera intensity data onto the model grid +using the camera-to-model registration information. Before projection, it +also interpolates the intensity data over "patches" identified in the +image plane corresponding to small, unpainted regions on the model surface. + +The output data from Phase 1 is essentially a large matrix, or +“solution”, containing the intensity value at each model node, at each +frame (*i.e.*, time step). The intensity solution is maintained +in-memory between Phase 1 and Phase 2 processing, distributed across one +or more computational nodes. + +Steps involved are as follows: + +1. A projection matrix is developed for each camera that maps pixels + from the first camera frame onto the model grid nodes, starting with + the camera-to-model registration solution from Phase 0. + + 1. A ray-tracing process is executed to identify and exclude portions + of the model grid that are: + + - not visible to the camera, or + - too oblique-facing relative to the camera line of sight + (obliqueness threshold angle between the model normal and the + camera line of sight is configurable in the Input Deck) + + 2. For every grid node with data from multiple cameras, combine + camera data using one of the following strategies: + + - use a linear-weighted combination of the measured value from + each camera; the sum of the weights is normalized to one, and + each weight is linearly proportional to the angle between the + surface normal and the ray from the camera to the grid node + (``average_view``) + - use the measured value from the camera with the best view angle + (``best_view``) + + 3. Any grid nodes that are not visible in any camera’s first frame + will be marked with ``NaN`` in the output solution. + +2. For each camera, and for each camera frame: + + 1. Because the model may have some small amount of motion between + frames, first “warp” the image to align with the first camera + frame. The warping process uses a pixel-based image registration + scheme assuming affine transformations. + 2. Fill in "fiducial" marking regions with polynomial patches from + Phase 0. Note that the same pixel coordinates can be used for the + polynomial patches for all frames because the previous step aligns + each frame with the first frame. + 3. Apply the projection matrix to the aligned-and-patched image + frame, to obtain the model grid node intensity values. + 4. For each time frame, sum the intensity values over each camera + using the previously established weighting strategy. + +Phase 2 processing +^^^^^^^^^^^^^^^^^^ + +Phase 2 processing maps the Phase 1 intensity solution to an equivalent +solution in physical pressure units. + +Currently the only method for converting intensity to pressure that has +been implemented in the software is the `method devised at Arnold +Engineering Development Complex +(AEDC) `__ that uses pre-test paint +calibrations and the steady state PSP solutions. + +The gain is computed at each grid node with Equation :eq:`gain`, where +:math:`T` is the surface temperature in :math:`^{\circ}F` and +:math:`P_{ss}` is the steady state PSP pressure in psf. The coefficients +(a-f) are specified in the paint calibration file. + +.. math:: + :label: gain + + Gain = a + bT + cT^2 + (d + eT + fT^2) P_{ss} + +The surface temperature is estimated to be the equilibrium temperature. +This calculation is shown in Equation :eq:`recoverytemp`, where +:math:`T_0` is the stagnation temperature, :math:`T_{\infty}` is the +freestream temperature, and :math:`r` is the turbulent boundary-layer +recovery factor (0.896), given by Schlichting. + +.. math:: + :label: recoverytemp + + T = r(T_0 - T_{\infty}) + T_{\infty} + +Before applying the gain, the data is detrended by fitting a +6\ :math:`^{th}`-order polynomial curve to the ratio of the average +intensity over intensity for each grid node. Then, the pressure is just +the AC signal times the :math:`Gain`. This process is shown in +Equations :eq:`intensity2pressure`, where :math:`f` is a frame number, +:math:`n` is a grid node, :math:`I` is the intensity, and +:math:`\bar{q}` is the dynamic pressure in psf. + +.. math:: + :label: intensity2pressure + + \begin{aligned} + \bar{I}_n &= \sum_{f=1}^F I_{f,n} \\ \nonumber + I_{f,n}\prime &= \bar{I}_n / I_{f,n} \\ \nonumber + I_{fit}(n) &= poly\_fit(I_n\prime) \\ \nonumber + I_{fit}(f,n) &= poly\_val(I_{fit}(n), f) \\ \nonumber + P_{f,n} &= (I_{f,n}\prime - I_{fit}(f,n)) * Gain \\ \nonumber + \Delta C_p(f,n) &= P_{f,n} * 12 * 12 / \bar{q} \\ \nonumber + \end{aligned} + +FIDUCIAL PATCHING + +1. Interpolate the camera pixel data to “patch” over small, unpainted + areas on the model surface. These small areas are referred to as + “fiducials” and may correspond to registration targets from the + previous step as well as to other known visible elements such as + blemishes, mechanical fasteners, etc. + + The interpolation process relies on the following inputs: + + - Known locations and circular diameters for each fiducial on the + model surface + - An updated camera calibration from the previous step + + The interpolation process is defined as follows: + + 1. Using the updated camera calibration, project all fiducials onto + the first camera frame. Ignore any points that are either: + + - Occluded by other features + - Oblique by more than :math:`oblique\_angle + 5^{\circ}` (i.e., + the angle between the surface normal and the ray from the + camera to the node is less than + :math:`180^{\circ} - (oblique\_angle + 5^{\circ})`) + + 2. Estimate the size of the fiducial in pixels using projection and + the defined 3D fiducial size. + 3. Cluster fiducials so that no coverage patch overlaps another + fiducial. + 4. Define the boundary of each cluster as :math:`bound\_pts` rows of + pixels outside the cluster with :math:`buffer\_pts` row of pixel + as a buffer. + 5. Define a threshold below which the data is either background or + very poor: + + 1. Compute a histogram of intensities for frame 1. + 2. Find the first local minimum after the first local maximum in + the histogram. This plus :math:`5^{\circ}` is the threshold. + + 6. Remove any boundary pixels that are within 2 pixels of a pixel + that is below the threshold. + 7. Fit a 3\ :math:`^{rd}` order 2D polynomial to the boundary pixels + of each cluster, then the interior (patched pixels) are set by + evaluating the polynomial. + +Memory usage and scalability +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The intensity-time history and pressure-time history data are usually +prohibitively large to be loaded in their entirety into the memory of a +single computer (for example, for approximately 1 million grid nodes and +40,000 camera frames, each time history solution requires approximately +150 GB). Without usage of parallel processing over multiple +computational nodes, the processing would need to process “blocks” of +the solution and write the output to disk periodically to operate within +the constraints of a single computer’s available memory. + +Instead, to greatly increase speed of processing, the ``psp_process`` +application is designed to execute across an arbitrary number of +computational nodes. In practice, for current test data sets, the +application is executed across approximately 20-50 computational nodes +using the NASA Advanced Supercomputing (NAS) Pleiades cluster. The +software leverages the Message Passing Toolkit (MPT) implementation +provided by the Pleiades cluster, and its execution environment is +controlled by the NAS-maintained Portable Batch System (PBS). + +The conceptual layout of the data items manipulated during the three +processing phases is shown in :numref:`memory-model-psp-process`. Each large +time history is divided into a series of “blocks” based on the number of +available computational nodes, or “ranks.” MPT is leveraged for +communication between ranks for the following operations: + +- Computing sums or averages over the entire time history. +- Computing the transpose of each time history. + +.. _memory-model-psp-process: +.. figure:: _static/memory-model-psp-process.png + :alt: ``psp_process`` memory model. + :width: 100.0% + + ``psp_process`` memory model. + +Phase 0 processing operations are duplicated identically across each +rank for simplicity, because they do not scale with the number of camera +frames. + +The design decision to divide the processing into three phases was +driven primarily by considerations of computational complexity and +ease-of-validation. Previously, the uPSP software prototype phases were +divided into separate applications to facilitate partial processing and +caching of intermediate data products. The operations performed in Phase +0 could be run standalone in order to check camera registration outputs, +and the intensity ratio outputs from Phase 1 could be analyzed prior to +Phase 2 operations for qualitative checks of frequency content of the +uPSP measurement data (conversion to pressure ratios is a scaling +operation that does not affect time-varying or frequency content). In +addition, the Phase 1 and Phase 2 operations are computationally +intense; previous software versions were deployable to a personal laptop +or desktop computer without massive parallelization, however, the +processing required several orders of magnitude more time to complete +than with the current parallelized code. + +.. raw:: html + + + +Choice of software language(s) +============================== + +All aspects of the external calibration pipeline were written in Python, +with the exception of the BVH which uses legacy C++ code (written by Tim +Sandstrom) and a Python binding for ease of use. + +Python 3 was selected due to developer expertise and the availability +and maturity of scientific and image processing modules such as OpenCV, +Tensorflow, Numpy, and Scipy. These modules are primarily written in C, +C++, and Fortran with Python bindings. This allows for a user-friendly +development environment where developers have expertise for quick +turnaround time, while retaining fast underlying operations. diff --git a/docs/sphinx/terminology.rst b/docs/sphinx/terminology.rst new file mode 100644 index 0000000..152e1ca --- /dev/null +++ b/docs/sphinx/terminology.rst @@ -0,0 +1,31 @@ +=========== +Terminology +=========== + +- **Data point**: A single wind tunnel test condition +- **External calibration**: Also called “extrinsic” calibration; + parameters that quantify the position and orientation of a camera + coordinate frame relative to another frame (for example, relative to + a test model-fixed frame or a wind tunnel-fixed frame) +- **Internal calibration**: Also called “intrinsic” calibration; + parameters that quantify the physical properties of the camera system + such as focal length and sensor center offset. +- **Javascript Object Notation (JSON)**: Text-based file format used + for storage of structured data, used throughout uPSP software for + configuration files. +- **Outer Mold Line (OML)**: The wetted surface area of an aerodynamic + body (to be painted with PSP) +- **NASA Advanced Supercomputing Division (NAS)**: Division at NASA + Ames Research Center that houses the HECC. +- **Pleaides Front End (PFE)**: Systems in the NAS HECC enclave for + user login to interact with the HECC clusters. +- **Portable Batch Scheduler (PBS)**: System service that handles + scheduling jobs on NAS HECC clusters. +- **Wind Tunnel Data (WTD) file**: File format for storage of wind + tunnel test conditions such as Mach number, Reynolds number, model + angle of attack, model sideslip angle, and model sting support + position. +- **Wind tunnel test model (“test model”, or “model”)**: The physical + body installed in the wind tunnel for test purposes +- **Wind tunnel test model grid (“model grid”, or “grid”)**: The 3D + representation of a wind tunnel test model in a digital file format diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..2415ee9 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,29 @@ +[project] +name = "upsp" +description = "NASA uPSP data processing applications and utilities" +authors = [{name = "NASA uPSP developers"}] +license = {file = "LICENSE.txt"} +dynamic = ["version"] +dependencies = [ + "matplotlib", + "numpy", + "opencv-python-headless==4.5.2.54", + "pandas", + "scipy", +] + +[build-system] +requires = [ + "setuptools>=61", + "setuptools_scm[toml]>=6.2", + "pybind11", + "cmake>=3.22", + "scikit-build>=0.15.0", +] +build-backend = "setuptools.build_meta" + +[tool.isort] +profile = "black" + +[tool.setuptools_scm] +write_to = "python/upsp/_version.py" diff --git a/python/upsp/__init__.py b/python/upsp/__init__.py index e69de29..6fa0cb5 100644 --- a/python/upsp/__init__.py +++ b/python/upsp/__init__.py @@ -0,0 +1,6 @@ +try: + from ._version import version as __version__ + from ._version import version_tuple +except ImportError: + __version__ = "unknown" + version_tuple = (0, 0, "unknown") diff --git a/python/upsp/cam_cal_utils/camera_tunnel_calibrate.py b/python/upsp/cam_cal_utils/camera_tunnel_calibrate.py new file mode 100644 index 0000000..a1ca500 --- /dev/null +++ b/python/upsp/cam_cal_utils/camera_tunnel_calibrate.py @@ -0,0 +1,363 @@ +import numpy as np +import os +import json + +from upsp.cam_cal_utils import ( + external_calibrate, + photogrammetry, + parsers, + visualization, +) + +debug_show_3D_targets = False +debug_show_img_targets = False +debug_show_matches = True + + +def camera_to_tunnel_calibrate( + ctc_dir, + imgs, # Camera specific + internal_cals, # Camera specific + manual_detections, # Camera specific + tunnel_vals, # Datapoint specific + tgts_all, # Test specific + test_config, # Test specific + vis_checker, # Test specific + match_thresh=0.8, # Optional +): + """Performs camera to tunnel calibration for the given cameras + + Generates ``camera_to_tunnel`` json files for each given camera and saved in + ``ctc_dir``. + + Any number of cameras can be given to this function. Each camera needs its own entry + in `imgs`, `internal_cals`, and `manual_detections`. If there are 10 cameras, each + of those inputs will be a list of length 10. + + If `debug_show_matches` or `debug_show_img_targets` are True, the debug images will + be saved to the current directory. Each image will be appended with '_?' where ? is + the index of the camera (If ? is 0, it corresponds to ``imgs[0]``, + ``internal_cals[0]``, and ``manual_detections[0]``) + + All inputs should correspond to the same test configuration. The inputs + `tunnel_vals`, `tgts_all`, `test_config`, and `vis_checker` will be used across all + cameras. + + Parameters + ---------- + ctc_dir : string + Directory to save the camera-to-tunnel calibrations + imgs : list + Each image should be ``np.array`` of shape (height, width) and 8-bit. + ``imgs[i]`` should correspond to ``internal_cals[i]`` and + ``manual_detections[i]`` + internal_cals : list + Each internal calibration should be of the form:: + + [cameraMatrix, distCoeffs, sensor_resolution, sensor_size] + + - ``cameraMatrix`` is the (openCV formatted) camera matrix for the camera + - ``distCoeffs`` is the (openCV formatted) distortion coefficients for the + camera + - ``sensor_resolution`` is a tuple of the full pixel resolution of the camera + (which can be larger than the images of the `imgs` input) + - ``sensor_size`` is a tuple of the physical sensor size in inches + + ``internal_cals[i]`` should correspond to ``imgs[i]`` and + ``manual_detections[i]`` + manual_detections : list + Each manual detection using PASCAL VOC format. Each manual detection is a + dict with following the keys: + + - 'class' denoting the target_type + - 'x1' denoting the left edge of the bounding box + - 'y1' denoting the top edge of the bounding box + - 'x2' denoting the right edge of the bounding box + - 'y2' denoting the bottom edge of the bounding box + + ``manual_detections[i]`` should correspond to ``imgs[i]`` and + ``internal_cals[i]`` + tunnel_vals : dict + `tunnel_vals` has the keys ALPHA, BETA, PHI, and STRUTZ which denote the model's + commanded position in the UPWT. For tests without this type of model positioning + mechanism, set all values to 0.0 + tgts_all : list + Each target is a dict with (at a minimum) a 'target_type', 'tvec', and 'norm' + attribute. The 'target_type' is a string for the type of target, usually 'dot'. + Currently, only targets with the 'dot' type are used. The 'tvec' attribute gives + the target's location and the 'norm' attribute gives the target's normal vector + Both are ``np.array`` vectors with shape (3, 1) + test_config : dict + The dict must contain the following keys and values: + - 'oblique_angle' : maximum allowable oblique viewing angle + - 'max_match_dist' : maximum allowable matching distance between tgt + projection and image target + - 'dot_pad' : pixel padding distance around center of dot target to use for + sub-pixel localization + - 'tunnel-cor_to_tgts_tvec' : translation vector from center of rotation to + tgts origin. For tests that do not have a center of rotation like the + UPWT, set this value to [0.0, 0.0, 0.0] + - 'tunnel-cor_to_tgts_rmat' : rotation matrix from center of rotation to + tgts origin. For tests that do not have a center of rotation like the + UPWT, set this value to [[1, 0, 0], [0, 1, 0], [0, 0, 1]] + - 'tunnel-cor_to_tunnel-origin_tvec' : translation vector from center of + rotation to tunnel origin. This can be [0.0, 0.0, 0.0] + - 'tunnel-cor_to_tunnel-origin_rmat' : rotation matrix from center of + rotation to tunnel origin. This can be [[1, 0, 0], [0, 1, 0], [0, 0, 1]] + vis_checker : ~upsp.cam_cal_utils.visibility.VisibilityChecker + Visibility checker object with the relevant BVH and oblique viewing angle + match_thresh : int or float, default = 0.80 + Proportion of matched needed to form a consensus, optional default=0.8 + """ + # Make the camera-to-tunnel directory if it does not exist + os.makedirs(ctc_dir, exist_ok=True) + + # For this optimization, only take only the dots from the target inputs if possible + target_type = "dot" + tgts = [] + for tgt in tgts_all: + if tgt["target_type"] == "dot": + tgts.append(tgt) + + # If there were not enough dots, use kulites + if len(tgts) < 4: + # Set the dot_found flag to False + target_type = "kulite" + for tgt in tgts_all: + if tgt["target_type"] == "kulite": + tgts.append(tgt) + + if debug_show_3D_targets: + import matplotlib.pyplot as plt + import target_bumping + + fig = plt.figure("3D Targets") + ax = fig.add_subplot(projection="3d") + + tvecs, norms = vis_checker.get_tvecs_and_norms() + ax.scatter(tvecs[:, 0], tvecs[:, 1], tvecs[:, 2]) + + internals = target_bumping.tgts_get_internals(tgts, vis_checker) + internal_names = [internal[0] for internal in internals] + external_tgts = [] + for tgt in tgts: + if tgt["name"] not in internal_names: + external_tgts.append(tgt) + visualization.plot_pts_and_norms(external_tgts, ax) + + ax.view_init(elev=45, azim=-90) + visualization.axisEqual3D(ax) + plt.savefig("Targets_3D.png") + plt.close("3D Targets") + + # Get transformation from tunnel frame to tgts frame + rmat_tunnel_tgts, tvec_tunnel_tgts = tunnel_transform( + **tunnel_vals, tvec__cor_tgts__tgts_frame=test_config["tunnel-cor_to_tgts_tvec"] + ) + rmat_tgts_tunnel, tvec_tgts_tunnel = photogrammetry.invTransform( + rmat_tunnel_tgts, tvec_tunnel_tgts + ) + + # Run the camera-tunnel calibration for each camera + for num, img, internal_cal, manual_detection in zip( + list(range(1, len(imgs) + 1)), imgs, internal_cals, manual_detections + ): + # Unpackage the internal calibration + cameraMatrix, distCoeffs, sensor_resolution, sensor_size = internal_cal + + # Get the sub-pixel localized imag_targets + img_targets = [] + for bbox in manual_detection: + # If it is a target of the appropriate class, and not flagged as difficult + # add it to the targets list + if (bbox["class"] == target_type) and not bbox["difficult"]: + x1, y1, x2, y2 = bbox["x1"], bbox["y1"], bbox["x2"], bbox["y2"] + center = ((x1 + x2) / 2, (y1 + y2) / 2) + img_target = {"target_type": target_type, "center": center} + img_targets.append(img_target) + + # Sub-pixel localize the manual img_targets + __, img_targets = external_calibrate.subpixel_localize( + img, img_targets, img_targets, test_config + ) + + # If debug_show_img_targets is turned on, generate the debug image + if debug_show_img_targets: + img_centers = np.array([img_target["center"] for img_target in img_targets]) + img_centers = np.squeeze(img_centers) + visualization.show_image_locations( + img, img_centers, str(num) + "_Image_Center_Locations" + ) + + # Using RANSAC, find the external calibration + rmat_opt, tvec_opt = external_calibrate.external_calibrate_RANSAC( + [cameraMatrix, distCoeffs], + tgts, + img_targets, + vis_checker, + max_dist=test_config["max_dist"], + match_thresh=match_thresh, + ) + + # If debug_show_matches is turned on, generate the debug image + if debug_show_matches: + visible_init_tgts = photogrammetry.get_visible_targets( + rmat_opt, tvec_opt, tgts, vis_checker + ) + external_calibrate.match_targets( + rmat_opt, + tvec_opt, + cameraMatrix, + distCoeffs, + visible_init_tgts, + img_targets, + max_dist=test_config["max_dist"], + debug=[img, str(num), None], + ) + + # Transform from camera to tgts to tunnel -> Get camera to tunnel transformation + rmat_camera_tunnel = np.matmul(rmat_opt, rmat_tgts_tunnel) + tvec_camera_tunnel = tvec_opt + np.matmul(rmat_opt, tvec_tgts_tunnel) + + # Package the camera calibration data + uPSP_cameraMatrix = parsers.convert_cv2_cm_to_uPSP_cm(cameraMatrix, img.shape) + datum = { + "uPSP_cameraMatrix": uPSP_cameraMatrix.tolist(), + "distCoeffs": distCoeffs.tolist(), + "rmat": rmat_camera_tunnel.tolist(), + "tvec": tvec_camera_tunnel.reshape(3,).tolist(), + "sensor_resolution": sensor_resolution.tolist(), + "sensor_size": sensor_size.tolist(), + } + + # Export the calibration as a json file + cal_file = "camera" + str(num).rjust(2, "0") + ".json" + with open(os.path.join(ctc_dir, cal_file), "w") as f: + json.dump(datum, f) + + +# TODO: need to implement use of rmat__cor_tgts__tgts_frame as an input +def tunnel_transform(ALPHA, BETA, PHI, STRUTZ, tvec__cor_tgts__tgts_frame): + """Calculates the transformation from the tunnel coordinate frame to the tgts frame + + Note: This is not necessarily to the tunnel origin, just some fixed point in the + tunnel (fixed within a tunnel test). If `STRUTZ` = `STRUTZ_abs` then it will be the + tunnel origin + + Parameters + ---------- + ALPHA : float + Tunnel alpha in degrees + BETA : float + Tunnel beta in degrees + PHI : float + Tunnel phi in degrees + STRUTZ : float + STRUTZ location of the UPWT strut + tvec__cor_tgts__tgts_frame : np.ndarray, shape (3, 1), dtype float + Translation vector from the tunnel center of rotation to the tgts frame + in the tgts frame. The tgts frame is a fixed distance from the tunnel point of + rotation from the tgts frame's point of view, that translation vector is always + along the x axis + + Returns + ---------- + rotation_matrix : np.ndarray, shape (3, 3) + Rotation matrix from tgts frame to tunnel frame + tvec__tunnel_tgts__tunnel_frame : np.ndarray, shape (3, 1) + Translation vector from tgts frame to tunnel frame + """ + + # Get the component rotation matrices + + # UPWT Tunnel Coordinates are RHS Aircraft Coordinates Pitched 180 degrees + # See UPWT AIAA Coordinate Systems Training Manual For Details + + # Positive Alpha is Positive Pitch + pitch = photogrammetry.rot(-ALPHA, "y") + + # Positive Beta is Negative Yaw + yaw = photogrammetry.rot(-BETA, "z") + + # Positive Phi is Positive Roll + roll = photogrammetry.rot(PHI, "x") + + # Combine into one rotation matrix + # Matrix Multiplication Order is [P][Y][R] + rotation_matrix = np.matmul(pitch, np.matmul(yaw, roll)) + + # We want the transformation from tunnel to tgts, so get the inverse + rotation_matrix = np.linalg.inv(rotation_matrix) + + # tvec__tgts_tunnel__tunnel_frame is the translation vector from the tunnel + # frame to the tgts frame, in the tunnel frame + # I.e. Since the knuckle sleeve rotates relative to the tunnel, the model + # translation vector somewhere in the cone of allowable rotation. This + # calculates that translation. Also, the strutz can move up and down in + # the z axis, so that needs to be taken into account as well + tvec__knuckle_tgts = np.matmul(rotation_matrix, tvec__cor_tgts__tgts_frame) + tvec__tunnel_tgts__tunnel_frame = tvec__knuckle_tgts + np.array( + [[0], [0], [STRUTZ]] + ) + + return rotation_matrix, tvec__tunnel_tgts__tunnel_frame + + +# TODO: This refers specifically to a model on the sting. We will need a function for +# a floor mounted model, and ideally something in the test_config file to specify +# TODO: need to implement use of tunnel-cor_to_tgts_rmat from test_config +def tf_camera_tgts_thru_tunnel(camera_tunnel_cal, wtd, test_config): + """Returns the transformation from the camera to the model (tgts frame) + + Parameters + ---------- + camera_cal : list + camera calibration in the form:: + + [rmat__camera_tunnel, tvec__camera_tunnel, cameraMatrix, distCoeffs] + wtd : dict + wind tunnel data as a dict with (at a minimum) the keys 'ALPHA', 'BETA', 'PHI', + and 'STRUTZ'. ALPHA, BETA, and PHI are tunnel angles in degrees. STRUTZ is the + offset of the tunnel center of rotation for the z axis in inches + test_config : dict + test configuration data as a dict with (at a minimum) the key + 'tunnel-cor_to_tgts_tvec' representing the translation vector from the tunnel + center of rotation to the model frame + + Returns + ---------- + rmat__camera_tgts : np.ndarray, shape (3, 3) + Rotation matrix + tvec__camera_tgts : np.ndarray, shape (3, 1) + Translation vector + """ + + # Turn the wind tunnel data into the transformation from tunnel to targets + wtd_transform = tunnel_transform( + wtd["ALPHA"], + wtd["BETA"], + wtd["PHI"], + wtd["STRUTZ"], + test_config["tunnel-cor_to_tgts_tvec"], + ) + rmat_tunnel_tgts, tvec_tunnel_tgts = wtd_transform + + # Transformation from tgts frame to tunnel frame + rmat_tgts_tunnel = np.linalg.inv(rmat_tunnel_tgts) + tvec_tgts_tunnel = -np.matmul(rmat_tgts_tunnel, tvec_tunnel_tgts) + + # Decompose the camera calibration into its parts + ( + rmat__camera_tunnel, + tvec__camera_tunnel, + cameraMatrix, + distCoeffs, + ) = camera_tunnel_cal + + # Combine the transformations to get the transformation from `camera to tgts frame + rmat__camera_tgts = np.matmul(rmat__camera_tunnel, np.linalg.inv(rmat_tgts_tunnel)) + tvec__camera_tgts = tvec__camera_tunnel + np.matmul( + rmat__camera_tunnel, tvec_tunnel_tgts + ) + + return rmat__camera_tgts, tvec__camera_tgts diff --git a/python/upsp/cam_cal_utils/external_calibrate.py b/python/upsp/cam_cal_utils/external_calibrate.py index 0dda5a7..247da56 100644 --- a/python/upsp/cam_cal_utils/external_calibrate.py +++ b/python/upsp/cam_cal_utils/external_calibrate.py @@ -3,27 +3,18 @@ import matplotlib.pyplot as plt import numpy as np import copy -import os -import sys -import time +import warnings -current_dir = os.path.dirname(os.path.abspath(__file__)) -parent_dir = os.path.dirname(current_dir) +from upsp.target_localization import gaussian_fitting_methods +from upsp.cam_cal_utils import ( + visualization, + visibility, + camera_tunnel_calibrate, + photogrammetry, + img_utils, +) -target_loc_methods = os.path.join(parent_dir, 'target_localization') -sys.path.append(target_loc_methods) - -import gaussian_fitting_methods -# Define the gaussian fit function -gauss_fit = gaussian_fitting_methods.gauss_fitter_func('super') - -utils = os.path.join(parent_dir, 'cam_cal_utils') -sys.path.append(utils) - -import visualization -import visibility -import photogrammetry -import img_utils +gauss_fit = gaussian_fitting_methods.gauss_fitter_func("super") log = logging.getLogger(__name__) @@ -35,78 +26,110 @@ debug_refined_optimization = True debug_show_localizations = False -#--------------------------------------------------------------------------------------- +# --------------------------------------------------------------------------------------- # Other Functions - + def compare_poses(pose0, pose1, more_info=False): """Returns the angle and distance in which the two poses differ - - Any two rotation matrices are related by a single rotation of theta about a given axis + + Any two rotation matrices are related by a single rotation of theta about a given + axis. Any two translation vectors are related by an [X, Y, Z] translation vector. This function returns the angle between the two poses, as well as the distance - formatted as [theta, dist] where theta is in degrees. If more_info is True, - [theta, axis, tvec_rel] is returned where axis is the axis of rotation. - + formatted as [theta, dist] where theta is in degrees and dist is in the units of + tvec. If `more_info` is True, [theta, axis, tvec_rel] is returned where axis is the + axis of rotation. + Parameters ---------- - pose0 : - pose1 : - more_info : - + pose0, pose1 : tuple + Pose to compare: ``(rmat, tvec)``, where ``rmat`` is the rotation matrix from + camera to object (:class:`numpy.ndarray` with shape (3, 3)) and ``tvec`` is the + translation vector from camera to object (:class:`numpy.ndarray` with shape (3, + 1)). + more_info : bool, optional + Changes the return value. Function returns (theta, distance) if `more_info` is + False, function returns (rvec, tvec) if `more_info` is True + Returns - ---------- - + ------- + tuple + If `more_info` is False, returns ``(theta, distance)``. If `more_info` is True, + return is ``(rvec, tvec)`` """ # Unpack the rmats and tvecs rmat0, tvec0 = pose0 rmat1, tvec1 = pose1 - # Get true relative transformations + # Get relative transformations rmat_rel = np.matmul(rmat1, rmat0.T) tvec_rel = tvec1 - np.matmul(rmat_rel, tvec0) - + # Get the axis and theta for rmat_rel rvec_rel, _ = cv2.Rodrigues(rmat_rel) - + # OpenCV's Rodrigues vector is a compact representation where theta is the magnitude # of the vector (convert from radians to degrees) theta = np.linalg.norm(rvec_rel) theta = np.rad2deg(theta) - # If more_info was not requested, return the angle and distance + # If more_info was not requested, return only the angle and distance if not more_info: - return [theta, np.linalg.norm(tvec_rel)] - - # If more_info was requested, return the angle, axis, and full tvec + return (theta, np.linalg.norm(tvec_rel)) + + # If more_info was requested, return the Rodrigues vector and full tvec else: - # In the compact OpenCV representation, the axis is the Rodrigues vector divided - # by the angle - axis = rvec_rel / theta - return [theta, axis, tvec_rel] - -#--------------------------------------------------------------------------------------- + return (rvec_rel, tvec_rel) + + +# --------------------------------------------------------------------------------------- # Calibration Functions -def subpixel_localize_robust(img, tgts, img_targets, test_config, - max_localize_delta=None): - """Find the sub-pixel localized position of the targets in a robust manner +# TODO: make this return a num_matches, all targets, and all unique img_targets just +# like to other targets functions +def subpixel_localize(img, tgts, img_targets, test_config, max_localize_delta=None): + """Find the sub-pixel localized position of the image targets - Find the location of the target centers with sub-pixel accuracy. This method filters - common bad localization solutions. I.e Localized position is too far initial guess - to make sense, invalid optimizer solution, outside crop or outside image + Find the location of the image target centers with sub-pixel accuracy. This method + filters common bad localization solutions. I.e The localized position is too far + initial guess to make sense, invalid optimizer solution (None flag), outside the + cropped region or outside the image Parameters ---------- - img - tgts - img_targets - test_config - max_localize_delta - + img : np.ndarray, shape (h, w) + Numpy 2D array of the image + tgts : list + Matched 3D targets. Each target should be a dict. The only strict requirement + set by this function is ``tgts[i]`` is associated with ``img_targets[i]`` + img_targets : list + Matched image targets. Each dict has, at a minimum, keys 'center', and + 'target_type'. 'center' has a value of the image target center location + (tuple/np.ndarray of length 2 of floats) and 'target_type' has the key of the + type of target (string). ``img_targets[i]`` is associated with ``tgts[i]`` + test_config : dict + Processing parameters with, at a minimum, a key for each target type in + `targets` and `img_targets`. The key is `target_type` + '_pad'. This is the + padding around the img target center location to use to sub-pixel localize + max_localize_delta : float, optional + The maximum allowable distance that subpixel_localize can change the + `img_target` position. If None, the max allowable distance will be set to the + padding distance minus 2. + Returns - ---------- - + ------- + targets : list + Target positions (`tgts`) that have not been filtered out. + img_targets : list + Refined (sub-pixel localized) target positions in the image. + + Notes + ----- + ``targets[i]`` is associated with ``img_targets[i]``. Return lists may not be the + same length as input `tgts` and/or `img_targets` inputs (some + `targets`/`img_targets` from the input may be rejected and thus are not included in + the output) """ if max_localize_delta is not None: filter_dist = max_localize_delta @@ -115,35 +138,64 @@ def subpixel_localize_robust(img, tgts, img_targets, test_config, img_targets_refined = [] for i, img_target in enumerate(img_targets): # Integer of center pixel - center_pixel = np.rint((img_target['center'][0], img_target['center'][1])).astype(np.int32) - - target_pad = test_config[img_target['target_type'] + '_pad'] + center_pixel = np.rint( + (img_target["center"][0], img_target["center"][1]) + ).astype(np.int32) + + target_pad = test_config[img_target["target_type"] + "_pad"] # cropped region around target (x1, y1), (x2, y2) - bbox = [[center_pixel[0] - target_pad, center_pixel[1] - target_pad], - [center_pixel[0] + target_pad + 1, center_pixel[1] + target_pad + 1]] + bbox = [ + [center_pixel[0] - target_pad, center_pixel[1] - target_pad], + [center_pixel[0] + target_pad + 1, center_pixel[1] + target_pad + 1], + ] # If bbox goes out of bounds of the image, ignore it - if ((bbox[0][0] < 0) or (bbox[0][1] < 0) or (bbox[1][0] >= img.shape[1]) or (bbox[1][1] >= img.shape[0])): + if ( + (bbox[0][0] < 0) + or (bbox[0][1] < 0) + or (bbox[1][0] >= img.shape[1]) + or (bbox[1][1] >= img.shape[0]) + ): out_of_bounds.add(i) img_targets_refined.append( - {'target_type' : img_target['target_type'], 'center' : (-1, -1)}) + {"target_type": img_target["target_type"], "center": (None, None)} + ) continue # Cropped image around target img_cropped = img[bbox[0][1] : bbox[1][1], bbox[0][0] : bbox[1][0]] - - fake_keypoint = cv2.KeyPoint(img_target['center'][0], img_target['center'][1], _size=1) - - gauss_center = gauss_fit(img_cropped, target_type=img_target['target_type'], - keypoint=fake_keypoint, img_offset=bbox[0])[0] - - img_target_refined = {'target_type' : img_target['target_type'], - 'center' : gauss_center} - - dist = np.linalg.norm([gauss_center[0] - img_target_refined['center'][0], - gauss_center[1] - img_target_refined['center'][1]]) - + + # Perform the sub-pixel localization + gauss_center = gauss_fit( + img_cropped, + target_type=img_target["target_type"], + center=img_target["center"], + img_offset=bbox[0], + )[0] + + # If the optimizer failed, continue + if gauss_center[0] is None: + out_of_bounds.add(i) + img_targets_refined.append( + { + "target_type": img_target["target_type"], + "center": np.array((None, None)), + } + ) + continue + + # Create a new img_target with the updated center location + img_target_refined = copy.deepcopy(img_target) + img_target_refined["center"] = gauss_center + + dist = np.linalg.norm( + [ + gauss_center[0] - img_target_refined["center"][0], + gauss_center[1] - img_target_refined["center"][1], + ] + ) + # A distance of target_pad - 1 would imply the center is on a pixel in the edge # of the crop. And since the taret is larger than 1 pixel, by definition this # is a bad localization. Not even to mention it is likely bad since it is so @@ -151,81 +203,130 @@ def subpixel_localize_robust(img, tgts, img_targets, test_config, # safety if max_localize_delta is None: filter_dist = target_pad - 2 - - if (dist > filter_dist) or (gauss_center[0] == -1): + + if dist > filter_dist: out_of_bounds.add(i) img_targets_refined.append( - {'target_type' : img_target['target_type'], 'center' : (-1, -1)}) - + { + "target_type": img_target["target_type"], + "center": np.array((None, None)), + } + ) + continue + if debug_show_localizations: - plt.imshow(img_cropped, cmap='gray') - plt.scatter([gauss_center[0] - bbox[0][0]], [gauss_center[1] - bbox[0][1]], c='g', s=2) - plt.scatter([img_target['center'][0] - bbox[0][0]], [img_target['center'][1] - bbox[0][1]], c='r', s=2) - plt.savefig(str(i).rjust(3, '0') + '_' + str(np.round(dist, 3)) + '_' - + str(center_pixel[0]) + '_' + str(center_pixel[1]) + '.png') + plt.imshow(img_cropped, cmap="gray") + plt.scatter( + [gauss_center[0] - bbox[0][0]], + [gauss_center[1] - bbox[0][1]], + c="g", + s=2, + ) + plt.scatter( + [img_target["center"][0] - bbox[0][0]], + [img_target["center"][1] - bbox[0][1]], + c="r", + s=2, + ) + plt.savefig( + str(i).rjust(3, "0") + + "_" + + str(np.round(dist, 3)) + + "_" + + str(center_pixel[0]) + + "_" + + str(center_pixel[1]) + + ".png" + ) plt.close() - + img_targets_refined.append(img_target_refined) - + # Remove the targets that had a bad localization - tgts_temp = [] - img_targets_temp = [] + tgts_loc = [] + img_targets_loc = [] for i, (tgt, img_target) in enumerate(zip(tgts, img_targets_refined)): if i not in out_of_bounds: - tgts_temp.append(tgt) - img_targets_temp.append(img_target) - - return tgts_temp, img_targets_temp + tgts_loc.append(tgt) + img_targets_loc.append(img_target) + + return tgts_loc, img_targets_loc # TODO: make this return a num_matches, all targets, and all unique img_targets just # like to other targets functions -def filter_partially_occluded(rmat, tvec, focal_length, tgts, img_targets, vis_checker, test_config): - """Checks if corners of crop used for localization are occluded. - - If the corners of the crop used for sub-pixel localization jumps surfaces, the - target, then the target is likely at least partially occluded. - - To get 3D positions of corners of the cropped image, start at the target tvec. - Approximate the physical distance (in inches) of test_config['*_pad'] (which is - given in pixels) using the focal length and distance from camera to model. Take - steps along the focal plane to get to the approximate corner locations in 3D. - - With those locations, ensure the targets are not inside the model (since the model - is curved the planar focal plane may put the corners slightly inside the model. Then - check for occlusions. If the corner is occluded, the target is partially occluded. - If none of the corners are occluded, the target is likely not occluded (but is still - potentially partially occluded). +def filter_partially_occluded( + rmat, tvec, focal_length, tgts, img_targets, vis_checker, test_config +): + """Checks corners of cropped area used for sub-pixel localization for occlusion + + If the corners of the crop used for sub-pixel localization jumps surfaces, then the + target is likely partially occluded. This most commonly occurs when a booster + partially occludes a target on the core of a launch vehicle + + To get the 3D positions of corners of the cropped area, start at the target tvec. + Approximate the physical distance (in inches) of the cropped area (which is + done in pixels) using the focal length and distance from camera to model. Take + steps along the camera image plane to get to the approximate corner locations in 3D. + + With the corner locations, ensure that they are not inside the model (since the + model is curved the steps along the image plane may put the corners slightly inside + the model. Then check for occlusions. If the corner is occluded, the target is + deemed partially occluded. If none of the corners are occluded, the target is deemed + not occluded (but is still potentially partially occluded). Only the corners are + checked to reduce computation Parameters ---------- - rmat - tvec - focal_length - tgts - img_targets - vis_checker - test_config - + rmat : np.ndarray, shape (3, 3), float + Rotation matrix from camera to object + tvec : np.ndarray, shape (3, 1), float + Translation vector from camera to object + focal_length : float + Focal length of the camera. Most easily accessible from a ``cameraMatrix[0][0]`` + tgts : list + 3D targets. Each target is a dict and has, at a minimum, the keys 'tvec', + 'target_type', 'norm'. 'tvec' has a :class:`numpy.ndarray` (3, 1) representing the + position of the target relative to the model origin for its associated value. + 'norm' has a :class:`numpy.ndarray` (3, 1) representing the normal vector of the + target relative to the model origin for its associated value. 'target_type' has + a string representing the type of target (most commonly 'dot') for its + associated value. ``tgts[i]`` is associated with ``img_targets[i]`` + img_targets : list + Image targets. Each image target should be a dict, but this function does not + set any strict requirements other than that ``img_targets[i]`` is associated + with ``tgts[i]`` + vis_checker : ~upsp.cam_cal_utils.visibility.VisibilityChecker + Visibility checker object with the relevant BVH and oblique viewing angle + test_config : dict + Processign parameters with, at a minimum, a key for each target type of the + targets in `tgts` and `img_targets`. They key is `target_type` + '_pad'. This is + the padding around the img target center location to use to sub-pixel localize + Returns - ---------- - + ------- + tgts_filtered : list + Target positions (`tgts`) that have not been filtered out + img_targets_filtered : list + Target positions in the image. """ # Get the direction for the u and v of pixel space - rmat_inv, tvec_inv = photogrammetry.invTransform(rmat, tvec) - tvec = tvec.ravel() + rmat_inv, tvec_inv = photogrammetry.invTransform(rmat, tvec) + # Check each target for partial occlusions tgts_filtered = [] img_targets_filtered = [] for tgt, img_target in zip(tgts, img_targets): # Get the scale factor - obj_dist = np.linalg.norm(tvec - tgt['tvec']) + obj_dist = np.linalg.norm(tvec.T - tgt["tvec"]) # For the scale factor, we use the similar triangles of the object distance by # actual distance (in) vs focal length by pixel distance (px) # The scale factor is the object distance divided by the focal length, times the # pixel distance. We add one to the pad distance for a margin of safety - step_sz = obj_dist * (test_config[tgt['target_type'] + '_pad'] + 1) / focal_length + step_sz = ( + obj_dist * (test_config[tgt["target_type"] + "_pad"] + 1) / focal_length + ) # Get the step vector for u and v for the corners # Get the u and v steps @@ -236,16 +337,12 @@ def filter_partially_occluded(rmat, tvec, focal_length, tgts, img_targets, vis_c # Package the corners. We have 1 corner for steps in the given directions corners = [] for x, y in [(-1, -1), (-1, 1), (1, -1), (1, 1)]: - corners.append({'tvec' : tgt['tvec'] + x * u + y * v, - 'norm' : tgt['norm']}) - - # We need to check if the corners are visible. It is a relatively safe assumption - # that since the targets are visible (and pass the back-face culling) that - # the corners pass the back face culling as well. So we only check for true - # occlusions + corners.append({"tvec": tgt["tvec"] + x * u + y * v, "norm": tgt["norm"]}) - # Get the occlusions. For each corner this has a boolean and an occlusion position - occlusions = photogrammetry.get_occlusions_targets(rmat, tvec, corners, vis_checker) + # Check for occlusions. Each corner has a boolean and an occlusion position + occlusions = photogrammetry.get_occlusions_targets( + rmat, tvec, corners, vis_checker + ) # Check that all corners are visible are_all_corners_visible = True @@ -254,35 +351,36 @@ def filter_partially_occluded(rmat, tvec, focal_length, tgts, img_targets, vis_c if not occlusion[0]: continue - # If there was an occlusion, check distance from the occlusion to the corner + # If there was an occlusion, check the distance from the occlusion to corner # If that distance is less than sqrt(2) * step_sz (since we step in both # x and y) it is possible that the corner is inside the model so it is # unfairly occluded. # If the distance is greater this corner is just occluded, no questions - dist = np.linalg.norm(occlusion[1] - corner['tvec']) - if (dist > step_sz * np.sqrt(2)): + dist = np.linalg.norm(occlusion[1] - corner["tvec"]) + if dist > step_sz * np.sqrt(2): are_all_corners_visible = False break # To fairly check this corner, bump it to the occluded location bumped_corner = copy.copy(corner) - bumped_corner['tvec'] = occlusion[1] + bumped_corner["tvec"] = occlusion[1] # Check this bumped corner for occlusion bumped_occlusion = photogrammetry.get_occlusions_targets( - rmat, tvec, [bumped_corner], vis_checker)[0] + rmat, tvec, [bumped_corner], vis_checker + )[0] # If there is still an occlusion, this corner is likely occluded - # There is a chance that the distance between the bumped occlusion and + # There is a chance that the distance between the bumped occlusion and # the original corner is still less than step_sz * np.sqrt(2). However - # the frequency seems small and not worthy of further checks. + # the occurrence seems small and not worthy of further checks. # This is a potential TODO to set up as a while loop if bumped_occlusion[0]: are_all_corners_visible = False break - - # Otherwise, this corner is fine and we can move onto the next corner + # If all corners passed the visibility check, this corner is fine and we can + # move onto the next corner if are_all_corners_visible: tgts_filtered.append(tgt) img_targets_filtered.append(img_target) @@ -290,33 +388,58 @@ def filter_partially_occluded(rmat, tvec, focal_length, tgts, img_targets, vis_c return tgts_filtered, img_targets_filtered -def filter_min_dist(tgts, matching_points, num_matches, min_dist=8, kwd='center'): - """Filters target projections that are too close together in the image +def filter_min_dist(tgts, img_targets, num_matches, min_dist=8): + """Filters the targets and image target that are too close together in the image If the image locations of any two targets are below the min_dist threshold, remove - both targets from the set of matched targets. + both targets from the set of matched targets. This is to avoid ambiguity in target + matching. Parameters ---------- - tgts - matching_points - num_matches - min_dist - kwd - + tgts : list + 3D targets. Each target should be a dict. The only strict requirement set by + this function is ``tgts[i]`` is associated with ``img_targets[i]`` for i from 0 + to `num_matches` + img_targets : list + Matched image targets. Each dict has, at a minimum, the key 'center' which has a + value of the image target center location (tuple/np.ndarray of floats). + ``img_targets[i]`` is associated with ``tgts[i]`` for i from 0 to `num_matches` + num_matches : int + An integer of the number of matches. Must be less than or equal to + ``min(len(tgts), len(img_targets))`` + min_dist : float, optional + The minimum distance between two image targets. Any image targets closer than + this distance are filtered out. + Returns - ---------- - + ------- + tgts_filtered : list + Target positions (`tgts`) that have been filtered so that their image locations + are separated by a distance larger than min_dist. + img_targets_filtered : list + Target positions in the image. + num_matches_filtered : int + + Notes + ----- + ``tgts_filtered[i]`` is associated with ``img_targets_filtered[i]`` for i from 0 to + `num_matches_filtered`. """ + # Find the image targets that are too close together supressed = set() for i in range(num_matches): - for j in range(i+1, num_matches): + for j in range(i + 1, num_matches): if (i in supressed) and (j in supressed): continue # Calculate the distance between point i and j - dist = np.linalg.norm([matching_points[i][kwd][0] - matching_points[j][kwd][0], - matching_points[i][kwd][1] - matching_points[j][kwd][1]]) + dist = np.linalg.norm( + [ + img_targets[i]["center"][0] - img_targets[j]["center"][0], + img_targets[i]["center"][1] - img_targets[j]["center"][1], + ] + ) # If the distance is below the given threshold, add both i and j to the list # of supressed points @@ -324,240 +447,328 @@ def filter_min_dist(tgts, matching_points, num_matches, min_dist=8, kwd='center' supressed.add(i) supressed.add(j) continue - - tgts_temp = [] - matching_points_temp = [] + + # First add targets that pass the min_dist check + tgts_filtered = [] + img_targets_filtered = [] for i in range(num_matches): if i in supressed: continue - tgts_temp.append(tgts[i]) - matching_points_temp.append(matching_points[i]) - - num_matches_temp = len(tgts_temp) - for tgt in tgts: - if tgt not in tgts_temp: - tgts_temp.append(tgt) - for mp in matching_points: - if mp not in matching_points_temp: - matching_points_temp.append(mp) + tgts_filtered.append(tgts[i]) + img_targets_filtered.append(img_targets[i]) - return tgts_temp, matching_points_temp, num_matches_temp + # The number of matches is the number after the filter + num_matches_filtered = len(tgts_filtered) + # Then add the remaining targets and image targets + tgts_filtered, img_targets_filtered = post_filter_append( + tgts, tgts_filtered, img_targets, img_targets_filtered + ) -def filter_bifilter(rmat, tvec, cameraMatrix, distCoeffs, tgts, matching_points, num_matches, max_dist): - """Consistency check to filer potential mismatches + return (tgts_filtered, img_targets_filtered, num_matches_filtered) - Check that only 1 img target is within a radius of max_dist from every projected + +def filter_bifilter( + rmat, tvec, cameraMatrix, distCoeffs, tgts, img_targets, num_matches, max_dist +): + """Filters the targets and image target for ambiguity + + Check that only 1 img target is within a radius of max_dist from every projected target location. If more than 1 img target is near a target, do not use that target - + Check that only 1 projected target location is within a radius of max_dist from every img target. If more than 1 projected target location is near an img target, do not use that img target Parameters ---------- - rmat - tvec - cameraMatrix - distCoeffs - tgts - matching_points - num_matches - max_dist - - Returns - ---------- + rmat : np.ndarray, shape (3, 3), float + Rotation matrix from camera to object + tvec : np.ndarray, shape (3, 1), float + Translation vector from camera to object + cameraMatrix : np.ndarray, shape (3, 3), float + The (openCV formatted) camera matrix for the camera + distCoeffs : np.ndarray, shape (1, 5), float + The (openCV formatted) distortion coefficients for the camera + tgts : list + 3D targets. Each target is a dict and has, at a minimum, the keys 'tvec', and + 'target_type'. 'tvec' has a :class:`numpy.ndarray` (3, 1) representing the + position of the target relative to the model origin for its associated value. + 'target_type' has a string representing the type of target (most commonly 'dot') + for its associated value. ``tgts[i]`` is associated with ``img_targets[i]`` for + i from 0 to `num_matches` + img_targets : list + Matched image targets. Each dict has, at a minimum, the key 'center' which has a + value of is the image target center location (tuple/np.ndarray of floats). + ``img_targets[i]`` is associated with ``tgts[i]`` for i from 0 to `num_matches` + num_matches : int + An integer of the number of matches. Must be less than or equal to + ``min(len(tgts), len(img_targets))`` + max_dist : float + The maximum distance between two image targets. Any image targets farther than + this distance are filtered out. + Returns + ------- + tgts_bifilter : list + Target positions (`tgts`) that have been bifiltered. + img_targets_bifilter : list + Target positions in the image. + num_matches_bifilter : int + + Notes + ----- + ``tgts_matched_bifilter[i]`` is associated with ``img_targets_bifilter[i]`` for i + from 0 to `num_matches_bifilter`. """ - # Project the points into the image - tgt_projs = photogrammetry.project_targets(rmat, tvec, cameraMatrix, distCoeffs, tgts) + tgt_projs = photogrammetry.project_targets( + rmat, tvec, cameraMatrix, distCoeffs, tgts + ) # Check that all targets have at most 1 img target nearby - tgts_matched_temp, matching_points_temp = [], [] + tgts_matched_temp, img_targets_temp = [], [] num_matches_temp = 0 for i in range(num_matches): tgt_proj = tgt_projs[i] - + bifilter_key = True - for j, mp in enumerate(matching_points): + for j, img_tgt in enumerate(img_targets): if i == j: continue - dist = np.linalg.norm(np.array(tgt_proj['proj']) - np.array(mp['center'])) - + dist = np.linalg.norm(tgt_proj["proj"] - img_tgt["center"]) + # If the distance is less than max_dist, this target fails the bifilter if dist < max_dist: bifilter_key = False break - + if bifilter_key: tgts_matched_temp.append(tgts[i]) - matching_points_temp.append(matching_points[i]) + img_targets_temp.append(img_targets[i]) num_matches_temp += 1 - for tgt in tgts: - if tgt not in tgts_matched_temp: - tgts_matched_temp.append(tgt) - for mp in matching_points: - if mp not in matching_points_temp: - matching_points_temp.append(mp) + tgts_matched_temp, img_targets_temp = post_filter_append( + tgts, tgts_matched_temp, img_targets, img_targets_temp + ) # Project the points into the image tgt_projs = photogrammetry.project_targets( - rmat, tvec, cameraMatrix, distCoeffs, tgts_matched_temp) + rmat, tvec, cameraMatrix, distCoeffs, tgts_matched_temp + ) # Check that all img targets have at most 1 target nearby - tgts_matched_bifilter, matching_points_bifilter = [], [] + tgts_bifilter, img_targets_bifilter = [], [] num_matches_bifilter = 0 for i in range(num_matches_temp): - mp = matching_points_temp[i] - + img_tgt = img_targets_temp[i] + bifilter_key = True for j, tgt_proj in enumerate(tgt_projs): if i == j: continue - dist = np.linalg.norm(np.array(mp['center']) - np.array(tgt_proj['proj'])) - + dist = np.linalg.norm( + np.array(img_tgt["center"]) - np.array(tgt_proj["proj"]) + ) + # If the distance is less than max_dist, this target fails the bifilter if dist < max_dist: bifilter_key = False break - + if bifilter_key: - tgts_matched_bifilter.append(tgts_matched_temp[i]) - matching_points_bifilter.append(matching_points_temp[i]) + tgts_bifilter.append(tgts_matched_temp[i]) + img_targets_bifilter.append(img_targets_temp[i]) num_matches_bifilter += 1 - for tgt in tgts: - if tgt not in tgts_matched_bifilter: - tgts_matched_bifilter.append(tgt) - for mp in matching_points: - if mp not in matching_points_bifilter: - matching_points_bifilter.append(mp) + tgts_bifilter, img_targets_bifilter = post_filter_append( + tgts, tgts_bifilter, img_targets, img_targets_bifilter + ) - return tgts_matched_bifilter, matching_points_bifilter, num_matches_bifilter + return tgts_bifilter, img_targets_bifilter, num_matches_bifilter -def filter_one2one(tgts, matching_points, num_matches): - """Ensures that each target is matched with at most 1 img target, and that each img +def filter_one2one(tgts, img_targets, num_matches): + """Filters the targets and image target to ensure matches are one-to-one + + Ensures that each target is matched with at most 1 img target, and that each img target is matched with at most 1 target. Remove anything that is not 1:1 as stated - + Parameters ---------- - tgts - matching_points - num_matches - + tgts : list + 3D targets. Each target should be a dict. The only strict requirement set by + this function is ``tgts[i]`` is associated with ``img_targets[i]`` for i from 0 + to `num_matches` + img_targets : list + Image targets. Each image target should be a dict, but this function does not + set any strict requirements other than that ``img_targets[i]`` is associated + with ``tgts[i]`` for i from 0 to `num_matches` + num_matches : int + An integer of the number of matches. Must be less than or equal to + ``min(len(tgts), len(img_targets))`` + Returns - ---------- - + ------- + tgts_filtered_one2one : list + Target positions (`tgts`) that have been filtered for pairs that are one-to-one. + img_targets_one2one : list + Target positions in the image. + num_matches_one2one : int + + Notes + ----- + ``tgts_one2one[i]`` is associated with ``img_targets_one2one[i]`` for i from 0 to + `num_matches_one2one`. """ # Check that each img_target is used at most once tgts_filtered_temp = [] - matching_points_temp = [] + img_targets_temp = [] num_matches_temp = 0 for i in range(num_matches): # Unpack the target and match - tgt, match = tgts[i], matching_points[i] + tgt, match = tgts[i], img_targets[i] # Create a copy of the all matching points minus the current - matching_points_subset = copy.deepcopy(matching_points[:num_matches]) - del matching_points_subset[i] + img_targets_subset = copy.deepcopy(img_targets[:num_matches]) + del img_targets_subset[i] + + # Convert the numpy arrays to lists for comparisons + match_temp = copy.deepcopy(match) + match_temp["center"] = match_temp["center"].flatten().tolist() + for j in range(len(img_targets_subset)): + img_targets_subset[j]["center"] = ( + img_targets_subset[j]["center"].flatten().tolist() + ) # If match is not in the list, there is a 1:1 matching and it can be included - if match not in matching_points_subset: + if match_temp not in img_targets_subset: tgts_filtered_temp.append(tgt) - matching_points_temp.append(match) + img_targets_temp.append(match) num_matches_temp += 1 - matching_points_subset = None # deleteme + img_targets_subset = None # deleteme # Check that each img_target is used at most once tgts_filtered_one2one = [] - matching_points_one2one = [] + img_targets_one2one = [] num_matches_one2one = 0 for i in range(num_matches_temp): # Unpack the target and match - tgt, match = tgts_filtered_temp[i], matching_points_temp[i] + tgt, match = tgts_filtered_temp[i], img_targets_temp[i] # Create a copy of the all matching points minus the current tgts_subset = copy.deepcopy(tgts_filtered_temp[:num_matches_temp]) del tgts_subset[i] + # Convert the numpy arrays to lists for comparisons + tgt_temp = copy.deepcopy(tgt) + tgt_temp["tvec"] = tgt_temp["tvec"].flatten().tolist() + tgt_temp["norm"] = tgt_temp["norm"].flatten().tolist() + for j in range(len(tgts_subset)): + tgts_subset[j]["tvec"] = tgts_subset[j]["tvec"].flatten().tolist() + tgts_subset[j]["norm"] = tgts_subset[j]["norm"].flatten().tolist() + # If match is not in the list, there is a 1:1 matching and it can be included - if tgt not in tgts_subset: + if tgt_temp not in tgts_subset: tgts_filtered_one2one.append(tgt) - matching_points_one2one.append(match) + img_targets_one2one.append(match) num_matches_one2one += 1 - for tgt in tgts: - if tgt not in tgts_filtered_one2one: - tgts_filtered_one2one.append(tgt) - for mp in matching_points: - if mp not in matching_points_one2one: - matching_points_one2one.append(mp) + tgts_filtered_one2one, img_targets_one2one = post_filter_append( + tgts, tgts_filtered_one2one, img_targets, img_targets_one2one + ) - return tgts_filtered_one2one, matching_points_one2one, num_matches_one2one + return tgts_filtered_one2one, img_targets_one2one, num_matches_one2one -def filter_max_dist(rmat, tvec, cameraMatrix, distCoeffs, tgts_matched, matching_points, num_matches, max_dist): - """Filter matches where the projected target and imag target are too far. +def filter_max_dist( + rmat, tvec, cameraMatrix, distCoeffs, tgts, img_targets, num_matches, max_dist +): + """Filters the targets and image target to pairs are not very far apart Any match where the distance between the projected target pixel position and the img target center is greater than max_dist is removed. - + Parameters ---------- - rmat - tvec - cameraMatrix - distCoeffs - tgts_matched - matching_points - num_matches - max_dist - + rmat : np.ndarray, shape (3, 3), float + Rotation matrix from camera to object + tvec : np.ndarray, shape (3, 1), float + Translation vector from camera to object + cameraMatrix : np.ndarray (3, 3), float + The (openCV formatted) camera matrix for the camera + distCoeffs : np.ndarray, shape (1, 5), float + The (openCV formatted) distortion coefficients for the camera + tgts : list + 3D targets. Each target is a dict and has, at a minimum, the keys 'tvec', and + 'target_type'. 'tvec' has a :class:`np.ndarray` (3, 1) representing the position + of the target relative to the model origin for its associated value. + 'target_type' has a string representing the type of target (most commonly 'dot') + for its associated value. ``tgts[i]`` is associated with ``img_targets[i]`` for + i from 0 to `num_matches`. + img_targets : list + Matched image targets. Each dict has, at a minimum, the key 'center' which has a + value of is the image target center location (tuple/np.ndarray of floats). + ``img_targets[i]`` is associated with ``tgts[i]`` for i from 0 to num_matches + num_matches : int + An integer of the number of matches. Must be less than or equal to + ``min(len(tgts), len(img_targets))`` + max_dist : float + The maximum matching distance between a 3D target's project and an image targets + center. Pairs with matching distance greater this distance are filtered out. + Returns - ---------- - + ------- + tgts_filtered : list + Target positions (`tgts`) that have been filtered such that the matching + distance is less than max_dist. + img_targets_filtered : list + Target positions in the image. + num_matches_filtered : int + + Notes + ----- + ``tgts_filtered[i]`` is associated with ``img_targets_filtered[i]`` for i from + 0 to `num_matches_filtered`. """ # Project the points into the image tgt_projs = photogrammetry.project_targets( - rmat, tvec, cameraMatrix, distCoeffs, tgts_matched - ) + rmat, tvec, cameraMatrix, distCoeffs, tgts + ) - tgts_matched_temp, matching_points_temp = [], [] + # First, add the targets that are within max_dist from their image targets + tgts_filtered, img_targets_filtered = [], [] for i in range(num_matches): - tgt_proj, img_target = tgt_projs[i], matching_points[i] - - dist = np.linalg.norm(np.array(tgt_proj['proj']) - np.array(img_target['center'])) + tgt_proj, img_target = tgt_projs[i], img_targets[i] + + dist = np.linalg.norm( + np.array(tgt_proj["proj"]) - np.array(img_target["center"]) + ) if dist < max_dist: - tgts_matched_temp.append(tgts_matched[i]) - matching_points_temp.append(matching_points[i]) + tgts_filtered.append(tgts[i]) + img_targets_filtered.append(img_targets[i]) - new_num_matches = len(tgts_matched_temp) + num_matches_filtered = len(tgts_filtered) + + tgts_filtered, img_targets_filtered = post_filter_append( + tgts, tgts_filtered, img_targets, img_targets_filtered + ) - for tgt in tgts_matched: - if tgt not in tgts_matched_temp: - tgts_matched_temp.append(tgt) - - for mp in matching_points: - if mp not in matching_points_temp: - matching_points_temp.append(mp) + return tgts_filtered, img_targets_filtered, num_matches_filtered - return tgts_matched_temp, matching_points_temp, new_num_matches - def filter_nones(tgts, img_targets, num_matches): - """Format matches to use a count instead of matching target or image target to None - - match_obj_and_img_pts matches each target to the closest img target. Therefore, it - is possible that not every img target will have a match. Additionally, if max_dist - is given to match_obj_and_img_pts, it is possible that not every target will have - a match. match_obj_and_img_pts solves this by matching unmatched items to None. + """Filters the targets and image target to remove None objects + + :func:`match_obj_and_img_pts` matches each target to the closest img target. + Therefore, it is possible that not every img target will have a match. Additionally, + if ``max_dist`` is given to :func:`match_obj_and_img_pts`, it is possible that not + every target will have a match. :func:`match_obj_and_img_pts` solves this by + matching unmatched items to None. That causes something to fail, so this function reformats that output. Instead of using Nones, this function reorders the list of targets to the first n target have @@ -566,73 +777,149 @@ def filter_nones(tgts, img_targets, num_matches): Parameters ---------- - tgts - img_targets - num_matches - + tgts : list + 3D targets. Each target should be a dict. The only strict requirement set by + this function is ``tgts[i]`` is associated with ``img_targets[i]`` for i from 0 + to `num_matches` + img_targets : list + Image targets. Each image target should be a dict, but this function does not + set any strict requirements other than that ``img_targets[i]`` is associated + with ``tgts[i]`` + num_matches : int + An integer of the number of matches. Must be less than or equal to + ``min(len(tgts), len(img_targets))`` + Returns - ---------- - tuple (tgts, img_targets, num_matches) + ------- + tgts_filtered : list + Target positions (`tgts`) that have been filtered to remove None values. + img_targets_filtered : list + Target positions in the image. + num_matches_filtered : int + + Notes + ----- + ``tgts_filtered[i]`` is associated with ``img_targets_filtered[i]`` for i from + 0 to `num_matches_filtered`. """ + # Initialized filtered lists and a count + tgts_filtered = [] + img_targets_filtered = [] + num_matches_filtered = 0 - tgts_temp = [] - img_targets_temp = [] - after_num_matches = 0 + # First add the matches where the img_targets are not None + # Increment the count of matched targets for i in range(num_matches): - tgt, match = tgts[i], img_targets[i] - if match is not None: - tgts_temp.append(tgt) - img_targets_temp.append(match) - after_num_matches += 1 - - for tgt in tgts: - if tgt not in tgts_temp: - tgts_temp.append(tgt) - - for img_target in img_targets: - if (img_target not in img_targets_temp) and (img_target is not None): - img_targets_temp.append(img_target) + tgt, img_target = tgts[i], img_targets[i] + if img_target is not None: + tgts_filtered.append(tgt) + img_targets_filtered.append(img_target) + num_matches_filtered += 1 + + tgts_filtered, img_targets_filtered = post_filter_append( + tgts, tgts_filtered, img_targets, img_targets_filtered + ) - return tgts_temp, img_targets_temp, after_num_matches + return tgts_filtered, img_targets_filtered, num_matches_filtered -def filter_matches(rmat, tvec, cameraMatrix, distCoeffs, tgts, img_targets, num_matches, test_config, debug=None): +def filter_matches( + rmat, + tvec, + cameraMatrix, + distCoeffs, + tgts, + img_targets, + num_matches, + test_config, + debug=None, +): """Wrapper to run multiple filtering functions Parameters ---------- - + rmat : np.ndarray, shape (3, 3), float + Rotation matrix from camera to object + tvec : np.ndarray, shape (3, 1), float + Translation vector from camera to object + cameraMatrix : np.ndarray, shape (3, 3), float + The (openCV formatted) camera matrix for the camera + distCoeffs : np.ndarray, shape (1, 5), float + The (openCV formatted) distortion coefficients for the camera + tgts : list + 3D targets. Each target is a dict that has, at a minimum, the keys 'tvec', and + 'target_type'. 'tvec' has a :class:`numpy.ndarray` (3, 1) representing the + position of the target relative to the model origin for its associated value. + 'target_type' has a string representing the type of target (most commonly 'dot') + for its associated value. ``tgts[i]`` is associated with ``img_targets[i]`` for + i from 0 to `num_matches` + img_targets : list + Matched image targets. Each dict has, at a minimum, the key 'center' which has a + value of is the image target center location (tuple/np.ndarray of floats). + ``img_targets[i]`` is associated with ``tgts[i]`` for i from 0 to `num_matches` + num_matches : int + An integer of the number of matches. Must be less than or equal to + ``min(len(tgts), len(img_targets))`` + test_config : dict + Processing parameters with, at a minimum, a key for 'min_dist', 'max_dist'. + 'min_dist' is the minimum distance between two image targets. 'max_dist' is the + maximum allowable distance between an image target and the projection of a 3D + target + debug : tuple, optional + tuple of length 2. First item is the image to use as the background. Second item + is the name of the debug image + Returns - ---------- - - - num_matches - number of matches - all visible tgts - all img_targets - - first n tgts match with first n img_targets - TODO: function description + ------- + tgts_filtered: list + Targets filtered to remove None values, matches greater than the maximum + matching distance, matches that are not one-to-one, matches that do not pass a + "bifilter" test, and matches that have image locations that are too close + together + img_targets_filtered: list + Associated image locations of `tgts_filtered` """ - + # If the targets aren't empty, perform all the filtering operations if (num_matches != 0) and (len(tgts) != 0) and (len(img_targets) != 0): - tgts_a, img_targets_a, num_matches_a = filter_nones(tgts, img_targets, num_matches) + tgts_a, img_targets_a, num_matches_a = filter_nones( + tgts, img_targets, num_matches + ) tgts_b, img_targets_b, num_matches_b = filter_max_dist( - rmat, tvec, cameraMatrix, distCoeffs, - tgts_a, img_targets_a, num_matches_a, test_config['max_dist'] - ) + rmat, + tvec, + cameraMatrix, + distCoeffs, + tgts_a, + img_targets_a, + num_matches_a, + test_config["max_dist"], + ) + + tgts_c, img_targets_c, num_matches_c = filter_one2one( + tgts_b, img_targets_b, num_matches_b + ) - tgts_c, img_targets_c, num_matches_c = filter_one2one(tgts_b, img_targets_b, num_matches_b) - tgts_d, img_targets_d, num_matches_d = filter_bifilter( - rmat, tvec, cameraMatrix, distCoeffs, - tgts_c, img_targets_c, num_matches_c, test_config['max_dist'] - ) - - tgts_e, img_targets_e, num_matches_e = filter_min_dist(tgts_d, img_targets_d, num_matches_d, test_config['min_dist']) + rmat, + tvec, + cameraMatrix, + distCoeffs, + tgts_c, + img_targets_c, + num_matches_c, + test_config["max_dist"], + ) + + tgts_e, img_targets_e, num_matches_e = filter_min_dist( + tgts_d, img_targets_d, num_matches_d, test_config["min_dist"] + ) tgts_filtered = tgts_e[:num_matches_e] img_targets_filtered = img_targets_e[:num_matches_e] - + + # If the targets are empty, populate the stages of filtering with the initialized + # values else: tgts_a, img_targets_a, num_matches_a = tgts, img_targets, num_matches tgts_b, img_targets_b, num_matches_b = tgts, img_targets, num_matches @@ -641,71 +928,98 @@ def filter_matches(rmat, tvec, cameraMatrix, distCoeffs, tgts, img_targets, num_ tgts_e, img_targets_e, num_matches_e = tgts, img_targets, num_matches tgts_filtered, img_targets_filtered = [], [] + # If the debug input is not None (default), create an image of the tgts and + # img_targets at each stage of the filtering operation if debug is not None: - tgts_img_targets_num_matches = [[tgts, img_targets, num_matches], - [tgts_a, img_targets_a, num_matches_a], - [tgts_b, img_targets_b, num_matches_b], - [tgts_c, img_targets_c, num_matches_c], - [tgts_d, img_targets_d, num_matches_d], - [tgts_e, img_targets_e, num_matches_e]] + tgts_img_targets_num_matches = [ + [tgts, img_targets, num_matches], + [tgts_a, img_targets_a, num_matches_a], + [tgts_b, img_targets_b, num_matches_b], + [tgts_c, img_targets_c, num_matches_c], + [tgts_d, img_targets_d, num_matches_d], + [tgts_e, img_targets_e, num_matches_e], + ] for i, data in enumerate(tgts_img_targets_num_matches): tgts_temp, img_targets_temp, num_matches_temp = data - - tgt_projs = photogrammetry.project_targets(rmat, tvec, cameraMatrix, distCoeffs, tgts_temp) - - proj_pts = np.array([tgt_proj['proj'] for tgt_proj in tgt_projs]) - img_centers = np.array([img_target['center'] for img_target in img_targets_temp]) - - suffix = {0: 'Original_match', - 1: 'filtered_a', - 2: 'filtered_b', - 3: 'filtered_c', - 4: 'filtered_d', - 5: 'filtered_e',}[i] - name = debug[1] + '_' + suffix + + tgt_projs = photogrammetry.project_targets( + rmat, tvec, cameraMatrix, distCoeffs, tgts_temp + ) + + proj_pts = np.array([tgt_proj["proj"] for tgt_proj in tgt_projs]) + img_centers = np.array( + [img_target["center"] for img_target in img_targets_temp] + ) + + suffix = { + 0: "Original_match", + 1: "filtered_a", + 2: "filtered_b", + 3: "filtered_c", + 4: "filtered_d", + 5: "filtered_e", + }[i] + name = debug[1] + "_" + suffix visualization.show_projection_matching( - debug[0], proj_pts, img_centers, - num_matches=num_matches_temp, name=name, - bonus_pt=None, scale=1, ax=None) + debug[0], + proj_pts, + img_centers, + num_matches=num_matches_temp, + name=name, + bonus_pt=None, + scale=1, + ax=None, + ) return tgts_filtered, img_targets_filtered -def match_obj_and_img_pts(rmat, tvec, cameraMatrix, distCoeffs, - tgts, img_targets, max_dist=np.inf): - """ +def match_obj_and_img_pts( + rmat, tvec, cameraMatrix, distCoeffs, tgts, img_targets, max_dist=np.inf +): + """Matches 3D targets to the image targets + + Projects the 3D targets into the image, then finds the closest image target. If the + closest image target is less than max_dist pixels away, it is matched. If it is + farther than `max_dist` pixels, it is matched to None. This matching scheme does not + ensure matches are one-to-one. + Parameters ---------- - - Returns - ---------- - - Given a pose from rmat and tvec and intrinsics cameraMatrix and distCoeffs, match - the 3D targets given with tgts to the 2D image locations of - img_targets - - Current matching scheme is to just match with the closest target. The image location - and projected location may be very far from each other. The matching may not be - one-to-one - - Inputs: - rmat and tvec are transformation from camera to the targets frame - rmat - rotation matrix - tvec - translation vector - tgts, img_targets - target information - tgts - 3D position of targets in tgts frame - img_targets - image location of targets - cameraMatrix, disCoeffs - intrinsics - cameraMatrix - camera matrix - distCoeffs - distortion coefficients + rmat : np.ndarray, shape (3, 3), float + Rotation matrix from camera to object + tvec : np.ndarray, shape (3,), float + Translation vector from camera to object + cameraMatrix : np.ndarray, shape (3, 3), float + The (openCV formatted) camera matrix for the camera + distCoeffs : np.ndarray, shape (5,), float + The (openCV formatted) distortion coefficients for the camera + tgts : np.ndarray, shape (n, 3) + List of 3D targets. + img_targets : list of dict + Matched image target information. Each dict has, at a minimum, keys 'center', + and 'target_type'. 'center' has a value of the image target center location + (tuple/np.ndarray of length 2 of floats) and 'target_type' has the key of the + type of target (string). ``img_targets[i]`` is associated with ``tgts[i]`` + max_dist : float + The maximum matching distance between a 3D target's project and an image targets + center. Pairs with matching distance greater this distance are filtered out. + + Returns: + ----------- + matching_img_targets : list + List of the items from the `img_targets` input such that + ``matching_img_targets[i]`` is the closest image target to the projected + position of ``tgts[i]``. If the closest image target is farther than `max_dist`, + ``matching_img_targets[i]`` is None """ # Project the points into the image tgt_projs = photogrammetry.project_targets( - rmat, tvec, cameraMatrix, distCoeffs, tgts - ) + rmat, tvec, cameraMatrix, distCoeffs, tgts + ) # For each target, find the matching image point of the same type # Matches are assumed to be the closest point @@ -715,140 +1029,343 @@ def match_obj_and_img_pts(rmat, tvec, cameraMatrix, distCoeffs, match_dist = max_dist for img_target in img_targets: # Check that the target types match - if img_target['target_type'] != tgt_proj['target_type']: + if img_target["target_type"] != tgt_proj["target_type"]: continue # Calculate the distance - dist = np.linalg.norm([img_target['center'][0] - tgt_proj['proj'][0], - img_target['center'][1] - tgt_proj['proj'][1]]) + dist = np.linalg.norm( + [ + img_target["center"][0] - tgt_proj["proj"][0], + img_target["center"][1] - tgt_proj["proj"][1], + ] + ) # If the distance is over the distance threshold and less than current # lowest distance, make this the new best match - if (dist < match_dist): + if dist < match_dist: match_dist = dist - match = img_target - + match = copy.deepcopy(img_target) + matching_img_targets.append(match) # Since the target ordering is not changes, we can return the visibles and # projected as was given - return tgts, matching_img_targets + return matching_img_targets -def match_targets(rmat, tvec, cameraMatrix, distCoeffs, tgts, img_targets, max_dist=np.inf, debug=None): +def match_targets( + rmat, tvec, cameraMatrix, distCoeffs, tgts, img_targets, max_dist=np.inf, debug=None +): """Matches each target to the closest img target - If max_dist is given, distance between target and img target must be less than + If `max_dist` is given, distance between target and img target must be less than max_dist. By default this value is infinite, so any match is valid. Parameters ---------- - rmat - tvec - cameraMatrix - distCoeffs - tgts - img_targets - max_dist - debug - + rmat : np.ndarray, shape (3, 3), float + Rotation matrix from camera to object + tvec : np.ndarray, shape (3, 1), float + Translation vector from camera to object + cameraMatrix : np.ndarray, shape (3, 3), float + The (openCV formatted) camera matrix for the camera + distCoeffs : np.ndarray, shape (1, 5), float + The (openCV formatted) distortion coefficients for the camera + tgts : list + 3D targets. Each target is a dict and has, at a minimum, the keys 'tvec', and + 'target_type'. 'tvec' has a :class:`numpy.ndarray` (3, 1) representing the + position of the target relative to the model origin for its associated value. + 'target_type' has a string representing the type of target (most commonly 'dot') + for its associated value. ``tgts[i]`` is associated with ``img_targets[i]`` for + i from 0 to `num_matches` + img_targets : list of dict + Each dict has, at a minimum, keys 'center', and 'target_type'. 'center' has a + value of the image target center location (tuple/np.ndarray of length 2 of + floats) and 'target_type' has the key of the type of target (string). + ``img_targets[i]`` is associated with ``tgts[i]`` + max_dist : float + The maximum matching distance between a 3D target's project and an image targets + center. Pairs with matching distance greater this distance are filtered out. + debug : tuple, optional + Debug option. For no debugging, give None. To generate debugging images, give + a tuple of length 3. First item is the image to use as the background. + Second item is the name of the debug image. Third item can be a dict or None. If + dict, matches will be filtered before the debug image is created. Dict needs to + follow test_config input requirements of filter_matches. If None, matches will + not be filtered in the debug image + Returns - ---------- - - Wrapper to match the 3D targets to the 2D image targets - - debug is [camera frame, figure name, optional] if debug images are to be generated - optional can be None or test_config + ------- + tgts_matched: list + 3D targets that are matched + matching_img_targets: list + Image locations of matched targets + num_matches: int + + Notes + ----- + ``tgts_matched[i]`` is matched with ``matching_img_targets[i]`` for i from 0 to + `num_matches`. """ # Match the visible targets to the closest image point - tgts_matched, matching_points = match_obj_and_img_pts( - rmat, tvec, cameraMatrix, distCoeffs, tgts, img_targets, max_dist) - - tgts_matched, matching_points, num_matches = filter_nones(tgts_matched, matching_points, len(tgts_matched)) + matching_img_targets = match_obj_and_img_pts( + rmat, tvec, cameraMatrix, distCoeffs, tgts, img_targets, max_dist + ) - for tgt in tgts: - if tgt not in tgts_matched: - tgts_matched.append(tgt) - for img_target in img_targets: - if img_target not in matching_points: - matching_points.append(img_target) + tgts_matched, matching_img_targets, num_matches = filter_nones( + tgts, matching_img_targets, len(tgts) + ) + tgts_matched, matching_img_targets = post_filter_append( + tgts, tgts_matched, img_targets, matching_img_targets + ) if debug is not None: if debug[2] is not None: tgts_matched_temp, img_targets_temp = filter_matches( - rmat, tvec, cameraMatrix, distCoeffs, tgts_matched, matching_points, - num_matches, debug[2], debug[:2]) + rmat, + tvec, + cameraMatrix, + distCoeffs, + tgts_matched, + matching_img_targets, + num_matches, + debug[2], + debug[:2], + ) num_matches_temp = len(tgts_matched_temp) else: - tgts_matched_temp, img_targets_temp = tgts_matched, matching_points + tgts_matched_temp, img_targets_temp = tgts_matched, matching_img_targets num_matches_temp = num_matches + tgts_matched_temp_temp = copy.deepcopy(tgts_matched_temp) + tgts_matched_temp_temp = [copy.deepcopy(tgt) for tgt in tgts_matched_temp] + for tgt in tgts_matched_temp_temp: + tgt["tvec"] = tgt["tvec"].flatten().tolist() + tgt["norm"] = tgt["norm"].flatten().tolist() + tgts_unmatched = [] for tgt in tgts: - if tgt not in tgts_matched_temp: + tgt_temp = copy.deepcopy(tgt) + tgt_temp["tvec"] = tgt_temp["tvec"].flatten().tolist() + tgt_temp["norm"] = tgt_temp["norm"].flatten().tolist() + if tgt_temp not in tgts_matched_temp_temp: tgts_unmatched.append(tgt) + img_targets_temp_temp = copy.deepcopy(img_targets_temp) + img_targets_temp_temp = [copy.deepcopy(tgt) for tgt in img_targets_temp] + for img_tgt in img_targets_temp_temp: + img_tgt["center"] = img_tgt["center"].flatten().tolist() + img_targets_unmatched = [] for img_tgt in img_targets: - if img_tgt not in img_targets_temp: + img_tgt_temp = copy.deepcopy(img_tgt) + img_tgt_temp["center"] = img_tgt_temp["center"].flatten().tolist() + if img_tgt_temp not in img_targets_temp_temp: img_targets_unmatched.append(img_tgt) tgt_projs = photogrammetry.project_targets( - rmat, tvec, cameraMatrix, distCoeffs, tgts_matched_temp + tgts_unmatched - ) - - proj_pts = np.array([tgt_proj['proj'] for tgt_proj in tgt_projs]) - img_centers = np.array([img_target['center'] for img_target in img_targets_temp + img_targets_unmatched]) + rmat, tvec, cameraMatrix, distCoeffs, tgts_matched_temp + tgts_unmatched + ) + + proj_pts = np.array([tgt_proj["proj"] for tgt_proj in tgt_projs]) + img_centers = np.array( + [ + img_target["center"] + for img_target in img_targets_temp + img_targets_unmatched + ] + ) visualization.show_projection_matching( - debug[0], proj_pts, img_centers, - num_matches = num_matches_temp, name=debug[1], bonus_pt=None, scale=1, ax=None) + debug[0], + proj_pts, + img_centers, + num_matches=num_matches_temp, + name=debug[1], + bonus_pt=None, + scale=1, + ax=None, + ) + + return tgts_matched, matching_img_targets, num_matches + + +def post_filter_append(tgts, tgts_filtered, img_targets, img_targets_filtered): + """Adds tgts that were filtered back into the list + + The match-and-filter scheme we use is to have two ordered lists, where + ``tgts_filtered[i]`` is matched to ``img_targets_filtered[i]`` for i from 0 to + `num_matches`. For i greater than num_matches, the tgt and img_target are not + matched. We leave all unmatched targets in the list, because some filtering + operations need them. Ex the the filters for min_dist and bi_filter. + + This function takes in the whole (randomly ordered) `tgts` and `img_targets`, as + well as the (ordered) `tgts_filtered` and `img_targets_filtered`. Any `tgts` not in + `tgts_filtered` are appended at the end, and similar for the `img_targets` + + Parameters + ---------- + tgts : list of dict + List of all targets, order is not important. Each target is a dict that has, at + a minimum, the keys 'tvec', and 'target_type'. If isMatched is False, 'norm' is + additionally needed as a key. 'tvec' has a :class:`numpy.ndarray` (3, 1) + representing the position of the target relative to the model origin for its + associated value. 'target_type' has a string representing the type of target + (most commonly 'dot') for its associated value. + tgts_filtered : list of dict + List of filtered targets, order is important. ``tgts_filtered[i]`` is associated + with ``img_targets_filtered[i]`` for i from 0 to `num_matches`. Each target is a + dict that has, at a minimum, the keys 'tvec', and 'target_type'. If isMatched is + False, 'norm' is additionally needed as a key. 'tvec' has a + :class:`numpy.ndarray` (3, 1) representing the position of the target relative + to the model origin for its associated value. 'target_type' has a string + representing the type of target (most commonly 'dot') for its associated value. + img_targets : list of dict + List of all matched image targets, order is not important. Each dict has, at a + minimum, keys 'center', and 'target_type'. 'center' has a value of the image + target center location (tuple/np.ndarray of length 2 of floats) and + 'target_type' has the key of the type of target (string). + img_targets_filtered : list of dict + List of filtered image targets, order is important. ``img_targets_filtered[i]`` + is associated with ``tgts_filtered[i]`` for i from 0 to `num_matches`. Each dict + has, at a minimum, keys 'center', and 'target_type'. 'center' has a value of the + image target center location (tuple/np.ndarray of length 2 of floats) and + 'target_type' has the key of the type of target (string). + """ + tgts_filtered_temp = copy.deepcopy(tgts_filtered) + img_targets_filtered_temp = copy.deepcopy(img_targets_filtered) + + tgts_filtered_list = copy.deepcopy(tgts_filtered) + for i in range(len(tgts_filtered_list)): + tgts_filtered_list[i]["tvec"] = tgts_filtered_list[i]["tvec"].flatten().tolist() + tgts_filtered_list[i]["norm"] = tgts_filtered_list[i]["norm"].flatten().tolist() + + # Add tgts that were not matched + for tgt in tgts: + tgt_list = copy.deepcopy(tgt) + tgt_list["tvec"] = tgt_list["tvec"].flatten().tolist() + tgt_list["norm"] = tgt_list["norm"].flatten().tolist() + if tgt_list not in tgts_filtered_list: + tgts_filtered_temp.append(tgt) + + img_targets_filtered_list = copy.deepcopy(img_targets_filtered) + + for i in range(len(img_targets_filtered_list)): + img_targets_filtered_list[i]["center"] = ( + img_targets_filtered_list[i]["center"].flatten().tolist() + ) + + # Add img_target that were not matched + for img_target in img_targets: + if img_target is not None: + img_target_list = copy.deepcopy(img_target) + img_target_list["center"] = img_target_list["center"].flatten().tolist() + if img_target_list not in img_targets_filtered_list: + img_targets_filtered_temp.append(img_target) - return tgts_matched, matching_points, num_matches + return tgts_filtered_temp, img_targets_filtered_temp -#--------------------------------------------------------------------------------------- +# --------------------------------------------------------------------------------------- # External Calibration Wrappers -def external_calibrate(img, rmat, tvec, # Frame specific - cameraMatrix, distCoeffs, # Camera specific - tgts, img_targets, vis_checker, test_config, # Config specific - isMatched=False, max_localize_delta=None, # Test specific - reprojectionError=6.0 # Test specific (should be stable between tests) - ): +def external_calibrate( + img, + rmat, + tvec, # Frame specific + cameraMatrix, + distCoeffs, # Camera specific + tgts, + img_targets, + vis_checker, + test_config, # Config specific + isMatched=False, + max_localize_delta=None, # Test specific + reprojectionError=6.0, # Test specific (should be stable between tests) +): """Set up and run solvePnPRansac to get the external calibration and inlier targets Parameters ---------- - img - rmat - tvec - cameraMatrix - distCoeffs - tgts - img_targets - vis_checker - test_config - isMatched - max_localize_delta - reprojectionError + img : np.ndarray, shape (height, width) + Image to use for calibration + rmat : np.ndarray, shape (3, 3), float + Rotation matrix from camera to object + tvec : np.ndarray, shape (3, 1), float + Translation vector from camera to object + cameraMatrix : np.ndarray, shape (3, 3), float + The (openCV formatted) camera matrix for the camera + distCoeffs : np.ndarray, shape (1, 5), float + The (openCV formatted) distortion coefficients for the camera + tgts : list of dict + Each target is a dict that has, at a minimum, the keys 'tvec', and + 'target_type'. If isMatched is False, 'norm' is additionally needed as a key. + 'tvec' has a np.ndarray (3, 1) representing the position of the target relative + to the model origin for its associated value. 'target_type' has a string + representing the type of target (most commonly 'dot') for its associated value. + ``tgts[i]`` is associated with ``img_targets[i]`` for i from 0 to `num_matches`. + 'norm' has a :class:`numpy.ndarray` (3, 1) representing the normal vector of the + target relative to the model coordinate system for its associated value + img_targets : list of dict + Matched image targets. Each dict has, at a minimum, keys 'center', and + 'target_type'. 'center' has a value of the image target center location + (tuple/np.ndarray of length 2 of floats) and 'target_type' has the key of the + type of target (string). ``img_targets[i]`` is associated with ``tgts[i]`` + vis_checker : ~upsp.cam_cal_utils.visibility.VisibilityChecker + Visibility checker object with the relevant BVH and oblique viewing angle + test_config : dict + dict with, at a minimum, a key for 'min_dist', 'max_dist', and each target type + in targets and img_targets. 'min_dist' is the minimum distance between two image + targets. 'max_dist' is the maximum allowable distance between an image target + and the projection of a 3D target. The key for each target type is + `target_type` + '_pad'. This is the padding around the img target center + location to use to sub-pixel localize + isMatched : bool + If True, denotes that ``tgts[i]`` is associated with ``img_targets[i]`` and all + targets are visible to the camera. If False, denotes `tgts` and `img_targets` + are not in any particular order. If False, targets are checked for visibility + and :func:`match_targets` is used to match the tgts to the image targets + max_localize_delta : float, optional + Parameter passed to :func:`subpixel_localize` + reprojectionError : float, optional + Maximum reprojection error between a target and image target to be considered an + inlier. ReprojectionError is often smaller than ``test_config['max_dist']`` + since it is the optimized distance between the target and image target. Returns - ---------- - + ------- + rmat_opt + optimized rotation matrix from the camera to the model + tvec_opt + optimized translation vector from the camera to the model + tgt_inliers + list of inlier targets of the optimization + img_target_inliers + list of the inlier image targets of the optimization + + Notes + ----- + ``tgt_inliers[i]`` is associated with ``img_target_inliers[i]`` """ - # If the inputs are not matched, get the visible targets and match them. + # If the inputs are not matched, get the visible targets and match them. if not isMatched: # Determine which targets are visible - visibles_tgts = photogrammetry.get_visible_targets(rmat, tvec, tgts, vis_checker) + visibles_tgts = photogrammetry.get_visible_targets( + rmat, tvec, tgts, vis_checker + ) # Match the projected locations to the image locations tgts_matched, img_targets_matched, num_matches_init = match_targets( - rmat, tvec, cameraMatrix, distCoeffs, visibles_tgts, img_targets, - test_config['max_dist']) - + rmat, + tvec, + cameraMatrix, + distCoeffs, + visibles_tgts, + img_targets, + test_config["max_dist"], + ) + # If the input is matched, short circuit the tgt_matched and # img_targets_matched with the inputs else: @@ -858,34 +1375,45 @@ def external_calibrate(img, rmat, tvec, # Frame specific # Filter the matched targets tgts_filtered, img_targets_filtered = filter_matches( - rmat, tvec, cameraMatrix, distCoeffs, - tgts_matched, img_targets_matched, num_matches_init, - test_config - ) - - # Subpixel localize the image targets - tgts_subpixel, img_targets_subpixel = subpixel_localize_robust( - img, tgts_filtered, img_targets_filtered, test_config, - max_localize_delta=max_localize_delta) + rmat, + tvec, + cameraMatrix, + distCoeffs, + tgts_matched, + img_targets_matched, + num_matches_init, + test_config, + ) + + # Sub-pixel localize the image targets + tgts_subpixel, img_targets_subpixel = subpixel_localize( + img, + tgts_filtered, + img_targets_filtered, + test_config, + max_localize_delta=max_localize_delta, + ) # If there are less than 4 matches, raise an error if len(tgts_subpixel) < 4: - raise ValueError("Less than 4 matches were found in external_calibrate. " + - "This can be due to blob detection finding too few targets, too few " + - "visible targets, a bad matching scheme due to a bad starting pose " + - "(rmat and tvec), and/or too many targets rejected during the sub-pixel " + - "localization.") + raise ValueError( + "Less than 4 matches were found in external_calibrate. " + + "This can be due to blob detection finding too few targets, too few " + + "visible targets, a bad matching scheme due to a bad starting pose " + + "(rmat and tvec), and/or too many targets rejected during the sub-pixel " + + "localization." + ) # Package the target tvecs tgt_tvecs = [] for tgt in tgts_subpixel: - tgt_tvecs.append(tgt['tvec']) + tgt_tvecs.append(tgt["tvec"]) tgt_tvecs = np.array(tgt_tvecs) # Package the image points img_centers = [] for target in img_targets_subpixel: - img_centers.append(target['center']) + img_centers.append(target["center"]) img_centers = np.array(img_centers) # Convert rmat to rvec @@ -893,9 +1421,15 @@ def external_calibrate(img, rmat, tvec, # Frame specific # Solve for the new tvec and rvec retval, rvec_opt, tvec_opt, inliers = cv2.solvePnPRansac( - tgt_tvecs, img_centers, cameraMatrix, distCoeffs, - copy.copy(rvec), copy.copy(tvec), reprojectionError=reprojectionError, - useExtrinsicGuess=True) + tgt_tvecs, + img_centers, + cameraMatrix, + distCoeffs, + copy.copy(rvec), + copy.copy(tvec), + reprojectionError=reprojectionError, + useExtrinsicGuess=True, + ) if len(inliers): inliers = np.squeeze(inliers, axis=1) @@ -913,307 +1447,515 @@ def external_calibrate(img, rmat, tvec, # Frame specific return rmat_opt, tvec_opt, tgt_inliers, img_target_inliers -def check_external_calibrate_two_stage_inputs(img, rmat_init_guess, tvec_init_guess, - camera_cal, tgts, test_config, vis_checker, - debug): +def check_external_calibrate_two_stage_inputs( + img, rmat_init_guess, tvec_init_guess, incal, tgts, test_config, vis_checker, debug +): """Check that the inputs to external calibrate are valid - Helper function to external_calibrate_two_stage. Checks that the input are of the - proper type, size, format, and have the relevant attributes. + Helper function to :func:`external_calibrate_two_stage`. Checks that the input are + of the proper type, size, format, and have the relevant attributes. Parameters ---------- - img - rmat_init_guess - tvec_init_guess - camera_cal - tgts - test_config - vis_checker - debug - + img : np.ndarray, shape (h, w) + Numpy 2D array of the image + rmat_init_guess : np.ndarray (3, 3), float + Initial guess of rotation matrix from camera to object + tvec_init_guess : np.ndarray (3, 1), float + Initial guess of translation vector from camera to object + incal : tuple + Camera internal calibration. + + - ``cameraMatrix`` (:class:`numpy.ndarray`, shape (3, 3)): The (OpenCV + formatted) camera matrix for the camera + - ``distCoeffs`` (:class:`numpy.ndarray`, shape (1, 5): The (OpenCV formatted) + distortion coefficients for the camera + tgts : list of dict + Each target is a dict and has, at a minimum, the keys 'tvec', 'target_type', + 'norm'. 'tvec' has a :class:`numpy.ndarray` (3, 1) representing the position of + the target relative to the model origin for its associated value. 'norm' has a + :class:`numpy.ndarray` (3, 1) representing the normal vector of the target + relative to the model origin for its associated value. 'target_type' has a + string representing the type of target (most commonly 'dot') for its associated + value. ``tgts[i]`` is associated with ``img_targets[i]`` + test_config : dict + Processing parameters with, at a minimum, a key for 'min_dist', 'max_dist', and + 2 keys for the primary target type in targets and img_targets. 'min_dist' is the + minimum distance between two image targets. 'max_dist' is the maximum allowable + distance between an image target and the projection of a 3D target. The primary + target type is 'dot' if there are 4+ dots in tgts. Otherwise the primary target + type is 'kulite'. The keys for the primary target type are target_type + '_pad' + and `target_type` + '_blob_parameters'. The first is the padding around the img + target center location to use to sub-pixel localize. The second is the blob + detection parameters for that type of target + vis_checker : ~upsp.cam_cal_utils.visibility.VisibilityChecker + Visibility checker object with the relevant BVH and oblique viewing angle + debug : string, optional + Name for all debug images (potentially camera name, test an camera name, etc). + Returns - ---------- - + ------- + valid : bool + True if the all inputs are valid, False if any input is invalid """ # TODO: Any time "isinstance(OBJ, list)" is called, np.array(OBJ) should be inside # of a try-except in case it has degenerate objects - #----------------------------------------------------------------------------------- + # ----------------------------------------------------------------------------------- # Check img input if isinstance(img, np.ndarray): if img.dtype != np.uint8: - print('img dtype should be uint8 not', img.dtype) + print("img dtype should be uint8 not", img.dtype) return False # If img is not 2 dimensional, it should be 3 dimensional, but single channel if len(img.shape) != 2: if len(img.shape) == 3: if img.shape[2] != 1: - print('img should be single channel, not', img.shape[2], 'channel') + print("img should be single channel, not", img.shape[2], "channel") return False else: # img is not 2 dimensional or 3 dimension. Must be an error - print('img should be 2 dimensional, or 3 dimensional and single channel. Not ', len(img.shape)) + print( + "img should be 2 dimensional, or 3 dimensional and single channel. Not ", + len(img.shape), + ) return False else: - print('img type should be np.ndarray. Not ', type(img)) + print("img type should be np.ndarray. Not ", type(img)) return False - #----------------------------------------------------------------------------------- + # ----------------------------------------------------------------------------------- # Check rmat_init_guess input - # if rmat_init_guess is a list, convert it to a numpy array to check the data - if isinstance(rmat_init_guess, list): - rmat_init_guess = np.array(rmat_init_guess) - + # Check rmat_init_guess is a numpy array of shape (3, 3) if isinstance(rmat_init_guess, np.ndarray): if rmat_init_guess.dtype != np.float32 and rmat_init_guess.dtype != np.float64: - print('rmat_init_guess.dtype should be float32 or float64 not', rmat_init_guess.dtype) + print( + "rmat_init_guess.dtype should be float32 or float64 not", + rmat_init_guess.dtype, + ) return False if rmat_init_guess.shape != (3, 3): - print('rmat_init_guess.shape should be (3, 3). Not ', rmat_init_guess.shape) + print("rmat_init_guess.shape should be (3, 3). Not ", rmat_init_guess.shape) return False if not photogrammetry.isRotationMatrix(rmat_init_guess): - print('rmat_init_guess is not a valid rotation matrix') + print("rmat_init_guess is not a valid rotation matrix") return False else: - print('rmat_init_guess should be np.ndarray or list. Not', type(rmat_init_guess)) + print("rmat_init_guess should be np.ndarray. Not", type(rmat_init_guess)) return False - - #----------------------------------------------------------------------------------- - # Check tvec_init_guess input - # if tvec_init_guess is a list, convert it to a numpy array to check the data - if isinstance(tvec_init_guess, list): - tvec_init_guess = np.array(tvec_init_guess) + # ----------------------------------------------------------------------------------- + # Check tvec_init_guess input + # Check tvec_init_guess is a numpy array of shape (3, 1) if isinstance(tvec_init_guess, np.ndarray): if tvec_init_guess.dtype != np.float32 and tvec_init_guess.dtype != np.float64: - print('tvec_init_guess.dtype should be float32 or float64 not', tvec_init_guess.dtype) + print( + "tvec_init_guess.dtype should be float32 or float64 not", + tvec_init_guess.dtype, + ) return False - if tvec_init_guess.shape != (3,) and tvec_init_guess.shape != (3, 1): - print('tvec_init_guess.shape should be (3,) or (3, 1). Not ', tvec_init_guess.shape) + if tvec_init_guess.shape != (3, 1): + print("tvec_init_guess.shape should be (3, 1). Not ", tvec_init_guess.shape) return False - + else: - print('tvec_init_guess should be np.ndarray or list. Not', type(tvec_init_guess)) + print("tvec_init_guess should be np.ndarray. Not", type(tvec_init_guess)) return False - - #----------------------------------------------------------------------------------- - # Check camera_cal input - # Only need to check camera_cal[0] and camera_cal[1] - - # if camera_cal[0] is a list, convert it to a numpy array to check the data - if isinstance(camera_cal[0], list): - camera_cal[0] = np.array(camera_cal[0]) - - if isinstance(camera_cal[0], np.ndarray): - if camera_cal[0].dtype != np.float32 and camera_cal[0].dtype != np.float64: - print('camera_cal[0].dtype should be float32 or float64 not', camera_cal[0].dtype) + + # ----------------------------------------------------------------------------------- + # Check incal input + + # if incal[0] is a list, convert it to a numpy array to check the data + if isinstance(incal[0], list): + incal[0] = np.array(incal[0]) + + if isinstance(incal[0], np.ndarray): + if incal[0].dtype != np.float32 and incal[0].dtype != np.float64: + print("incal[0].dtype should be float32 or float64 not", incal[0].dtype) return False - if camera_cal[0].shape != (3, 3): - print('camera_cal[0].shape should be (3, 3). Not ', camera_cal[0].shape) + if incal[0].shape != (3, 3): + print("incal[0].shape should be (3, 3). Not ", incal[0].shape) return False - - # camera_cal should be of the form [[fx, 0, cx], [0, fy, cy], [0, 0, 1]] - if (camera_cal[0][2] != [0.0, 0.0, 1.0]).any(): - print('camera_cal[0][2] must be [0, 0, 1]') + + # incal should be of the form [[fx, 0, cx], [0, fy, cy], [0, 0, 1]] + if (incal[0][2] != [0.0, 0.0, 1.0]).any(): + print("incal[0][2] must be [0, 0, 1]") return False - if camera_cal[0][0][1] != 0: - print('camera_cal[0][0][1] must be 0') + if incal[0][0][1] != 0: + print("incal[0][0][1] must be 0") return False - if camera_cal[0][1][0] != 0: - print('camera_cal[0][1][0] must be 0') + if incal[0][1][0] != 0: + print("incal[0][1][0] must be 0") return False - + else: - print('camera_cal[0] should be np.ndarray or list. Not', type(camera_cal[0])) + print("incal[0] should be np.ndarray or list. Not", type(incal[0])) return False - - # if camera_cal[1] is a list, convert it to a numpy array to check the data - if isinstance(camera_cal[1], list): - camera_cal[1] = np.array(camera_cal[1]) - - if isinstance(camera_cal[1], np.ndarray): - if camera_cal[1].dtype != np.float32 and camera_cal[1].dtype != np.float64: - print('camera_cal[1].dtype should be float32 or float64 not', camera_cal[1].dtype) + + # if incal[1] is a list, convert it to a numpy array to check the data + if isinstance(incal[1], list): + incal[1] = np.array(incal[1]) + + if isinstance(incal[1], np.ndarray): + if incal[1].dtype != np.float32 and incal[1].dtype != np.float64: + print("incal[1].dtype should be float32 or float64 not", incal[1].dtype) return False - if camera_cal[1].shape != (1, 5): - print('camera_cal[1].shape should be (1, 5). Not ', camera_cal[1].shape) + if incal[1].shape != (1, 5): + print("incal[1].shape should be (1, 5). Not ", incal[1].shape) return False - + else: - print('camera_cal[1] should be np.ndarray or list. Not', type(camera_cal[1])) + print("incal[1] should be np.ndarray or list. Not", type(incal[1])) return False - #----------------------------------------------------------------------------------- + # ----------------------------------------------------------------------------------- # Check tgts input - + # Check that each target is as expected for tgt in tgts: - if 'tvec' not in tgt or 'norm' not in tgt or 'target_type' not in tgt: - print('Each item in tgts must have a tvec, norm, and target_type. One or more of the items in tgts did not meet this criteria') + if "tvec" not in tgt or "norm" not in tgt or "target_type" not in tgt: + print( + "Each item in tgts must have a tvec, norm, and target_type. One or more of the items in tgts did not meet this criteria" + ) return False - + if isinstance(tgt["tvec"], np.ndarray) and isinstance(tgt["norm"], np.ndarray): + if (tgt["tvec"].shape != (3, 1)) or (tgt["norm"].shape != (3, 1)): + print( + "tgt tvec and norm must have shape (3, 1). One or more of the items in tgts did not meet this criteria" + ) + return False + else: + print( + "tgt tvec and norm must be np.ndarray. One or more of the items in tgts did not meet this criteria" + ) + return False + if len(tgts) < 4: - print('tgts must have at least 4 items. Only', len(tgts), 'were found') + print("tgts must have at least 4 items. Only", len(tgts), "were found") return False - #----------------------------------------------------------------------------------- + # ----------------------------------------------------------------------------------- # Check test_config input - - if ('dot_blob_parameters' not in test_config.keys()) and ('kulite_blob_parameters' not in test_config.keys()): - print("Neither 'dot_blob_parameters' nor 'kulite_blob_parameters' in test_config.keys()") + + if ("dot_blob_parameters" not in test_config.keys()) and ( + "kulite_blob_parameters" not in test_config.keys() + ): + print( + "Neither 'dot_blob_parameters' nor 'kulite_blob_parameters' in test_config.keys()" + ) return False - if 'dot_blob_parameters' in test_config.keys(): - if not isinstance(test_config['dot_blob_parameters'], list): - print("test_config['dot_blob_parameters'] should be a list, not", type(test_config['dot_blob_parameters'])) + if "dot_blob_parameters" in test_config.keys(): + if not isinstance(test_config["dot_blob_parameters"], list): + print( + "test_config['dot_blob_parameters'] should be a list, not", + type(test_config["dot_blob_parameters"]), + ) return False - if 'kulite_blob_parameters' in test_config.keys(): - if not isinstance(test_config['kulite_blob_parameters'], list): - print("test_config['kulite_blob_parameters'] should be a list, not", type(test_config['kulite_blob_parameters'])) + if "kulite_blob_parameters" in test_config.keys(): + if not isinstance(test_config["kulite_blob_parameters"], list): + print( + "test_config['kulite_blob_parameters'] should be a list, not", + type(test_config["kulite_blob_parameters"]), + ) return False # TODO: Need to check that test_config['blob_parameters'] contains all valid # items. But that is a lot of work, and it fails early enough in # external_calibration_two_stage that it doesn't matter much - if 'max_dist' not in test_config.keys(): + if "max_dist" not in test_config.keys(): print("'max_dist' not in test_config.keys()") return False - if not isinstance(test_config['max_dist'], (int, float)): + if not isinstance(test_config["max_dist"], (int, float)): print("test_config['max_dist'] must be numeric") return False - if 'min_dist' not in test_config.keys(): + if "min_dist" not in test_config.keys(): print("'min_dist' not in test_config.keys()") return False - if not isinstance(test_config['min_dist'], (int, float)): + if not isinstance(test_config["min_dist"], (int, float)): print("test_config['min_dist'] must be numeric") return False - #----------------------------------------------------------------------------------- + # ----------------------------------------------------------------------------------- # Check vis_checker input - - if not type(vis_checker).__name__ == 'VisibilityChecker': - print('vis_checker should be visibility.VisibilityChecker object. Not', type(vis_checker)) + + if not type(vis_checker).__name__ == "VisibilityChecker": + print( + "vis_checker should be visibility.VisibilityChecker object. Not", + type(vis_checker), + ) return False - if not type(vis_checker.scene).__name__ == 'BVH': - print('vis_checker.scene should be upsp.raycast.CreateBVH object. Not', type(vis_checker.scene)) + if not type(vis_checker.scene).__name__ == "BVH": + print( + "vis_checker.scene should be upsp.raycast.CreateBVH object. Not", + type(vis_checker.scene), + ) return False - #----------------------------------------------------------------------------------- + # ----------------------------------------------------------------------------------- # Check debug input # If debug is not None object, a string, or integer then it is improper if debug != None and not isinstance(debug, str) and not isinstance(debug, int): - print('debug should be None, str, or int. Not', type(debug)) + print("debug should be None, string, or int. Not", type(debug)) return False return True -def external_calibrate_two_stage_from_wtd(img, # Frame specific - tunnel_vals, # Datapoint specific - camera_cal, # Camera specific - tgts, test_config, vis_checker, # Test/configuration specific - debug=None): - """Wrapper function to external_calibrate_two_stage. Parses wtd into rmat and tvec +def external_calibrate_two_stage_from_wtd( + img, # Frame specific + tunnel_vals, # Datapoint specific + camera_tunnel_cal, # Camera specific + tgts, + test_config, + vis_checker, # Test/configuration specific + debug=None, +): + """Wrapper function to :func:`external_calibrate_two_stage`. + + The `tunnel_vals` plus `test_config` are used to estimate an initial guess of `rmat` + and `tvec`. That initial guess should project each target's within ~5 pixels of the + associated image target Parameters ---------- - img - tunnel_vals - camera_cal - tgts - test_config - vis_checker - debug - - Returns - ---------- + img : np.ndarray, shape (h, w) + Numpy 2D array of the image + tunnel_vals : dict + Wind tunnel data as a dict with (at a minimum) the keys 'ALPHA', 'BETA', 'PHI', + and 'STRUTZ'. ALPHA, BETA, and PHI are tunnel angles in degrees. STRUTZ is the + offset of the tunnel center of rotation for the z axis in inches + camera_tunnel_cal : tuple + Camera-tunnel calibration + + - ``rmat_camera_tunnel`` (:class:`numpy.ndarray`, shape (3, 3)): Rotation matrix + from camera to tunnel at wind off condition + - ``tvec_camera_tunnel`` (:class:`numpy.ndarray`, shape (3, 1)): Translation + vector from camera to tunnel at wind off condition + - ``cameraMatrix`` (:class:`numpy.ndarray`, shape (3, 3)): The (OpenCV + formatted) camera matrix for the camera + - ``distCoeffs`` (:class:`numpy.ndarray`, shape (1, 5)): The (OpenCV formatted + distortion coefficients for the camera + tgts : list of dict + 3D targets. Each target is a dict and has, at a minimum, the keys 'tvec', + 'target_type', 'norm'. 'tvec' has a np.ndarray (3, 1) representing the position + of the target relative to the model origin for its associated value. 'norm' has + a :class:`numpy.ndarray` (3, 1) representing the normal vector of the target + relative to the model origin for its associated value. 'target_type' has a + string representing the type of target (most commonly 'dot') for its associated + value. ``tgts[i]`` is associated with ``img_targets[i]`` + test_config : dict + Processing parameters with, at a minimum, a key for 'min_dist', 'max_dist', and + 2 keys for the primary target type in `targets` and `img_targets`. 'min_dist' is + the minimum distance between two image targets. 'max_dist' is the maximum + allowable distance between an image target and the projection of a 3D target. + The primary target type is 'dot' if there are 4+ dots in tgts. Otherwise the + primary target type is 'kulite'. The keys for the primary target type are + `target_type` + '_pad' and `target_type` + '_blob_parameters'. The first is the + padding around the img target center location to use to sub-pixel localize. The + second is the blob detection parameters for that type of target + vis_checker : ~upsp.cam_cal_utils.visibility.VisibilityChecker + Visibility checker object with the relevant BVH and oblique viewing angle + debug : string, optional + Name for all debug images (potentially camera name, test an camera name, etc) + Returns + ------- + rmat: np.ndarray, shape (3, 3), float + camera-to-model external calibration rotation matrix + tvec: np.ndarray, shape (3, 1), float + camera-to-model external calibration translation vector """ - - rmat_init_guess, tvec_init_guess = photogrammetry.tf_camera_tgts_thru_tunnel( - camera_cal, tunnel_vals, test_config + + # Check that the inputs are valid + rmat_camera_tunnel, tvec_camera_tunnel, cameraMatrix, distCoeffs = camera_tunnel_cal + check_bool = check_external_calibrate_two_stage_inputs( + img, + rmat_camera_tunnel, + tvec_camera_tunnel, + [cameraMatrix, distCoeffs], + tgts, + test_config, + vis_checker, + debug, + ) + + tunnel_vals_check_bool = True + if isinstance(tunnel_vals, dict): + keys = ['ALPHA', 'BETA', 'PHI', 'STRUTZ'] + if len(set(keys).intersection(tunnel_vals.keys())) != 4: + tunnel_vals_check_bool = False + print( + "tunnel_vals is missing 1 or more keys. tunnel_vals.keys():", + tunnel_vals.keys() + ) + + else: + for key in keys: + if not isinstance(tunnel_vals[key], float): + tunnel_vals_check_bool = False + print( + "tunnel_vals values should have type float. Instead", + "tunnel_vals[" + str(key) + "] is ", + str(type(tunnel_vals[key])) + ) + break + else: + tunnel_vals_check_bool = False + print( + "tunnel_vals should have type dict. Instead tunnel_vals has type ", + str(type(tunnel_vals)) ) - - rmat, tvec = external_calibrate_two_stage( - img, rmat_init_guess, tvec_init_guess, camera_cal, tgts, test_config, vis_checker, debug + + if not (check_bool and tunnel_vals_check_bool): + raise ValueError( + "One or more bad inputs were given to external_calibrate_two_stage_from_wtd" ) - + + ( + rmat_init_guess, + tvec_init_guess, + ) = camera_tunnel_calibrate.tf_camera_tgts_thru_tunnel( + camera_tunnel_cal, tunnel_vals, test_config + ) + + rmat, tvec = external_calibrate_two_stage( + img, + rmat_init_guess, + tvec_init_guess, + camera_tunnel_cal[2:], + tgts, + test_config, + vis_checker, + debug, + ) + return rmat, tvec -def external_calibrate_two_stage(img, # Frame specific - rmat_init_guess, tvec_init_guess, # Datapoint specific - camera_cal, # Camera specific - tgts, test_config, vis_checker, # Test/configuration specific - debug=None): +def external_calibrate_two_stage( + img, # Frame specific + rmat_init_guess, + tvec_init_guess, # Datapoint specific + incal, # Camera specific + tgts, + test_config, + vis_checker, # Test/configuration specific + debug=None, +): """Performs external calibration from an inaccurate initial guess Runs blob detection to find the img targets, then matches and filters the 3D targets to the img targets. Performs a coarse optimization to improve the intial guess. Then - calls external_calibrate_one_step to get the refined optimization - + calls :func:`external_calibrate_one_step` to get the refined optimization + + The initial guess of rmat and tvec should project each target's within ~5 pixels of + the associated image target + Parameters ---------- - img - rmat_init_guess - tvec_init_guess - camera_cal - tgts - test_config - vis_checker - debug - + img : np.ndarray, shape (h, w) + Numpy 2D array of the image + rmat_init_guess : np.ndarray (3, 3), float + Initial guess of rotation matrix from camera to object + tvec_init_guess : np.ndarray (3, 1), float + Initial guess of translation vector from camera to object + incal : tuple + Camera internal calibration. + + - ``cameraMatrix`` (:class:`numpy.ndarray`, shape (3, 3)): The (OpenCV + formatted) camera matrix for the camera + - ``distCoeffs`` (:class:`numpy.ndarray`, shape (1, 5): The (OpenCV formatted) + distortion coefficients for the camera + tgts : list of dict + Each target is a dict and has, at a minimum, the keys 'tvec', 'target_type', + 'norm'. 'tvec' has a :class:`numpy.ndarray` (3, 1) representing the position of + the target relative to the model origin for its associated value. 'norm' has a + :class:`numpy.ndarray` (3, 1) representing the normal vector of the target + relative to the model origin for its associated value. 'target_type' has a + string representing the type of target (most commonly 'dot') for its associated + value. ``tgts[i]`` is associated with ``img_targets[i]`` + test_config : dict + Processing parameters with, at a minimum, a key for 'min_dist', 'max_dist', and + 2 keys for the primary target type in targets and img_targets. 'min_dist' is the + minimum distance between two image targets. 'max_dist' is the maximum allowable + distance between an image target and the projection of a 3D target. The primary + target type is 'dot' if there are 4+ dots in tgts. Otherwise the primary target + type is 'kulite'. The keys for the primary target type are target_type + '_pad' + and target_type + '_blob_parameters'. The first is the padding around the img + target center location to use to sub-pixel localize. The second is the blob + detection parameters for that type of target + vis_checker : ~upsp.cam_cal_utils.visibility.VisibilityChecker + Visibility checker object with the relevant BVH and oblique viewing angle + debug : string or None. Optional, default=None + Name for all debug images (potentially camera name, test an camera name, etc) + Returns - ---------- - + ------- + rmat : np.ndarray, shape (3, 3) + Valid solution camera-to-model rotation matrix + tvec : np.ndarray, shape (3, 1) + Valid solution camera-to-model translation vector """ # Check that the inputs are valid check_bool = check_external_calibrate_two_stage_inputs( - img, rmat_init_guess, tvec_init_guess, camera_cal, tgts, test_config, vis_checker, debug + img, + rmat_init_guess, + tvec_init_guess, + incal, + tgts, + test_config, + vis_checker, + debug, ) - + # Scale the image img = img_utils.scale_image_max_inlier(img) - + # Check the inputs if not check_bool: - raise ValueError("One or more bad inputs were given to external_calibrate_two_stage") + raise ValueError( + "One or more bad inputs were given to external_calibrate_two_stage" + ) # Unpack the intrinsics - cameraMatrix, distCoeffs = camera_cal[0], camera_cal[1], + cameraMatrix, distCoeffs = ( + incal[0], + incal[1], + ) # Check if there are enough dots to use for targets dots_found = True - init_tgts = [] + primary_tgts = [] for tgt in tgts: - if tgt['target_type'] == 'dot': - init_tgts.append(tgt) - + if tgt["target_type"] == "dot": + primary_tgts.append(tgt) + # If there were enough dots, use the dots blob parameters - if len(init_tgts) > 4: + if len(primary_tgts) > 4: blob_parameters = test_config["dot_blob_parameters"] - + # If there were not enough dots, collect the kulites and the kulite blob parameters else: # Set the dot_found flag to False dots_found = False blob_parameters = test_config["kulite_blob_parameters"] - + + primary_tgts = [] for tgt in tgts: - if tgt['target_type'] == 'kulite': - init_tgts.append(tgt) + if tgt["target_type"] == "kulite": + primary_tgts.append(tgt) # Define the blob detector based on the parameters in the test_config file params = cv2.SimpleBlobDetector_Params() @@ -1227,256 +1969,717 @@ def external_calibrate_two_stage(img, # Frame specific # Repackage the blob detector keypoints to function like taret image locations init_img_targets = [] for keypoint in keypoints: - init_img_target = {'target_type' : {True:'dot', False:'kulite'}[dots_found], # target_type depends on if enough dots were found - 'center' : keypoint.pt} + init_img_target = { + "target_type": {True: "dot", False: "kulite"}[ + dots_found + ], # target_type depends on if enough dots were found + "center": np.array(keypoint.pt), + } init_img_targets.append(init_img_target) # Debug for raw matching information if debug_raw_matches: tgts_visible = photogrammetry.get_visible_targets( - rmat_init_guess, tvec_init_guess, init_tgts, vis_checker) + rmat_init_guess, tvec_init_guess, primary_tgts, vis_checker + ) tgts_match_raw, img_targets_match_raw, num_matches_raw = match_targets( - rmat_init_guess, tvec_init_guess, cameraMatrix, distCoeffs, - tgts_visible, init_img_targets, test_config['max_dist']) - + rmat_init_guess, + tvec_init_guess, + cameraMatrix, + distCoeffs, + tgts_visible, + init_img_targets, + test_config["max_dist"], + ) + tgts_filtered_raw, img_targets_filtered_raw = filter_matches( - rmat_init_guess, tvec_init_guess, cameraMatrix, distCoeffs, - tgts_match_raw, img_targets_match_raw, num_matches_raw, - test_config) - - tgts_subpixel_raw, img_targets_subpixel_raw = subpixel_localize_robust( - img, tgts_filtered_raw, img_targets_filtered_raw, test_config, - max_localize_delta=None) + rmat_init_guess, + tvec_init_guess, + cameraMatrix, + distCoeffs, + tgts_match_raw, + img_targets_match_raw, + num_matches_raw, + test_config, + ) + + tgts_subpixel_raw, img_targets_subpixel_raw = subpixel_localize( + img, + tgts_filtered_raw, + img_targets_filtered_raw, + test_config, + max_localize_delta=None, + ) num_matches = len(img_targets_subpixel_raw) + tgts_subpixel_raw_temp = [copy.deepcopy(tgt) for tgt in tgts_subpixel_raw] + for tgt in tgts_subpixel_raw_temp: + tgt["tvec"] = tgt["tvec"].flatten().tolist() + tgt["norm"] = tgt["norm"].flatten().tolist() + rms, max_dist = photogrammetry.reprojection_error( - rmat_init_guess, tvec_init_guess, cameraMatrix, distCoeffs, - tgts_subpixel_raw, img_targets_subpixel_raw) + rmat_init_guess, + tvec_init_guess, + cameraMatrix, + distCoeffs, + tgts_subpixel_raw, + img_targets_subpixel_raw, + ) log.info( - 'Raw Num Points: %d RMS: %f Max Error: %f', - len(tgts_subpixel_raw), rms, max_dist + "Raw Num Points: %d RMS: %f Max Error: %f", + len(tgts_subpixel_raw), + rms, + max_dist, ) - + # Get a list of the targets that are visible, but weren't matched visible_but_not_matched = [] for tgt in tgts_visible: - if tgt not in tgts_subpixel_raw: + tgt_temp = copy.deepcopy(tgt) + tgt_temp["tvec"] = tgt_temp["tvec"].flatten().tolist() + tgt_temp["norm"] = tgt_temp["norm"].flatten().tolist() + if tgt_temp not in tgts_subpixel_raw_temp: visible_but_not_matched.append(tgt) # Get projected location the visible target centers in the image tgt_projs = photogrammetry.project_targets( - rmat_init_guess, tvec_init_guess, cameraMatrix, distCoeffs, - tgts_subpixel_raw + visible_but_not_matched) - proj_pts = np.array([tgt_proj['proj'] for tgt_proj in tgt_projs]) - + rmat_init_guess, + tvec_init_guess, + cameraMatrix, + distCoeffs, + tgts_subpixel_raw + visible_but_not_matched, + ) + proj_pts = np.array([tgt_proj["proj"] for tgt_proj in tgt_projs]) + found_but_not_matched = [] - for tgt_match_raw, img_target_match_raw in zip(tgts_match_raw, img_targets_match_raw): - if tgt_match_raw not in tgts_subpixel_raw: + for tgt_match_raw, img_target_match_raw in zip( + tgts_match_raw, img_targets_match_raw + ): + tgt_temp = copy.deepcopy(tgt_match_raw) + tgt_temp["tvec"] = tgt_temp["tvec"].flatten().tolist() + tgt_temp["norm"] = tgt_temp["norm"].flatten().tolist() + if tgt_temp not in tgts_subpixel_raw_temp: found_but_not_matched.append(img_target_match_raw) - + all_img_targets = img_targets_subpixel_raw + found_but_not_matched - img_centers = [inlier_pt['center'] for inlier_pt in all_img_targets] + img_centers = [inlier_pt["center"] for inlier_pt in all_img_targets] img_centers = np.array(img_centers) # Get the debug name. If debug was given, use it otherwise don't - debug_name = str(debug)+'_raw' if debug != None else 'raw' + debug_name = str(debug) + "_raw" if debug is not None else "raw" # Output a debug image of the projected locations and image target center locations - visualization.show_projection_matching(img, proj_pts, img_centers, num_matches=num_matches, - name=debug_name, scale=2.) - + visualization.show_projection_matching( + img, + proj_pts, + img_centers, + num_matches=num_matches, + name=debug_name, + scale=2.0, + ) + # Check that enough blobs were found if len(keypoints) < 4: - raise ValueError("Less than 4 blobs were found in external_calibrate_two_stage.") + raise ValueError( + "Less than 4 blobs were found in external_calibrate_two_stage." + ) - # Do the coarse external calibration + # Do the coarse external calibration coarse_outputs = external_calibrate( - img, rmat_init_guess, tvec_init_guess, cameraMatrix, distCoeffs, - init_tgts, init_img_targets, vis_checker, test_config, - isMatched=False, max_localize_delta=None, reprojectionError=test_config['max_dist']) - + img, + rmat_init_guess, + tvec_init_guess, + cameraMatrix, + distCoeffs, + primary_tgts, + init_img_targets, + vis_checker, + test_config, + isMatched=False, + max_localize_delta=None, + reprojectionError=test_config["max_dist"], + ) + # Unpack the output variables - rmat_coarse, tvec_coarse, tgts_inliers_coarse, img_target_inliers_coarse = coarse_outputs + ( + rmat_coarse, + tvec_coarse, + tgts_inliers_coarse, + img_target_inliers_coarse, + ) = coarse_outputs if debug_coarse_optimization: rms, max_dist = photogrammetry.reprojection_error( - rmat_coarse, tvec_coarse, cameraMatrix, distCoeffs, - tgts_inliers_coarse, img_target_inliers_coarse) + rmat_coarse, + tvec_coarse, + cameraMatrix, + distCoeffs, + tgts_inliers_coarse, + img_target_inliers_coarse, + ) log.info( - 'Coarse Num Points: %d RMS: %f Max Error: %f', - len(tgts_inliers_coarse), rms, max_dist + "Coarse Num Points: %d RMS: %f Max Error: %f", + len(tgts_inliers_coarse), + rms, + max_dist, ) - + # Get projected location the visible targets in the image tgt_projs = photogrammetry.project_targets( - rmat_coarse, tvec_coarse, cameraMatrix, distCoeffs, - tgts_inliers_coarse) + rmat_coarse, tvec_coarse, cameraMatrix, distCoeffs, tgts_inliers_coarse + ) + + proj_pts = np.array([tgt_proj["proj"] for tgt_proj in tgt_projs]) + img_pts = np.array( + [inlier_pt["center"] for inlier_pt in img_target_inliers_coarse] + ) - proj_pts = np.array([tgt_proj['proj'] for tgt_proj in tgt_projs]) - img_pts = np.array([inlier_pt['center'] for inlier_pt in img_target_inliers_coarse]) - # Get the debug name. If debug was given, use it otherwise don't - debug_name = str(debug)+'_coarse' if debug != None else 'coarse' - - visualization.show_projection_matching(img, proj_pts, img_pts, - name=debug_name, scale=2) + debug_name = str(debug) + "_coarse" if debug is not None else "coarse" + + visualization.show_projection_matching( + img, proj_pts, img_pts, name=debug_name, scale=2 + ) # Call the one step function for the refined optimization rmat_refined, tvec_refined = external_calibrate_one_step( - img, rmat_coarse, tvec_coarse, camera_cal, init_tgts, test_config, vis_checker, debug) - + img, + rmat_coarse, + tvec_coarse, + incal, + primary_tgts, + test_config, + vis_checker, + debug, + ) + return rmat_refined, tvec_refined -def external_calibrate_one_step(img, # Frame specific - rmat_coarse, tvec_coarse, # Datapoint specific - camera_cal, # Camera specific - init_tgts, test_config, vis_checker, # Test/configuration specific - debug=None): +def external_calibrate_one_step( + img, # Frame specific + rmat_coarse, + tvec_coarse, # Datapoint specific + incal, # Camera specific + tgts, + test_config, + vis_checker, # Test/configuration specific + debug=None, +): """Performs external calibration from a seim-accurate coarse guess - + + The coarse guess of `rmat` and `tvec` should project each target's within 1 pixel of + the associated image target + Parameters ---------- - img - rmat_coarse - tvec_coarse - camera_cal - init_tgts - test_config - vis_checker - debug - + img : np.ndarray, shape (h, w) + Numpy 2D array of the image + rmat_coarse : np.ndarray, shape (3, 3), float + Coarsely refined rotation matrix from camera to object + tvec_coarse : np.ndarray, shape (3, 1), float + Coarsely refined translation vector from camera to object + incal : tuple + Camera internal calibration. + + - ``cameraMatrix`` (:class:`numpy.ndarray`, shape (3, 3)): The (OpenCV + formatted) camera matrix for the camera + - ``distCoeffs`` (:class:`numpy.ndarray`, shape (1, 5): The (OpenCV formatted) + distortion coefficients for the camera + tgts : list of dict + Each target is a dict and has, at a minimum, the keys 'tvec', 'target_type', + 'norm'. 'tvec' has a :class:`numpy.ndarray` (3, 1) representing the position of + the target relative to the model origin for its associated value. 'norm' has a + :class:`numpy.ndarray` (3, 1) representing the normal vector of the target + relative to the model origin for its associated value. 'target_type' has a + string representing the type of target (most commonly 'dot') for its associated + value. ``tgts[i]`` is associated with ``img_targets[i]`` + test_config : dict + Processing parameters with, at a minimum, a key for 'min_dist', 'max_dist', and + 2 keys for the primary target type in targets and img_targets. 'min_dist' is the + minimum distance between two image targets. 'max_dist' is the maximum allowable + distance between an image target and the projection of a 3D target. The primary + target type is 'dot' if there are 4+ dots in tgts. Otherwise the primary target + type is 'kulite'. The keys for the primary target type are target_type + '_pad' + and target_type + '_blob_parameters'. The first is the padding around the img + target center location to use to sub-pixel localize. The second is the blob + detection parameters for that type of target + vis_checker : ~upsp.cam_cal_utils.visibility.VisibilityChecker + Visibility checker object with the relevant BVH and oblique viewing angle + debug : string or None. Optional, default=None + Name for all debug images (potentially camera name, test an camera name, etc) + Returns - ---------- - + ------- + rmat: np.ndarray, shape (3, 3), float + camera-to-model external calibration rotation matrix + tvec: np.ndarray, shape (3, 1), float + camera-to-model external calibration translation vector """ # Scale the image img = img_utils.scale_image_max_inlier(img) # Unpack the intrinsics - cameraMatrix, distCoeffs = camera_cal[0], camera_cal[1], + cameraMatrix, distCoeffs = ( + incal[0], + incal[1], + ) # Get the visible targets - visible_tgts = photogrammetry.get_visible_targets(rmat_coarse, tvec_coarse, - init_tgts, vis_checker) - + visible_tgts = photogrammetry.get_visible_targets( + rmat_coarse, tvec_coarse, tgts, vis_checker + ) + # Get the projections of the visible targets - tgt_projs = photogrammetry.project_targets(rmat_coarse, tvec_coarse, - cameraMatrix, distCoeffs, - visible_tgts) + tgt_projs = photogrammetry.project_targets( + rmat_coarse, tvec_coarse, cameraMatrix, distCoeffs, visible_tgts + ) # Package the target projections into img targets img_targets = [] for tgt_proj in tgt_projs: - img_target = {'target_type' : tgt_proj['target_type'], - 'center' : tgt_proj['proj']} + img_target = { + "target_type": tgt_proj["target_type"], + "center": tgt_proj["proj"], + } img_targets.append(img_target) - + # Debuf for refined matches if debug_refined_matches: # Filter the matched targets tgts_filtered, img_targets_filtered = filter_matches( - rmat_coarse, tvec_coarse, cameraMatrix, distCoeffs, - visible_tgts, img_targets, len(visible_tgts), - test_config - ) - + rmat_coarse, + tvec_coarse, + cameraMatrix, + distCoeffs, + visible_tgts, + img_targets, + len(visible_tgts), + test_config, + ) + # Subpixel localize the image targets - tgts_subpixel, img_targets_subpixel = subpixel_localize_robust( - img, tgts_filtered, img_targets_filtered, test_config, - max_localize_delta=None) + tgts_subpixel, img_targets_subpixel = subpixel_localize( + img, + tgts_filtered, + img_targets_filtered, + test_config, + max_localize_delta=None, + ) num_matches = len(img_targets_subpixel) - + + tgts_subpixel_temp = [copy.deepcopy(tgt) for tgt in tgts_subpixel] + for tgt in tgts_subpixel_temp: + tgt["tvec"] = tgt["tvec"].flatten().tolist() + tgt["norm"] = tgt["norm"].flatten().tolist() + # Get a list of the targets that are visible, but weren't matched visible_but_not_matched = [] for tgt in visible_tgts: - if tgt not in tgts_subpixel: + tgt_temp = copy.deepcopy(tgt) + tgt_temp["tvec"] = tgt_temp["tvec"].flatten().tolist() + tgt_temp["norm"] = tgt_temp["norm"].flatten().tolist() + if tgt_temp not in tgts_subpixel_temp: visible_but_not_matched.append(tgt) # Get projected location the visible target centers in the image tgt_projs = photogrammetry.project_targets( - rmat_coarse, tvec_coarse, cameraMatrix, distCoeffs, - tgts_subpixel + visible_but_not_matched) - proj_pts = np.array([tgt_proj['proj'] for tgt_proj in tgt_projs]) - - img_centers = np.array([img_target['center'] for img_target in img_targets_subpixel]) + rmat_coarse, + tvec_coarse, + cameraMatrix, + distCoeffs, + tgts_subpixel + visible_but_not_matched, + ) + proj_pts = np.array([tgt_proj["proj"] for tgt_proj in tgt_projs]) + + img_centers = np.array( + [img_target["center"] for img_target in img_targets_subpixel] + ) rms, max_dist = photogrammetry.reprojection_error( - rmat_coarse, tvec_coarse, cameraMatrix, distCoeffs, - tgts_subpixel, img_targets_subpixel) + rmat_coarse, + tvec_coarse, + cameraMatrix, + distCoeffs, + tgts_subpixel, + img_targets_subpixel, + ) # Get the debug name. If debug was given, use it otherwise don't - debug_name = str(debug)+'_refined' if debug != None else 'refined' + debug_name = str(debug) + "_refined" if debug is not None else "refined" # Output a debug image of the projected locations and image target center locations - visualization.show_projection_matching(img, proj_pts, img_centers, num_matches=num_matches, - name=debug_name, scale=2.) + visualization.show_projection_matching( + img, + proj_pts, + img_centers, + num_matches=num_matches, + name=debug_name, + scale=2.0, + ) # Run the refined external calibration refined_outputs = external_calibrate( - img, rmat_coarse, tvec_coarse, cameraMatrix, distCoeffs, - visible_tgts, img_targets, vis_checker, test_config, - isMatched=True, max_localize_delta=None, reprojectionError=test_config['max_dist']) - + img, + rmat_coarse, + tvec_coarse, + cameraMatrix, + distCoeffs, + visible_tgts, + img_targets, + vis_checker, + test_config, + isMatched=True, + max_localize_delta=None, + reprojectionError=test_config["max_dist"], + ) + # Unpack the refined results - rmat_refined, tvec_refined, tgts_inliers_refined, img_target_inliers_refined = refined_outputs + ( + rmat_refined, + tvec_refined, + tgts_inliers_refined, + img_target_inliers_refined, + ) = refined_outputs # Debug for refined optimization if debug_refined_optimization: rms, max_dist = photogrammetry.reprojection_error( - rmat_refined, tvec_refined, cameraMatrix, distCoeffs, - tgts_inliers_refined, img_target_inliers_refined) + rmat_refined, + tvec_refined, + cameraMatrix, + distCoeffs, + tgts_inliers_refined, + img_target_inliers_refined, + ) log.info( - 'Refined Num Points: %d RMS: %f Max Error: %f', - len(tgts_inliers_refined), rms, max_dist + "Refined Num Points: %d RMS: %f Max Error: %f", + len(tgts_inliers_refined), + rms, + max_dist, ) - + # Get projected location the visible targets in the image - tgt_projs = photogrammetry.project_targets(rmat_refined, tvec_refined, - cameraMatrix, distCoeffs, - tgts_inliers_refined) + tgt_projs = photogrammetry.project_targets( + rmat_refined, tvec_refined, cameraMatrix, distCoeffs, tgts_inliers_refined + ) # Plot the kulites in blue and dots in red - plt.imshow(img, cmap='gray') + plt.imshow(img, cmap="gray") for tgt_proj in tgt_projs: - plt.scatter([tgt_proj['proj'][0]], [tgt_proj['proj'][1]], - c={'kulite':'b', 'dot':'r'}[tgt_proj['target_type']], - marker='o', s=0.05) + plt.scatter( + [tgt_proj["proj"][0]], + [tgt_proj["proj"][1]], + c={"kulite": "b", "dot": "r"}[tgt_proj["target_type"]], + marker="o", + s=0.05, + ) # Get the debug name. If debug was given, use it otherwise don't - debug_name = str(debug)+'_refined_optimization.png' if debug != None else 'refined_optimization.png' + debug_name = ( + str(debug) + "_refined_optimization.png" + if debug != None + else "refined_optimization.png" + ) - plt.savefig(debug_name, dpi = 400) + plt.savefig(debug_name, dpi=400) plt.close() # Secondary debug to project all visible targets, not just the targets used in the # optimization if debug_visible_projections: # Get visible targets - visible_tgts = photogrammetry.get_visible_targets(rmat_refined, tvec_refined, - init_tgts, vis_checker) - + visible_tgts = photogrammetry.get_visible_targets( + rmat_refined, tvec_refined, tgts, vis_checker + ) + # Get projected location the visible targets in the image - tgt_projs = photogrammetry.project_targets(rmat_refined, tvec_refined, - cameraMatrix, distCoeffs, - visible_tgts) + tgt_projs = photogrammetry.project_targets( + rmat_refined, tvec_refined, cameraMatrix, distCoeffs, visible_tgts + ) # Plot the kulites in blue and dots in red - plt.imshow(img, cmap='gray') + plt.imshow(img, cmap="gray") for i, tgt_proj in enumerate(tgt_projs): - plt.scatter([tgt_proj['proj'][0]], [tgt_proj['proj'][1]], - c={'kulite':'b', 'dot':'r'}[tgt_proj['target_type']], - marker='o', s=0.05) - + plt.scatter( + [tgt_proj["proj"][0]], + [tgt_proj["proj"][1]], + c={"kulite": "b", "dot": "r"}[tgt_proj["target_type"]], + marker="o", + s=0.05, + ) + # Get the debug name. If debug was given, use it otherwise don't - debug_name = str(debug)+'_visible_projection.png' if debug != None else 'visible_projection.png' - - plt.savefig(debug_name, dpi = 400) + debug_name = ( + str(debug) + "_visible_projection.png" + if debug is not None + else "visible_projection.png" + ) + + plt.savefig(debug_name, dpi=400) plt.close() return rmat_refined, tvec_refined + +def external_calibrate_RANSAC( + incal, tgts, img_targets, vis_checker, max_iter=0.999, max_dist=8, match_thresh=0.80 +): + """Use RANSAC to find an external calibration with brute force + + To find an external calibration, select at random 3 targets and 3 img targets. Solve + the associated P3P problem to get the external calibration(s). For each of the + external calibrations (P3P can yield up to 4), find the visible targets, project + them, and match them to the image targets. If there is sufficient consensus amoung + the matches (i.e. the randomly selected targets and image targets yields a solution + where many other unselected targets project to a location close to an image target), + return that external calibration. If there is not sufficient consensus, repeat this + process. + + Parameters + ---------- + incal : tuple + Camera internal calibration. + + - ``cameraMatrix`` (:class:`numpy.ndarray`, shape (3, 3)): The (OpenCV + formatted) camera matrix for the camera + - ``distCoeffs`` (:class:`numpy.ndarray`, shape (1, 5): The (OpenCV formatted) + distortion coefficients for the camera + tgts : list of dict + Each target is a dict with (at a minimum) 'tvec', 'norm', and 'target_type' + attributes. The 'tvec' attribute gives the target's location and the 'norm' + attribute gives the target's normal vector. 'target_type' is a string denoting + the type of target (most commonly 'dot' or 'kulite') + img_targets : list of dict + Each img_target is a dict with (at a minimum) a 'center' attribute. The 'center' + attribute gives the pixel position of the target's center in the image. + ``img_targets[i]`` is associated with ``tgts[i]`` + vis_checker : ~upsp.cam_cal_utils.visibility.VisibilityChecker + VisibilityChecker object with the relevant BVH and oblique viewing angle + max_iter : int or float, optional + If int, it represents the maximum number of iterations for RANSAC. If float must + be greater than 0 but less than 1. Float means it represents the number of + estimate number of iterations in order to have that probability of finding a + solution should one exist (0.999 means there's a 99.9% chance of finding a + solution should one exist) + max_dist : int or float, optional + Maximum matching distance between image targets and projected target location + match_thresh : int or float, optional + If int, this is the number of matches that must be found to deem it a consensus. + If float, must be greater than 0 but less than 1. If float, this is the + proportion of tgts or img_targets (whichever is lower) that must be matched. + I.e. If match_thresh is 0.8 and there are 50 target and 20 img_targets, there + must be 16+ matches for it to be deemed a consensus. + + Returns + ------- + rmat : np.ndarray, shape (3, 3) + Rotation matrix. May be ``None`` if the maximum number of iterations is met + without finding a valid solution. + tvec : np.ndarray, shape (3, 1) + Translation vector. May be ``None`` if the maximum number of iterations is met + without finding a valid solution. + """ + # Seed the random number generator at the start of the routine + np.random.seed(0) + + cameraMatrix, distCoeffs = incal + + # 'Fake' visibility checker object with no grid which is used to speed up computation + vis_checker_nogrid = visibility.VisibilityChecker( + None, + oblique_angle=vis_checker.oblique_angle, + epsilon=vis_checker.epsilon, + debug=False, + debug_nogrid=True, + ) + + # If match_thresh is between 0 and 1, get the number of matches required + if 0 < match_thresh <= 1: + # Take the minimum of the proportion of tgts and img_targets + match_thresh = min( + np.floor(len(tgts) * match_thresh).astype(np.int32), + np.floor(len(img_targets) * match_thresh).astype(np.int32), + ) + + # match_thresh must be at least 4 for any consensus (3 are used for P3P, so at + # least 1 additional is required for the smallest consensus) + match_thresh = int(max(match_thresh, 4)) + + # If max_iter is a float of 1.0, user wanted probability of 1, so repeat infinite times + if isinstance(max_iter, float) and (max_iter == 1.0): + max_iter = np.inf + + # If max_iter is None, find the value of max_iter such that there is a 99% chance of + # find the solution if it exists + if 0 < max_iter < 1: + # Estimated number of visible targets based on oblique viewing angle and + # assuming targets are equally distributed + est_vis_targs = len(tgts) * vis_checker.oblique_angle / 180 + + # Expected number of iterations to find 3 visible targets + exp_3_vis_num = len(tgts) * (len(tgts) - 1) * (len(tgts) - 2) + exp_3_vis_den = est_vis_targs * (est_vis_targs - 1) * (est_vis_targs - 2) + exp_3_vis = exp_3_vis_num / exp_3_vis_den + + # Expected number of iterations to choose all 3 correct image target + exp_img = len(img_targets) * (len(img_targets) - 1) * (len(img_targets) - 2) + + # Probability of find a solution per iteration + P = 1 / (exp_3_vis * exp_img) + + # Binomial equations is: b(x, n, p) = [n! / (x! (n - x)!)] * P^x * (1-P)^(n-x) + # Using that equation, simply knowing we want the probabily of x=0 to be + # (1-max_iter) + # (1-max_iter) = (1-P)^n => n = log(1-max_iter) / log(1-P) + # Where n is max_iter + max_iter = np.rint(np.log(1 - max_iter) / np.log(1 - P)).astype(np.int32) + + # Get a list of indicies of the tgts + tgt_tvecs = np.array([tgt["tvec"] for tgt in tgts]) + tgt_tvecs_idxs = np.arange(len(tgt_tvecs)) + + # Get a list of indicies of the image targets + img_target_projs = np.array([img_tgt["center"] for img_tgt in img_targets]) + img_target_projs_idxs = np.arange(len(img_target_projs)) + + print( + "tgt_tvecs:", str(len(tgt_tvecs)) + ", img_target_projs:", len(img_target_projs) + ) + print("match_thresh:", str(match_thresh) + ", max_iter:", max_iter) + + # RANSAC Operation: + # Select at random 3 tgts and 3 img targats. Solve the corresponding P3P problem, + # and check for consensus with the other target projections and image targets. + # If there is good consensus, leave the loop + # We do not cache random guesses because there are likely hundreds of millions of + # possible guesses (depending on length of tgts and img_targets), and it takes on + # the order of a couple hundred thousand to find a solution (again depending on + # length of tgts and img_targets). So randomly generating the same guess twice + # will be relatively rare, and it is faster to generate the same guess twice + # rather than have to cache each guess and check each guess against the cache + max_num_matches = 0 + max_rmat, max_tvec = None, None + n = 0 + while n < max_iter: + # Select 3 random targets + rand_tgts_idxs = np.random.choice(tgt_tvecs_idxs, 3, replace=False) + rand_tgts = tgt_tvecs[rand_tgts_idxs] + + # Select 3 random image targets + rand_img_targets_idxs = np.random.choice( + img_target_projs_idxs, 3, replace=False + ) + rand_img_targets = img_target_projs[rand_img_targets_idxs] + + # Solve the associated P3P problem + retval, rvecs, tvecs = cv2.solveP3P( + rand_tgts, + rand_img_targets, + cameraMatrix, + distCoeffs, + flags=cv2.SOLVEPNP_P3P, + ) + + # Check each potential solution from the P3P solutions + for i in range(retval): + # Get the rmat and tvec + rvec, tvec = rvecs[i], tvecs[i] + rmat, _ = cv2.Rodrigues(rvec) + + # Get the visible targets + tgts_vis = photogrammetry.get_visible_targets( + rmat, tvec, tgts, vis_checker_nogrid + ) + + # If there are too few visible targets, continue + if len(tgts_vis) < match_thresh: + continue + + # Match the visible targets + tgts_matched, matching_points, num_matches = match_targets( + rmat, + tvec, + cameraMatrix, + distCoeffs, + tgts_vis, + img_targets, + max_dist=max_dist, + ) + + # Filter the visible targets + tgts_matched, matching_points, num_matches = filter_one2one( + tgts_matched, matching_points, num_matches + ) + + if num_matches > max_num_matches: + max_num_matches = num_matches + print("\r\tmax number of matches found:", num_matches, "\t\t\t", end="") + max_rmat, max_tvec = rmat, tvec + + # If there are enough matches, repeat checking process with vis_checker + # to ensure this solution is valid + if num_matches >= match_thresh: + # Get the visible targets + tgts_vis = photogrammetry.get_visible_targets( + rmat, tvec, tgts, vis_checker + ) + + # Match the visible targets + tgts_matched, matching_points, num_matches = match_targets( + rmat, + tvec, + cameraMatrix, + distCoeffs, + tgts_vis, + img_targets, + max_dist=max_dist, + ) + + # Filter the visible targets + tgts_matched, matching_points, num_matches = filter_one2one( + tgts_matched, matching_points, num_matches + ) + + # Package the target tvecs and image points + tgt_tvecs = np.array( + [tgts_matched[k]["tvec"] for k in range(num_matches)] + ) + img_centers = np.array( + [matching_points[k]["center"] for k in range(num_matches)] + ) + + # SolvePnP but with all targets + retval, rvec_opt, tvec_opt = cv2.solvePnP( + tgt_tvecs, + img_centers, + cameraMatrix, + distCoeffs, + copy.copy(rvec), + copy.copy(tvec), + useExtrinsicGuess=True, + ) + rmat_opt, _ = cv2.Rodrigues(rvec_opt) + + # If there are still enough matches, return the solution found + # Otherwise just go back into the loop + if num_matches >= match_thresh: + print() + return rmat_opt, tvec_opt + + n += 1 + print() + + warn_str = str( + "Maximum number of iterations exceeded in " + + "external_calibrate_RANSAC. Please check that inputs are correct. " + + "If they are, increase max_iter, increase max_dist, or decrease " + + "match_thresh. Returning the best found solution" + ) + + warnings.warn(warn_str, RuntimeWarning) + return max_rmat, max_tvec diff --git a/python/upsp/cam_cal_utils/img_utils.py b/python/upsp/cam_cal_utils/img_utils.py index cffbf8c..27b139b 100755 --- a/python/upsp/cam_cal_utils/img_utils.py +++ b/python/upsp/cam_cal_utils/img_utils.py @@ -1,19 +1,46 @@ import numpy as np -import scipy.stats +import scipy.interpolate + def convert_12bit_to_8bit(img): + """Proportionally scales image values from (0, 4095) to (0, 255) + + Scales input image values from (0, 4095) to (0, 255). Values in input image that + are greater than scale will be clipped to 255 in the output image + + Parameters + ---------- + img : np.ndarray, shape (y, x), np.uint8 + Input image + + Returns + ------- + scaled_img : np.ndarray, shape (y, x), np.uint8 + 8 bit scaled version of input image + """ return scale_image(img, scale=(2**12) - 1) def scale_image(img, scale=(2**12) - 1): - """ - Proportionally scales image values between 0 and scale to 0 to 255 - I.e. If scale is 50, all pixel values are multiplied by 5.1 (255/50) then - rounded to an integer - Returns unsigned, 8 bit scaled version of input image + """Proportionally scales image values from (0, `scale`) to (0, 255) + + Scales input image values from (0, `scale`) to (0, 255). Values in input image that + are greater than `scale` will be clipped to 255 in the output image + + Parameters + ----------- + img : np.ndarray, shape (y, x) + Input image + scale : float or int, optional + Maximum value for scaling input image. Default=4095 (max int for 12 bit) + + Returns + ------- + scaled_img : np.ndarray, shape (y, x), np.uint8 + 8 bit scaled version of input image """ - # Ensure the image is scaled properly + # Clip the image to the max value img_temp = np.minimum(img, scale) # Normalize the image @@ -28,8 +55,23 @@ def scale_image(img, scale=(2**12) - 1): def scale_image_max_inlier(img): - """ - Convert from 12 bit image to 8 bit image by scaling by the max inlier image value + """Proportionally scales image values from (0, `max_inlier`) to (0, 255) + + Scales input image values from (0, `max_inlier`) to (0, 255). Values in input image + that are greater than the max inlier will be clipped to 255 in the output image + + In a sorted list of the pixel intensities, the max inlier is the largest value that + satisfies the following condition: ``intensity[i] * 0.9 <= intensity[i*0.999]`` + + Parameters + ---------- + img : np.ndarray, shape (y, x), np.unit8 + Input image + + Returns + ------- + scaled_img : np.ndarray, shape (y, x), np.unit8 + 8 bit scaled version of input image """ # Get the maximum value, ignoring outliers @@ -47,79 +89,34 @@ def scale_image_max_inlier(img): return scale_image(img, scale=img_flat[i]) -# TODO: see external_calibration_monte_carlo.interpolation_based_association for -# inspiration to implement a fast version that only checks neighbors within max distance -def interp(pt, neighbors, method, scale=1, max=2): - method_funcs = {'nearest' : interp_nearest, - 'lanczos' : interp_lanczos, - 'inv_prop' : interp_inv_prop, - 'gauss' : interp_gauss, - 'expo_decay' : interp_expo_decay, - 'logistic' : interp_logistic} - - assert(method in method_funcs.keys()), "method must be in " + str(list(method_funcs.keys())) - - scoring_func = method_funcs[method] - - # Calculate the distance between the point and its neighbors - dists = np.linalg.norm(neighbors - pt, axis=1) - - # Get coefficients based on given method - coeffs = scoring_func(dists, scale) +def interp(pts, img, method='nearest'): + """Interpolate values for `pts` from `img` - # Remove points farther than max - coeffs = np.where(dists <= max, coeffs, 0.0) + Interpolates `img` at the locations defined by `pts` using the given method - total_sum = np.sum(coeffs) - - # If the sum is 0 (usually only happens when max is too small), use interp_nearest - if (total_sum == 0.0): - coeffs = interp_nearest(dists, scale) - total_sum = 1.0 - - # Scale so sum of coeffs is 1 - coeffs /= total_sum + Parameters + ---------- + pts : np.ndarray, shape (n, 2), float + n points to be interpolated + img : np.ndarray, shape (y, x), np.unit8 + Input image + method : {'linear', 'nearest', 'slinear', 'cubic', 'quintic'}, optional + Interpolation method. See :class:`scipy.interpolate.RegularGridInterpolator` for + details - # Return the final coefficients - return coeffs - - -def interp_nearest(dists, scale): - """Returns 1 for closest neighbor and 0 for all else - scaling does nothing, but is kept so all interp functions have the same inputs - """ - return np.where(dists == dists.min(), 1.0, 0.0) - - -def interp_lanczos(dists, scale): - """lanczos function - Note: some coefficients are negative - """ - # Calculate the lanczos parameters - return np.sinc(dists) * np.sinc(dists / scale) - - -def interp_inv_prop(dists, scale): - """inverse proportional + Returns + ------- + out : np.ndarray, shape (n, 2), float + Value of pts interpolated from img """ - # Calculate the inv proportional parameters - return 1 / (scale + dists) + # Ensure the method is one supported by scipy + assert method in ['linear', 'nearest', 'slinear', 'cubic', 'quintic'] + # Define the X and Y domains + X, Y = np.arange(img.shape[1]), np.arange(img.shape[0]) -def interp_gauss(dists, scale): - """normal distribution - """ - return scipy.stats.norm.pdf(dists, scale=scale) - - -def interp_expo_decay(dists, scale): - """exponential decay - """ - return np.exp(-dists / scale) - - -def interp_logistic(dists, scale): - """logistic function - """ - return scipy.stats.logistic.pdf(dists, scale=scale) + # Define the scipy interpolation function + f = scipy.interpolate.RegularGridInterpolator((X, Y), img.T, method=method) + # Return the interpolated values for the points + return f(pts) diff --git a/python/upsp/cam_cal_utils/internal_calibration.py b/python/upsp/cam_cal_utils/internal_calibration.py new file mode 100644 index 0000000..36c1a3a --- /dev/null +++ b/python/upsp/cam_cal_utils/internal_calibration.py @@ -0,0 +1,749 @@ +import numpy as np +import json +import os +from scipy.spatial import Delaunay +import matplotlib.pyplot as plt +import cv2 + +from upsp.cam_cal_utils import photogrammetry, parsers, visualization + +debug_show_cal_bounds = True +meter2inch = 39.3701 + +# NOTE: This is implemented for Calib.io version v1.6.5a + + +def get_safe_pts( + rmat, + tvec, + cameraMatrix, + distCoeffs, + obj_pts, + cal_area_is_safe, + cal_vol_is_safe, + critical_pt=None, +): + """Determines which `obj_pts` are inside the well behaved region of the internal calibration + + Performs 3 checks on each of the `obj_pts`. The first check and second check is that + each obj_pt and its projection are within the safe volume and safe area as defined + by cal_area_is_safe and cal_vol_is_safe. These checks are always performed. The + third check is not always necessary, and is more involved. + + If the highest order (non-zero) radial distortion term of `distCoeffs` is negative, + then at some real world distance from the camera optical axis, increasing the real + world distance from the camera optical axis will decrease pixel distance of that + point's projection. I.e. moving farther out of frame will move the point closer to + the image center. This is not a bug, but is not an accurate model of the physical + system. This means that some object points very far out of the field of view can + be projected into the frame. These points have a 'negative derivative' of the + projection curve. As the input (real world distance from camera optical axis) + increases, the output (projected distance from the image center) decreases. + + The points need not be outside the field of view to have a negative derivative. So a + simple FOV check is unfortunately insufficient. The third check is unnecessary if + `cal_area_is_safe` and `cal_vol_is_safe` are well constructed. If they are not well + constructed, the third check ensures the object points are in the well-behaved + region of the internal calibration. For most cases, this means the object points + are in the positive region of the internal calibration. + + Parameters + ---------- + rmat : np.ndarray, shape (3, 3), float + Rotation matrix from camera to object + tvec : np.ndarray (3, 1), float + Translation vector from camera to object + cameraMatrix : np.ndarray, shape (3, 3), float + The (openCV formatted) camera matrix for the camera + distCoeffs : np.ndarray, shape (5, 1) or (5,), float + The (openCV formatted) distortion coefficients for the camera + obj_pts : np.ndarray, shape (n, 3), float + List of 3d points. Points are relative to the object frame + cal_area_is_safe : callable + ``alpha_shape_is_safe`` function that takes `img_pts` as an array, (shape + ``(n, 2)``) of floats and returns an array-like object of booleans. + ``return[i]`` corresonds to ``img_pts[i]``. If ``return[i]`` is True, then + ``img_pts[i]`` is within the safe internal calibration image area. If it is + False, it is not within that safe area + cal_vol_is_safe : callable + ``alpha_shape_is_safe`` function that takes `obj_pts` and returns a array-like + object of booleans. ``return[i]`` corresonds to ``img_pts[i]``. If ``return[i]`` + is True, then ``img_pts[i]`` is within the safe internal calibration 3D volume. + If it is False, it is not within that safe volume + critical_pt : str, optional + Criteria for step 3. Must be one of: + + - If 'first' then `obj_pts` cannot have a homogeneous coordinate with a + magnitude past the first maxima of the distortion curve. This is the most + useful option for the majority of lenses. + - If 'infinity', the magnitude must be less than the final maxima of the + distortion curve, and the distortion curve must be decreasing after the final + maxima. This is the most useful option for mustache or otherwise complex + distortion, if the mustache distortion effect is accurate + - If ``None``, step 3 will be omitted. If `cal_area_is_safe` and + `cal_vol_is_safe` are available (and not dummy return True functions from + :func:`incal_calibration_bounds_debug`), the step 3 check is likely + unnecessary + + Returns + ------- + obj_pts_safe : np.ndarray, shape (n,), bool + Array of booleans where ``return[i]`` cooresponds to ``obj_pts[i]``. If + ``return[i]`` is True, ``obj_pts[i]`` is well behaved. If ``return[i]`` is + False, ``obj_pts[i]`` may not be well behaved + + See Also + -------- + incal_calibration_bounds : creates ``alpha_shape_is_safe`` functions + """ + assert critical_pt in ["first", "final", None] + + # Define a list of booleans to denote if a variable is safe + obj_pts_safe = np.full(len(obj_pts), True) + + # Step 1: Check that the points project to within the safe image area region + img_pts = photogrammetry.project_3d_point( + rmat, tvec, cameraMatrix, distCoeffs, obj_pts + ) + obj_pts_safe *= cal_area_is_safe(img_pts) + + # Step 2: Check that the points are within the safe 3D volume + pts_rel_cam = photogrammetry.transform_3d_point(rmat, tvec, obj_pts) + obj_pts_safe *= cal_vol_is_safe(pts_rel_cam) + + # Step 3a: Check for shortcuts to determine if the full step 3 is necessary + + # First shortcut skips if step 3 is omitted + if critical_pt is None: + return obj_pts_safe + + # Second shortcut skips if all of the distortion coefficients are non-negative, + # since there is no maxima + if (distCoeffs >= 0).all(): + return obj_pts_safe + + # Third shortcut skips if critical_pt == 'final' and the projection curve has a + # positive derivative as the magnitude of the homogeneous coordinates goes to inf + # To determine that, we find the highest order, nonzero term. The highest order term + # for both the x projected position and y projected position is k3, then k2, then + # k1. After that, the highest order term for the x projected position is p2 and + # the highest order term for the y projected position is p1. Since step 2 checked + # for no distortion, if k3, k2, and k1 are zero then p1 or (or both) p2 must be + # nonzero. If the minimum of those 2 is positive, they are both positive. After p1 + # and p2, The remainining terms are linear and a constant offset. Neither of which + # cause the issue step 3 addresses so neither need to be checked. + highest_nonzero_term = ( + distCoeffs[0][4] + if distCoeffs[0][4] != 0.0 + else distCoeffs[0][1] + if distCoeffs[0][1] != 0.0 + else distCoeffs[0][0] + if distCoeffs[0][0] != 0.0 + else min(p2, p1) + ) + if (critical_pt == "final") and highest_nonzero_term > 0.0: + return obj_pts_safe + + # Step 3b: Perform the full step 3 + + # First step is to get the location of the points relative to the camera. + obj_pts_rel_cam = photogrammetry.transform_3d_point(rmat, tvec, obj_pts) + + # Define the homogenous coordiantes + x_homo_vals = (obj_pts_rel_cam[:, 0] / obj_pts_rel_cam[:, 2]).astype(np.complex) + y_homo_vals = (obj_pts_rel_cam[:, 1] / obj_pts_rel_cam[:, 2]).astype(np.complex) + + # Define the distortion terms, and vectorize calculating of powers of x_homo_vals + # and y_homo_vals + k1, k2, p1, p2, k3 = distCoeffs[0] + x_homo_vals_2 = np.power(x_homo_vals, 2) + y_homo_vals_2 = np.power(y_homo_vals, 2) + x_homo_vals_4 = np.power(x_homo_vals, 4) + y_homo_vals_4 = np.power(y_homo_vals, 4) + x_homo_vals_6 = np.power(x_homo_vals, 6) + y_homo_vals_6 = np.power(y_homo_vals, 6) + + # This step cannot be vectorized since the numpy roots function is numerical + # Find the bounds on the x_homo coordinate to ensure it is closer than the + # inflection point of x_proj as a function of x_homo + x_homo_min = np.full(x_homo_vals.shape, -np.inf) + x_homo_max = np.full(x_homo_vals.shape, np.inf) + for i in range(len(y_homo_vals)): + # If obj_pts_safe[i] is False already, no need to calculate the roots + if obj_pts_safe[i] == False: + continue + + # Expanded projection function polynomial coefficients + x_proj_coeffs = np.array( + [ + k3, + 0, + k2 + 3 * k3 * y_homo_vals_2[i], + 0, + k1 + 2 * k2 * y_homo_vals_2[i] + 3 * k3 * y_homo_vals_4[i], + 3 * p2, + 1 + + k1 * y_homo_vals_2[i] + + k2 * y_homo_vals_4[i] + + k3 * y_homo_vals_6[i] + + 2 * p1 * y_homo_vals[i], + p2 * y_homo_vals_2[i], + ] + ) + + # Projection function derivative polynomial coefficients + x_proj_der_coeffs = np.polyder(x_proj_coeffs) + + # Find the root of the derivative + roots = np.roots(x_proj_der_coeffs) + + # Get the real roots + # Approximation of real[np.where(np.isreal(roots))] + real_roots = np.real(roots[np.where(np.abs(np.imag(roots)) < 1e-10)]) + + for real_root in real_roots: + if critical_pt == "first": + if real_root > 0.0: + x_homo_max[i] = np.minimum(x_homo_max[i], real_root) + else: + x_homo_min[i] = np.maximum(x_homo_min[i], real_root) + else: + x_homo_min[i] = np.minimum(x_homo_min[i], real_root) + x_homo_max[i] = np.maximum(x_homo_max[i], real_root) + + # Check that the x_homo values are within the bounds + obj_pts_safe *= np.where(x_homo_vals > x_homo_min, True, False) + obj_pts_safe *= np.where(x_homo_vals < x_homo_max, True, False) + + # Find the bounds on the y_homo coordinate to ensure it is closer than the + # inflection point of y_proj as a function of y_homo + y_homo_min = np.full(y_homo_vals.shape, -np.inf) + y_homo_max = np.full(y_homo_vals.shape, np.inf) + for i in range(len(x_homo_vals)): + # If obj_pts_safe[i] is False already, no need to calculate the roots + if obj_pts_safe[i] == False: + continue + + # Expanded projection function polynomial coefficients + y_proj_coeffs = np.array( + [ + k3, + 0, + k2 + 3 * k3 * x_homo_vals_2[i], + 0, + k1 + 2 * k2 * x_homo_vals_2[i] + 3 * k3 * x_homo_vals_4[i], + 3 * p1, + 1 + + k1 * x_homo_vals_2[i] + + k2 * x_homo_vals_4[i] + + k3 * x_homo_vals_6[i] + + 2 * p2 * x_homo_vals[i], + p1 * x_homo_vals_2[i], + ] + ) + + # Projection function derivative polynomial coefficients + y_proj_der_coeffs = np.polyder(y_proj_coeffs) + + # Find the root of the derivative + roots = np.roots(y_proj_der_coeffs) + + # Get the real roots + # Approximation of real[np.where(np.isreal(roots))] + real_roots = np.real(roots[np.where(np.abs(np.imag(roots)) < 1e-10)]) + + for real_root in real_roots: + if critical_pt == "first": + if real_root > 0.0: + y_homo_max[i] = np.minimum(y_homo_max[i], real_root) + else: + y_homo_min[i] = np.maximum(y_homo_min[i], real_root) + else: + y_homo_min[i] = np.minimum(y_homo_min[i], real_root) + y_homo_max[i] = np.maximum(y_homo_max[i], real_root) + + # Check that the x_homo values are within the bounds + obj_pts_safe *= np.where(y_homo_vals > y_homo_min, True, False) + obj_pts_safe *= np.where(y_homo_vals < y_homo_max, True, False) + + # Return the list of is_safe booleans + return obj_pts_safe + + +def incal_from_calibio(calibio_path): + """Returns the internal calibration values from the calib.io output json + + Parameters + ---------- + calibio_path : str + Path to the calib.io saved calibration json + + Returns + ------- + img_size : np.ndarray, shape (2,) + Image size (height, width) + uPSP_cameraMatrix : np.ndarray, shape (3, 3) + Camera matrix for uPSP applications (same as openCV ``cameraMatrix``, but cx and + cy are vectors from image center to principal point rather than the principal + point itself). + distCoeffs : np.ndarray, shape (5, 1) + openCV distortion coefficients + """ + # Read the internal calibration data + calibio_data = parsers.read_json(calibio_path)["calibration"]["cameras"][0][ + "model" + ]["ptr_wrapper"]["data"] + + # Get the image size + img_size = calibio_data["CameraModelCRT"]["CameraModelBase"]["imageSize"] + img_size = np.array((img_size["height"], img_size["width"])) + + # Read the internal calibration parameters + params = calibio_data["parameters"] + + # Parse the camera matrix + cameraMatrix = np.array( + [ + [params["f"]["val"], 0.0, params["cx"]["val"]], + [0.0, params["f"]["val"], params["cy"]["val"]], + [0.0, 0.0, 1.0], + ] + ) + uPSP_cameraMatrix = parsers.convert_cv2_cm_to_uPSP_cm(cameraMatrix, img_size) + + # Parse the distortion coefficients + distCoeffs = np.array( + [ + [ + params["k1"]["val"], + params["k2"]["val"], + params["p1"]["val"], + params["p2"]["val"], + params["k3"]["val"], + ] + ] + ) + + return img_size, uPSP_cameraMatrix, distCoeffs + + +def write_incal_from_calibio(calibio_path, camera_name, sensor_size, save_dir=None): + """Writes the internal calibration to a json file + + Saves internal calibration as ``'{camera_name}.json'`` to `save_dir` + + Parameters + ---------- + calibio_path : str + Path to the calib.io saved calibration json + camera_name : str + Name of the camera + sensor_size : np.ndarray, shape (2,) of floats + Physical size of the image sensor in inches + save_dir : str, optional + Path of directory to save the internal calibration json file. If None, save_dir + is set to the directory containing the Calib.io json + + """ + img_size, uPSP_cameraMatrix, distCoeffs = incal_from_calibio(calibio_path) + sensor_size = np.array(sensor_size) + + incal = { + "uPSP_cameraMatrix": uPSP_cameraMatrix.tolist(), + "distCoeffs": distCoeffs.tolist(), + "sensor_resolution": img_size.tolist(), + "sensor_size": sensor_size.tolist(), + } + + if save_dir is None: + save_dir = os.basename(calibio_path) + + # Export the internal calibration as a json file + cal_file = camera_name + ".json" + with open(os.path.join(save_dir, cal_file), "w") as f: + json.dump(incal, f) + + +def uncertainties_from_calibio(calibio_path): + """Returns uncertainties for OpenCV terms + + Parameters + ---------- + calibio_path : str + Path to the calib.io saved calibration json + + Returns + ------- + tuple + Standard deviation of calibration terms: (focal length, principal point x, + principal point y, k1, k2, p1, p2, k3) + """ + + covariance_data = parsers.read_json(calibio_path)["covariance"]["covarianceMatrix"] + size = (covariance_data["size"]["height"], covariance_data["size"]["width"]) + covarianceMatrix = np.array(covariance_data["pixels"]).reshape(size) + + f_stddev = np.sqrt(covarianceMatrix[0][0]) + cx_stddev = np.sqrt(covarianceMatrix[2][2]) + cy_stddev = np.sqrt(covarianceMatrix[3][3]) + k1_stddev = np.sqrt(covarianceMatrix[4][4]) + k2_stddev = np.sqrt(covarianceMatrix[5][5]) + p1_stddev = np.sqrt(covarianceMatrix[10][10]) + p2_stddev = np.sqrt(covarianceMatrix[11][11]) + k3_stddev = np.sqrt(covarianceMatrix[6][6]) + + return ( + f_stddev, + cx_stddev, + cy_stddev, + k1_stddev, + k2_stddev, + p1_stddev, + p2_stddev, + k3_stddev, + ) + + +def incal_calibration_bounds( + calibio_path, cal_vol_alpha, cal_area_alpha, dof=None, num_vol_figs=10 +): + """Creates objects that determines if a point is inside the calibration regions + + From the `calibio_path`, the 3D points from the calibration board are used to define + the safe 3D volume. The 2D points of the image detections are used to define the + safe image area. The area/volume are defined using an alpha shape (technically an + alpha complex). An alpha shape is similar to a convex hull, but sets a limit on the + distance between two vertices for them to be connected. The max distances for the + volume and area is 1 / `cal_vol_alpha` and 1 / `cal_area_alpha` respectively. + + If the global variable `debug_show_cal_bounds` is True, this method also creates + debug images. + + One debug image is created for the safe image area. That image is green for pixels + inside the 'safe' region and red for pixels outside the safe region. The region is + not discretized to pixels, so the image is a (good) approximation. + + Several debug images are created for the safe 3D volume. Each debug image is a slice + of the 3D safe object. The slices are done at planes progressively farther from the + camera (not spherical shells of constant distance from the camera). Points in the + field of view (not accounting for lens distortion) are given. Points colored red are + unsafe. Points colored green are safe. + + Generates the following images when `debug_show_cal_bounds` (global) is True: + + ``3d_cal_points_camera.png`` + A scatter plot of the internal calibration points as viewed from the camera + position. X is the real world position relative to the camera horizontal + axis. Z is the real world position relative to the camera optical axis. + ``3d_cal_points_side.png`` + A scatter plot of the internal calibration points as viewed from a location + to the side of the camera. X is the real world position relative to the + camera horizontal axis. Y is the real world position relative to the camera + vertical axis. + ``image_area.png`` + The safe and unsafe locations in the image based on the internal calibration + points and the cal_area_alpha parameter + ``unsafe_volume_Z=*_inches.png`` + Where * is Z location of the planar slice in inches for the given image. + These images are like a wedding cake stack of the 3D calibration volume. + Each image is a slice of the volume at a given Z distance from the camera. + The image shows the safe and unsafe locations in that planar slice. + + Parameters + ---------- + calibio_path : str + Path to the calib.io saved calibration json + cal_vol_alpha : float + Used to define the 3D volume alpha shape. Alpha parameter is 1 / `cal_vol_alpha` + cal_area_alpha : float + Used to define the 3D image area alpha shape. Alpha parameter is 1 / + `cal_vol_alpha` + dof : tuple, optional + Depth of field. Optional, but must be given if global variable + `debug_show_cal_bounds` is True since it is used to generate debug images. The + first item of tuple is distance to the first slice of the 3D volume. The second + item is distance to the last slice. The slices are done in planes (not in + spherical shells of constant distance to the camera). + num_vol_figs : int, optional + Number of volume figures. Optional, but must be given if global variable + `debug_show_cal_bounds` is True since it is used to generate debug images. The + debug images of the calibration volume slice the volume into planes at + progressively farther distances. This input determines the number of slices + + Returns + ------- + cal_area_is_safe : callable + Function that takes image points in an array-like object and returns a + corresponding array of bools indicating which points are safe. + cal_vol_is_safe : callable + Function that takes 3D points in an array-like object and returns a + corresponding array of bools indicating which points are safe. + """ + # If the debug_show_cal_bounds is True, dof is required + assert (not debug_show_cal_bounds) or (debug_show_cal_bounds and dof is not None) + + calibio = parsers.read_json(calibio_path) + + # Read in the internal calibration from the file + img_size, uPSP_cameraMatrix, distCoeffs = incal_from_calibio(calibio_path) + cameraMatrix = parsers.convert_uPSP_cm_to_cv2_cm(uPSP_cameraMatrix, img_size) + + # Define the calibrated image area + detections = calibio["detections"] + img_pts = [] + for detection in detections: + featurePoints = detection["featurePoints"] + for pt in featurePoints: + img_pts.append((pt["point"]["x"], pt["point"]["y"])) + + img_pts = np.array(img_pts) + cal_area_is_safe = alpha_shape_is_safe(img_pts, cal_area_alpha) + + if debug_show_cal_bounds: + pixels = np.array(list(np.ndindex(img_size[1], img_size[0]))) + pixel_bools = cal_area_is_safe(pixels) + unsafe_points = np.array(pixels[np.where(~pixel_bools)[0]]) + safe_points = np.array(pixels[np.where(pixel_bools)[0]]) + + plt.figure("area") + ax = plt.gca() + ax.scatter(unsafe_points[:, 0], unsafe_points[:, 1], s=1, c="r", label="Unsafe") + ax.scatter(safe_points[:, 0], safe_points[:, 1], s=1, c="g", label="Safe") + ax.scatter( + img_pts[:, 0], img_pts[:, 1], s=0.25, c="b", label="Calibration Points" + ) + ax.invert_yaxis() + plt.legend(loc="upper left") + plt.xlim(0, img_size[1]) + plt.ylim(img_size[0], 0) + plt.xlabel("X (pixels)") + plt.ylabel("Y (pixels)") + plt.savefig("image_area.png") + plt.close("area") + + # Get the calibration target positions relative to the board frame + cal_pts_metric = [] + for cal_pt_metric in calibio["targets"][0]["objectPoints"]: + cal_pts_metric.append( + [cal_pt_metric["x"], cal_pt_metric["y"], cal_pt_metric["z"]] + ) + cal_pts_metric = np.array(cal_pts_metric) + cal_pts = cal_pts_metric * meter2inch + + # Get the locations of all 3D targets + poses = calibio["calibration"]["poses"] + cal_pts_rel_cam = [] + for pose in poses: + transformation = pose["transform"] + + # Get the rotation matrix from the quaternions + rvec = np.array( + [ + transformation["rotation"]["rx"], + transformation["rotation"]["ry"], + transformation["rotation"]["rz"], + ] + ) + rmat, __ = cv2.Rodrigues(rvec) + + # Get the translation vector + tvec_metric = np.array( + [ + transformation["translation"]["x"], + transformation["translation"]["y"], + transformation["translation"]["z"], + ] + ) + tvec = (tvec_metric * meter2inch).reshape(3, 1) + + cal_pts_rel_cam += photogrammetry.transform_3d_point( + rmat, tvec, cal_pts + ).tolist() + cal_pts_rel_cam = np.array(cal_pts_rel_cam) + cal_vol_is_safe = alpha_shape_is_safe(cal_pts_rel_cam, cal_vol_alpha) + + if debug_show_cal_bounds: + plt.figure("volume") + ax = plt.axes(projection="3d") + ax.scatter( + cal_pts_rel_cam[:, 0], + cal_pts_rel_cam[:, 1], + cal_pts_rel_cam[:, 2], + s=1, + c="b", + marker="o", + depthshade=False, + ) + ax.set_xlabel("X (inches)") + ax.set_ylabel("Y (inches)") + + ax.view_init(elev=-90.0, azim=-90) + visualization.axisEqual3D(ax) + plt.savefig("3d_cal_points_ceiling.png") + + ax.set_ylabel("") + ax.set_zlabel("Z (inches)") + ax.view_init(elev=0.0, azim=-90) + plt.savefig("3d_cal_points_camera.png") + plt.close("volume") + + if debug_show_cal_bounds: + pixel_extremes = [ + max(np.abs((pixels[:, 0] - cameraMatrix[0][2]))), + max(np.abs((pixels[:, 1] - cameraMatrix[1][2]))), + ] + max_reach = max(pixel_extremes) / cameraMatrix[0][0] * dof[1] + max_reach *= 1.1 + + for z in np.linspace(dof[0], dof[1], num_vol_figs): + xs = (pixels[:, 0] - cameraMatrix[0][2]) / cameraMatrix[0][0] * z + ys = (pixels[:, 1] - cameraMatrix[1][2]) / cameraMatrix[0][0] * z + zs = np.full(xs.shape, z) + pts = np.array([xs, ys, zs]).T + + pt_bools = cal_vol_is_safe(pts) + unsafe_points = np.array(pts[np.where(~pt_bools)[0]]) + safe_points = np.array(pts[np.where(pt_bools)[0]]) + + plt.figure("volume") + ax = plt.axes(projection="3d") + ax.scatter( + unsafe_points[:, 0], + unsafe_points[:, 1], + unsafe_points[:, 2], + s=1, + c="r", + marker="o", + depthshade=False, + label="Unsafe", + ) + ax.scatter( + safe_points[:, 0], + safe_points[:, 1], + safe_points[:, 2], + s=1, + c="g", + marker="o", + depthshade=False, + label="Safe", + ) + ax.set_xlim(-max_reach, max_reach) + ax.set_ylim(-max_reach, max_reach) + ax.view_init(elev=-90.0, azim=-90) + visualization.axisEqual3D(ax) + + digits = np.floor(np.log10(dof[1])).astype(np.int64) + 1 + fig_idx = "{:.2f}".format(z).rjust(digits + 3, "0") + plt.xlabel("X (inches)") + plt.ylabel("Y (inches)") + plt.legend() + plt.savefig("unsafe_volume_Z=" + fig_idx + "_inches.png") + plt.close("volume") + + return cal_area_is_safe, cal_vol_is_safe + + +def incal_calibration_bounds_debug(): + """Returns debugging :func:`incal_calibration_bounds` objects + + The returned :func:`incal_calibration_bounds` functions True for every input point + """ + + def is_always_safe(pts): + return np.full(pts.shape[0], True) + + return is_always_safe, is_always_safe + + +def alpha_shape_is_safe(pts, alpha): + """Returns an alpha shape generated from points and the `alpha` parameter + + Returns an alpha shape constructed from the points and the given alpha parameters. + An alpha shape is similar to a convex hull, but sets a maximum on the distance + between two vertices. The distances is 1 / `alpha`. An alpha shape is a convex hull + if the alpha parameter is 0. + + The shape may be more accurately referred to as an alpha complex (rather than alpha + shape) since it has poligonal edges, but many sources use alpha shape to refer to + both. + + Parameters + ---------- + pts : np.ndarray, shape (n, k) of floats + Calibration points to create the alpha shape from. ``n`` is the number of points + ``k`` is the dimensionality of the points (2 for image points, 3 for real world + points) + alpha : float + Alpha parameter of the alpha shape. Larger means more points are rejected. Must + be non-negative. Negative values are clipped to 0. + + Returns + ------- + is_safe : callable + Function that accepts an array-like object (list, np.array, etc) of points and + returns a list of booleans of the same length. ``return[i]`` corresponds to + ``point[i]`` and True means that point is inside the alpha shape (False means it + is not). + """ + # For numerical stability, clip alpha to a very small value + alpha = np.clip(alpha, 1e-30, None) + + # Get the Delaunay tessellation of the pts + tess = Delaunay(pts) + + # Scipy Delaunay returns tessellation as incides, get tessellation as points + tess_pts = np.take(pts, tess.simplices, axis=0) + + # Get the number of dimensions + num_dims = tess.simplices.shape[1] - 1 + + # Get the radius of the circumsphere (circumcircle in 2D) for each of the tessellation + # Equations for circumcircle and circumsphere radius from wolframalpha: + # https://mathworld.wolfram.com/Circumcircle.html + # https://mathworld.wolfram.com/Circumsphere.html + ones = np.ones((tess_pts.shape[0], tess_pts.shape[1], 1)) + normsq = np.sum(tess_pts ** 2, axis=2)[:, :, None] + a = np.concatenate((tess_pts, ones), axis=2) + D = np.concatenate((normsq, a), axis=2) + a_det = np.linalg.det(a) + c_det = np.power(-1, num_dims + 1) * np.linalg.det(np.delete(D, -1, axis=2)) + + det_sum = 0 + for k in range(num_dims): + sign = np.power(-1, num_dims + k + 1) + D_k = sign * np.linalg.det(np.delete(D, k + 1, axis=2)) + det_sum += np.power(D_k, 2) + + # Calculate the discriminant + # Clip values that are negative due to machine precision + disc = det_sum - 4 * a_det * c_det + disc = np.where( + (disc < 0.0) & (np.abs(disc) < np.finfo(np.float64).eps), + np.finfo(np.float32).eps, + disc, + ) + + # Calculate the denominator, but clip the abs value of a_det to avoid 0/0 issues + # a_det is clipped 10 orders of magnitude larger than disc so 0/0 = 0 + # That way 0/0 tessellations are rejected + den = 2 * np.clip(np.abs(a_det), 1e-20, None) + + # Calculate the radius of each Delaunay tessellation + tess_radii = np.divide(np.sqrt(disc), den) + + # Get a set of the indices of the tessellation that are a part of the alpha shape + alpha_shape_tess = np.where(tess_radii < np.divide(1.0, alpha))[0] + + # Return a fuction that determines if a point is safe + # is_safe returns True if the point is inside the alpha shape boundary, and + # returns False if it is outside the alpha shape boundary + def is_safe(pts): + # Find the simplex that contains the point (if it exists) + simplex_idx = tess.find_simplex(pts) + + # Check if the index is in the set of safe indexes + # If it is, return True. Otherwise return False + return np.where(np.isin(simplex_idx, alpha_shape_tess), True, False) + + # Return the is_safe function + return is_safe diff --git a/python/upsp/cam_cal_utils/parsers.py b/python/upsp/cam_cal_utils/parsers.py index ab6bf6c..6c5c07e 100644 --- a/python/upsp/cam_cal_utils/parsers.py +++ b/python/upsp/cam_cal_utils/parsers.py @@ -1,96 +1,117 @@ import xml.etree.ElementTree as ET import json import csv -import os import numpy as np -def read_tgts(tgts_file_path, output_target_types=None, read_all_fields=False, read_pK=False): - """ - Read in the tgts file at the specified tgts_file_path - - Inputs: - tgts_file_path - file path to the tgts file - output_target_types - Options. Can be given as a string or list of strings. - If given, only targets with a type in output_target_types - will be in the output - Outputs: - targets - list of targets. Each target is of the form - {'type' : class_string, 'tvec' : [x, y, z], 'norm', : [x, y, z]} - - """ +def read_tgts(tgts_file_path, output_target_types=None): + """Returns the targets in the tgts file + + Parameters + ---------- + tgts_file_path : str + File path to the tgts file + output_target_types : str, list of str, optional + If not None, only targets with a type of or in `output_target_types` will be + read. If None, all target types will be read + + Returns + ------- + targets : list of dict + List of targets. Each target is of the form:: + { + 'target_type' : class_string, 'tvec' : [x, y, z], 'norm', : [x, y, z], + 'size': float, 'name': str, 'idx': int, 'zones': [i, j, k] + } + """ + # Package output_target_types if (output_target_types is not None) and (type(output_target_types) is not list): output_target_types = [output_target_types] targets = [] - with open(tgts_file_path, 'r') as f: - csv_reader = csv.reader(f, delimiter=' ') + with open(tgts_file_path, "r") as f: + csv_reader = csv.reader(f, delimiter=" ") line_type = None for row in csv_reader: + # Populate the line from the csv line = [] for item in row: - if (item != ''): + if item != "": line.append(item) - if (len(line) != 1): - # target block contains all sharpie dots and masked kulites - if (line_type == 'target'): + # If the length of the line is greater than 1, attempt to populate the + # the list of targets + if len(line) > 1: + # Read in items listed under '*Targets' + if line_type == "*Targets": # If the last element has 'st' for 'sharpie target' it is a dot - if 'st' in line[-1]: - target_type = 'dot' - + if "st" in line[-1]: + target_type = "dot" + # If the last element has 'mK' for 'masked Kulite' it is a Kulite - elif 'mK' in line[-1]: - target_type = 'kulite' - - elif (read_pK and 'pK' in line[-1]): - target_type = 'kulite' + # (and visible to the camera) + elif "mK" in line[-1]: + target_type = "kulite" + + # If the last element has 'pK' for 'painted Kulite' it is a Kulite + # (and not visible to the camera) + elif "pK" in line[-1]: + target_type = "painted_kulite" - # Otherwise this item is junk (unmasked Kulite, misc type, etc) + # Otherwise this item is unknown else: - continue + target_type = line[-1] - # Otherwise the target type is unknown + # Ignore items in other categories of the tgts file else: continue - - # TODO: Read in tvec and norm as numpy arrays + # If output_target_types was not given, or the target_type is one of the # output_target_types given, grab this target - if (output_target_types is None) or (target_type in output_target_types): - target = {'target_type' : target_type, - 'tvec' : [float(x) for x in line[1:4]], - 'norm' : [float(x) for x in line[4:7]], - 'size' : float(line[7]), - 'name' : line[-1]} - - if read_all_fields: - target['idx'] = int(line[0]) - target['zones'] = (int(line[8]), int(line[9]), int(line[10])) - - targets.append(target) - + if (output_target_types is None) or ( + target_type in output_target_types + ): + targets.append( + { + "target_type": target_type, + "tvec": np.expand_dims([float(x) for x in line[1:4]], 1), + "norm": np.expand_dims([float(x) for x in line[4:7]], 1), + "size": float(line[7]), + "name": line[-1], + "idx": int(line[0]), + "zones": (int(line[8]), int(line[9]), int(line[10])), + } + ) + + # If the length of the line is 1 or 0, set the line_type to the line's item + # if it has one, or set it to None if it has no items else: - if line[0] == '*Targets': - line_type = 'target' - else: - line_type = None + line_type = line[0] if len(line) == 1 else None return targets -def read_pascal_voc(annot_path, read_full_annot=False): - """ - read_pascal_voc inputs the path(s) to label(s) and returns an list of dicts. Where - each dict contains the class, bounding box, and flags of an image object - Input: - annot_path - this can be a string, or a list of strings (or list-like). Each - string is the path to a label - Output: - If annot_path is a single label path, the output is a list of dicts - If annot_path is a list of label paths (or list-like), the output is a list of - lists. Each inner list if a list of dicts. +def read_pascal_voc(annot_path): + """Return image objects from a PASCAL VOC XML file + + Reads the input file(s) and returns an list of dicts, where each dict contains the + class, bounding box, and flags of an image object. + + Parameters + ---------- + annot_path : list or str + This can be a string, or a list of strings (or list-like). Each string is the + path to a label + + Returns + ------- + out : dict or list + If `annot_path` is a single label path, the output is a list of dicts. If + `annot_path` is a list of label paths (or list-like), the output is a list of + lists, where each inner list if a list of dicts. + Each dict is represents an image object and has the keys 'class', 'x1', + 'x2', 'y1', 'y2', 'difficult', and 'truncated'. """ # Assert that annot_path is of the right form @@ -110,78 +131,75 @@ def read_pascal_voc(annot_path, read_full_annot=False): element = et.getroot() # Parses PASCAL VOC data into python objects - element_objs = element.findall('object') - - # Read the annotation information if applicable - if read_full_annot: - annotation_data = { - 'filename': element.find('filename').text, - 'width': int(element.find('size').find('width').text), - 'height': int(element.find('size').find('height').text), - 'targets': []} - + element_objs = element.findall("object") + # Populate the annotation_data with the filepath, and image # width & height targets = [] for element_obj in element_objs: # Populate the class_count and class_mapping return variables - class_name = element_obj.find('name').text + class_name = element_obj.find("name").text # Populate bounding box information - obj_bbox = element_obj.find('bndbox') - x1 = int(round(float(obj_bbox.find('xmin').text))) - y1 = int(round(float(obj_bbox.find('ymin').text))) - x2 = int(round(float(obj_bbox.find('xmax').text))) - y2 = int(round(float(obj_bbox.find('ymax').text))) - difficulty = int(element_obj.find('difficult').text) - truncated = int(element_obj.find('truncated').text) + obj_bbox = element_obj.find("bndbox") + x1 = int(round(float(obj_bbox.find("xmin").text))) + y1 = int(round(float(obj_bbox.find("ymin").text))) + x2 = int(round(float(obj_bbox.find("xmax").text))) + y2 = int(round(float(obj_bbox.find("ymax").text))) + difficulty = int(element_obj.find("difficult").text) + truncated = int(element_obj.find("truncated").text) # LabelImg used the same coordinate system as OpenCV # LabelImg clips x1 and y1 to 1, even if the true value is 0 # For future work, that should be fixed in the annotations. if truncated: - if (x1 == 1): + if x1 == 1: x1 = 0 - if (y1 == 1): + if y1 == 1: y1 = 0 - - # Populate annotation_data's bounding box field - targets.append({'class': class_name, - 'x1': x1, 'x2': x2, 'y1': y1, 'y2': y2, - 'difficult': difficulty, - 'truncated': truncated}) - # Return the annotation information if applicable - if read_full_annot: - annotation_data['targets'] = targets - return annotation_data + # Populate annotation_data's bounding box field + targets.append( + { + "class": class_name, + "x1": x1, + "x2": x2, + "y1": y1, + "y2": y2, + "difficult": difficulty, + "truncated": truncated, + } + ) - else: - return targets + return targets -def read_wind_tunnel_data(wtd_file_path, items=['ALPHA', 'BETA', 'PHI', 'STRUTZ']): - """ - Returns specified wind tunnel data from wtd_file_path +def read_wind_tunnel_data(wtd_file_path, items=("ALPHA", "BETA", "PHI", "STRUTZ")): + """Read specified wind tunnel data from `wtd_file_path` - Inputs: - wtd_file_path - filepath to wind tunnel data file - items - items requested from that file. By default ALPHA, BETA, PHI and STRUTZ - are returned + Parameters + ---------- + wtd_file_path : str + Filepath to wind tunnel data file + items : container, optional + Items requested from the file. By default ALPHA, BETA, PHI and STRUTZ are + returned. - Output: - dictionary with keys of specified items and values of associated wtd values + Returns + ------- + tunnel_vals : dict + Dictionary with keys of specified items and values of associated wtd values. """ # Read in the wind tunel data file - with open(wtd_file_path, 'r') as f: + with open(wtd_file_path, "r") as f: f.readline() - csv_reader = csv.DictReader(f, delimiter='\t') + csv_reader = csv.DictReader(f, delimiter="\t") tunnel_vals = next(csv_reader) # Convert values to floats - tunnel_vals = {k : float(v) for k, v in tunnel_vals.items()} + tunnel_vals = {k: float(v) for k, v in tunnel_vals.items()} # Remove all but the relevant for k in list(tunnel_vals.keys()): @@ -192,28 +210,43 @@ def read_wind_tunnel_data(wtd_file_path, items=['ALPHA', 'BETA', 'PHI', 'STRUTZ' def convert_cv2_cm_to_uPSP_cm(cameraMatrix, dims): - """ - # Converts standard camera matrix to uPSP - - Inputs: - cameraMatrix - [[fx, 0, cx], [0, fy, cy], [0, 0, 1]] - dims - image dimensions - - Output: - uPSP_cameraMatrix - [[fx, 0, dcx], [0, fy, dcy], [0, 0, 1]] - + """Converts a standard camera matrix to a uPSP camera matrix + + OpenCV (and the standard) cameraMatrix uses the absolute position of the optical + principal point. Since uPSP often crops images, the absoulte position varies + from configuration to configuration even if the optics haven't changed. So instead, + the position of the principal point relative to the image center is saved. + + Parameters + ---------- + cameraMatrix : np.ndarray, shape (3, 3) + Camera matrix of the form: ``[[fx, 0, cx], [0, fy, cy], [0, 0, 1]]`` + where f is the focal length and (cx, cy) is + the absolute position of the optical principal point + dims : tuple, length 2 + Image dimensions (width, height) + + Returns + ------- + uPSP_cameraMatrix : np.ndarray, shape (3, 3) + Converted camera matrix of the form: ``[[fx, 0, dcx], [0, fy, dcy], [0, 0, 1]]`` + cx = w/2 + dcx and cy = h/2 + dcy where w and h are the image width and height + + See Also + -------- + convert_uPSP_cm_to_cv2_cm : inverse conversion """ # Cx (Principal Point X) cx = cameraMatrix[0][2] - + # Delta Cx (Principal Point Y) cy = cameraMatrix[1][2] # Get offset from image center and principal point dcx = cx - (dims[1] / 2) dcy = cy - (dims[0] / 2) - + # Return a copy of the uPSP cameraMatrix with the modified values uPSP_cameraMatrix = np.copy(cameraMatrix) uPSP_cameraMatrix[0][2] = dcx @@ -223,28 +256,37 @@ def convert_cv2_cm_to_uPSP_cm(cameraMatrix, dims): def convert_uPSP_cm_to_cv2_cm(uPSP_cameraMatrix, dims): + """Converts a uPSP camera matrix to a standard camera matrix + + OpenCV (and the standard) cameraMatrix uses the absolute position of the optical + principal point. Since uPSP often crops images, the absoulte position varies + from configuration to configuration even if the optics haven't changed. So instead, + the position of the principal point relative to the image center is saved. To use + OpenCV functions, it has to be converted back to the standard OpenCV camera matrix. + + Parameters + ---------- + uPSP_cameraMatrix : np.ndarray, shape (3, 3) + Camera matrix of the form: ``[[fx, 0, cx], [0, fy, cy], [0, 0, 1]]`` + where f is the focal length and (cx, cy) is + the absolute position of the optical principal point + dims : tuple, length 2 + Image dimensions (width, height) + + Returns + ------- + cameraMatrix : np.ndarray, shape (3, 3) + Converted camera matrix of the form: ``[[fx, 0, cx], [0, fy, cy], [0, 0, 1]]`` + cx = w/2 + dcx and cy = h/2 + dcy where w and h are the image width and height + + See Also + -------- + convert_uPSP_cm_to_cv2_cm : inverse conversion """ - The standard method of storing camera intrinsic parameters is by saving the absolute - position of the principal point. This is the case because in the standard use - case the image used from a camera has a constant resolution - uPSP uses variable resolution to save storage/memory when possible. The resolution - change comes from a center from of the image center. Therefore, instead of the - absolute position of the principal point reamining constant, the offset from - the image center remains constant - This function converts between the uPSP offset and the industry standard absolute - position - - Inputs: - uPSP_cameraMatrix - [[fx, 0, dcx], [0, fy, dcy], [0, 0, 1]] - dims - image dimensions - - Output: - cameraMatrix - [[fx, 0, cx], [0, fy, cy], [0, 0, 1]] - """ - + # Delta Cx (Principal Point X) dcx = uPSP_cameraMatrix[0][2] - + # Delta Cx (Principal Point Y) dcy = uPSP_cameraMatrix[1][2] @@ -260,81 +302,162 @@ def convert_uPSP_cm_to_cv2_cm(uPSP_cameraMatrix, dims): return cameraMatrix -def read_internal_params(camera, internal_cal_dir, dims, read_sensor=False): +def read_internal_params(internal_cal_path, dims, read_sensor=False): + """ Returns the internal (intrinsic) camera parameters + + Parameters + ---------- + internal_cal_dir : str + Path to internal calibrations + dims : tuple + Image dimensions (image height, image width) + read_sensor: bool, optional + If True, additionally read and return the sensor resolution and size + + Returns + ------- + cameraMatrix : np.ndarray, shape (3, 3) + Camera matrix of the form ``[[fx, 0, cx], [0, fy, cy], [0, 0, 1]]`` + where f is the focal length and (cx, cy) + is the position of the optical principal point relative to the image center + distCoeffs : np.ndarray, shape (5,) + Distortion coefficients of the form ``[k1, k2, p1, p2, k3]`` + where k1, k2, and k3 are the radial distortion terms and p1 and p2 are the + tangential distortion terms + sensor_resolution : np.ndarray, shape (3,) + Camera sensor resolution, only returned if `read_sensor` is True + sensor_size : np.ndarray, shape (2,) + Camera sensor physical size, only returned if `read_sensor` is True """ - Given the internal calibraiton directory and camera number, returns the internal - (intrinsic) camera parameters - - Input: - Camera - camera number (int or str) - internal_cal_dir - path to internal calibrations - dims - (image height, image width) - - Output: - [cameraMatrix, distCoeffs] - intrinsics in openCV format - cameraMatrix - [[fx, 0, cx], [0, fy, cy], [0, 0, 1]] - distCoeffs - [k1, k2, p1, p2, k3] - """ - - camera = str(camera) - with open(os.path.join(internal_cal_dir, 'camera0' + camera + '.json'), 'r') as f: + # Read the internal calibration parameters + with open(internal_cal_path, "r") as f: incal = json.load(f) - uPSP_cameraMatrix = np.array(incal['uPSP_cameraMatrix']) - distCoeffs = np.array(incal['distCoeffs']) + uPSP_cameraMatrix = np.array(incal["uPSP_cameraMatrix"]) + distCoeffs = np.array(incal["distCoeffs"]) + # Convert the written uPSP_cameraMatrix to the OpenCV cameraMatrix cameraMatrix = convert_uPSP_cm_to_cv2_cm(uPSP_cameraMatrix, dims) # If read_sensor is False, just return the cameraMatrix and distCoeffs if not read_sensor: return cameraMatrix, distCoeffs - + # Otherwise return the sensor parameters as well else: - sensor_resolution = np.array(incal['sensor_resolution']) - sensor_size = np.array(incal['sensor_size']) + sensor_resolution = np.array(incal["sensor_resolution"]) + sensor_size = np.array(incal["sensor_size"]) + return cameraMatrix, distCoeffs, sensor_resolution, sensor_size -def read_camera_params(camera, cal_dir, dims, read_sensor=False): - """ - Given calibration directory and camera number, returns the camera calibration - parameters - - Input: - Camera - camera number (int or str) - cal_dir - path to camera calibrations - dims - (image height, image width) - - Output: - [cameraMatrix, distCoeffs, rmat, tvec] - calibration parameters in openCV format - cameraMatrix - [[fx, 0, cx], [0, fy, cy], [0, 0, 1]] - distCoeffs - [k1, k2, p1, p2, k3] - rmat - (3x3) rotation matrix from camera to tunnel - tvec - (3,) translation vector from camera to tunnel +def read_camera_tunnel_cal(cal_path, dims, read_sensor=False): + """ Returns the internal (intrinsic) and external (extrinsic) camera calibration parameters + + Parameters + ---------- + cal_path : str + Path to camera calibration + dims : tuple + Image dimensions (image height, image width) + read_sensor: bool, optional + If True, additionally read and return the sensor resolution and size + + Returns + ------- + cameraMatrix : np.ndarray, shape (3, 3) + Camera matrix of the form ``[[fx, 0, cx], [0, fy, cy], [0, 0, 1]]`` + where f is the focal length and (cx, cy) + is the position of the optical principal point relative to the image center + distCoeffs : np.ndarray, shape (5,) + Distortion coefficients of the form ``[k1, k2, p1, p2, k3]`` + where k1, k2, and k3 are the radial distortion terms and p1 and p2 are the + tangential distortion terms + rmat : np.ndarray, shape (3, 3) + Rotation matrix from camera to tunnel + tvec : np.ndarray, shape (3,) + Translation vector from camera to tunnel + sensor_resolution : np.ndarray, shape (2,) + Sensor size of camera in pixels, only returned if `read_sensor` is True + sensor_size : np.ndarray, shape (2,) + Sensor size of camera in inches, only returned if `read_sensor` is True """ - - camera = str(camera).rjust(2, '0') - with open(os.path.join(cal_dir, 'camera' + camera + '.json'), 'r') as f: + # Read the camera calibration parameters + with open(cal_path, "r") as f: cal = json.load(f) + uPSP_cameraMatrix = np.array(cal["uPSP_cameraMatrix"]) + distCoeffs = np.array(cal["distCoeffs"]) + tvec = np.array(cal["tvec"]).reshape(3, 1) + rmat = np.array(cal["rmat"]) - uPSP_cameraMatrix = np.array(cal['uPSP_cameraMatrix']) + # Convert the written uPSP_cameraMatrix to the OpenCV cameraMatrix cameraMatrix = convert_uPSP_cm_to_cv2_cm(uPSP_cameraMatrix, dims) - distCoeffs = np.array(cal['distCoeffs']) - tvec = np.array(cal['tvec']) - rmat = np.array(cal['rmat']) - # If read_sensor is False, just return the cameraMatrix and distCoeffs if not read_sensor: - return cameraMatrix, distCoeffs, rmat, tvec - + return rmat, tvec, cameraMatrix, distCoeffs + # Otherwise return the sensor parameters as well else: - sensor_resolution = np.array(cal['sensor_resolution']) - sensor_size = np.array(cal['sensor_size']) - return cameraMatrix, distCoeffs, rmat, tvec, sensor_resolution, sensor_size + sensor_resolution = np.array(cal["sensor_resolution"]) + sensor_size = np.array(cal["sensor_size"]) + + return rmat, tvec, cameraMatrix, distCoeffs, sensor_resolution, sensor_size + + +def read_json(path): + """Safely reads a json file and returns the associated dict -# TODO: Convert this into a read proc-default.json -def read_test_config(test_config_path): - with open(test_config_path, 'r') as f: + Parameters + ---------- + path : path-like + File path to the tgts file + + Returns + ------- + dict + dict of json file items + """ + # Safely read the json file + with open(path, "r") as f: return json.load(f) + + +def read_test_config(path): + """Safely reads a test config file and returns the associated dict with arrays + + Parameters + ---------- + path : path-like + File path to the tgts file + + Returns + ------- + dict + dict of json file items + """ + # Safely read the json file + with open(path, "r") as f: + test_config = json.load(f) + + # Transcribe the test config into numpy arrays if possible + test_config_np = {} + for key, val in test_config.items(): + try: + # If the length is 3, it is a (3, 1) or (3, 3) + if len(val) == 3: + if type(val[0]) == float: + test_config_np[key] = np.expand_dims(val, 1) + elif len(val[0]) == 3: + test_config_np[key] = np.array(val) + else: + test_config_np[key] = val + + # If the length if not 3, it is a float or long array + else: + test_config_np[key] = val + + # If an exception is raised, just add the actual value for the key + except Exception as e: + test_config_np[key] = val + + return test_config_np diff --git a/python/upsp/cam_cal_utils/photogrammetry.py b/python/upsp/cam_cal_utils/photogrammetry.py index 52689ce..49a5333 100644 --- a/python/upsp/cam_cal_utils/photogrammetry.py +++ b/python/upsp/cam_cal_utils/photogrammetry.py @@ -3,27 +3,27 @@ import copy from scipy.spatial.transform import Rotation as scipy_Rotation -import visualization - -np.set_printoptions(suppress=True) +np.set_printoptions(linewidth=180, precision=4, threshold=np.inf) #--------------------------------------------------------------------------------------- # General Photogrammetry Utility Functions def rot(angle, axis): - """Returns rotation matrix when rotating about the given axis by angle degrees - + """Rotation matrix for angle-axis transformation + + An identity matrix is rotated about the given axis by the given angle (in degrees) + Parameters ---------- angle : float angle of rotation (in degrees) about the given axis - axis : + axis : {'x', 'y', 'z'} axis to rotate about Returns ---------- - np.ndarray (3, 3), float + np.ndarray, shape (3, 3), float Rotation matrix of an identity matrix rotated about the given axis by the given amount """ @@ -50,20 +50,21 @@ def rot(angle, axis): def invTransform(R, t): - """Returns the inverse transformation of the given R and t - + """Returns the inverse transformation of the given `R` and `t` + Parameters ---------- - R : np.ndarray (3, 3), float + R : np.ndarray, shape (3, 3), float Rotation Matrix - t : np.ndarray (3, 1) or (3,), float + t : np.ndarray (3, 1), float Translation Vector Returns ---------- - tuple - (rmat_i, tvec_i) where rmat_i is the inverse rotation matrix (np.ndarray (3,3), - float) and tvec_i is the inverse translation vector (np.ndarray (3,1), float) + rmat_i : np.ndarray, shape (3, 3), float + Inverse rotation matrix + tvec_i : np.ndarray, shape (3, 1), float + Inverse translation vector """ R_transpose = R.transpose() @@ -72,15 +73,15 @@ def invTransform(R, t): def isRotationMatrix(R): """Checks if a matrix is a valid rotation matrix - + Parameters ---------- - R : np.ndarray (3, 3), float + R : np.ndarray, shape (3, 3), float Rotation Matrix Returns ---------- - Boolean + bool True if R is a valid rotation matrix, and False if it is not """ @@ -91,80 +92,120 @@ def isRotationMatrix(R): def isRotationMatrixRightHanded(R): - """ - Returns True if matrix is right handed, and False is left handed - """ + """ Returns True if matrix is right handed, and False it is not + + Parameters + ---------- + R : np.ndarray (3, 3), float + Rotation Matrix + Returns + ---------- + boolean + True if R is a right handed rotation matrix, and False if it is not + """ return np.dot(np.cross(R[:,0], R[:,1]), R[:,2]) > 0. -def rotationMatrixToTunnelAngles(R) : +def rotationMatrixToTunnelAngles(R): """Converts rotation matrix to tunnel angles - + Parameters ---------- - R : np.ndarray (3, 3), float + R : np.ndarray, shape (3, 3), float Rotation Matrix - + Returns ---------- - np.ndarray (3,), float + np.ndarray (3, 1), float Array of tunnel angle (alpha, beta, phi) - + See Also ---------- isRotationMatrix : Checks if a matrix is a valid rotation matrix """ # Check that the given matrix is a valid roation matrix assert(isRotationMatrix(R)) - + # Use scipy to convert to yzx euler angles r = scipy_Rotation.from_matrix(R) alpha, beta, neg_phi = np.array(r.as_euler('yzx', degrees=True)) - + # Return the tunnel angles - return np.array([alpha, beta, -neg_phi]) + return np.expand_dims([alpha, beta, -neg_phi], 1) + + +def transform_3d_point(rmat, tvec, obj_pts): + """Transform 3D points from the object frame to the camera frame. + + `rmat` and `tvec` are the transformation from the camera to the object's frame. + ``obj_pts[i]`` is the position of point ``i`` relative to the object frame. The + function returns the points relative to the camera. + + Parameters + ---------- + rmat : np.ndarray, shape (3, 3), float + Rotation matrix from camera to object + tvec : np.ndarray (3, 1), float + Translation vector from camera to object + obj_pts : np.ndarray (n, 3), float + List-like of 3D positions. The positions are relative to the object frame + defined by rmat and tvec + + Returns + ---------- + np.ndarray (n, 3) of floats + Array of transformed points. return[i] is the transformed obj_pts[i] + """ + obj_pts_tf = np.squeeze(np.array([np.matmul(rmat, np.expand_dims(pt, 1)) + tvec for pt in obj_pts]), axis=2) + return obj_pts_tf def project_3d_point(rmat, tvec, cameraMatrix, distCoeffs, obj_pts, ret_jac=False, ret_full_jac=False): """Projects targets into an image. - + Parameters ---------- - rmat : np.ndarray (3, 3), float + rmat : np.ndarray, shape (3, 3), float Rotation matrix from camera to object - tvec : np.ndarray (3, 1) or (3,), float + tvec : np.ndarray, shape (3, 1), float Translation vector from camera to object - cameraMatrix : np.ndarray (3x3), float + cameraMatrix : np.ndarray, shape (3, 3), float The (openCV formatted) camera matrix for the camera - distCoeffs : np.ndarray (5x1) or (5,), float + distCoeffs : np.ndarray (1, 5), float The (openCV formatted) distortion coefficients for the camera - ret_jac : Boolean, optional default=False - If True, returns the jacobian. If False only the projection is returned - ret_full_jac : Boolean, optional default=False - If True, returns full 15 term jacobian (delta rvec, delta tvec, delta focal + obj_pts : np.ndarray, shape (n, 3), float + The 3D position of the points on the object (relative to the object frame) to + be projected + ret_jac : bool, optional, default=False + If True, returns the Jacobian. If False only the projection is returned + ret_full_jac : bool, optional, default=False + If True, returns full 15 term Jacobian (delta rvec, delta tvec, delta focal length, delta principal point, delta distortion). If False, returns the 6 term - jacobian (delta rvec, delta tvec) + Jacobian (delta rvec, delta tvec) Returns ---------- - projs or (projs, jacs) - If ret_jac is False, returns just projs. If ret_jac is True, returns projs and - jacs as a tuple. - projs is np.ndarray (N,2), float and the projected pixel positions - jacs is np.ndarray (N, 2, 6) or (N, 2, 15), float and is the jacobian of the - projected pixel location. jacs[i] is associated with projs[i]. jacs[:, 0, :] is - the x axis, jacs[:, 1, :] is the y axis. Axis 2 of jacs is the partial - derivative related to the inputs (delta rvec, delta tvec, etc) + projs, np.ndarray, shape (n, 2), float + Projected pixel positions + jacs : np.ndarray, shape (n, 2, 6) or (n, 2, 15), float + Jacobian of the projected pixel location. Only returned if `ret_jac` is True. + ``jacs[i]`` is associated with ``projs[i]``. ``jacs[:, 0, :]`` is the x axis, + ``jacs[:, 1, :]`` is the y axis. Axis 2 of `jacs` is the partial derivative + related to the inputs (delta rvec, delta tvec, etc). """ + if not len(obj_pts): + return np.array([]) + # Convert the rmat into an rvec rvec, _ = cv2.Rodrigues(rmat) # Project the points onto the image + obj_pts = np.array(obj_pts).astype(np.float64) projs, jacs = cv2.projectPoints(obj_pts, rvec=rvec, tvec=tvec, cameraMatrix=cameraMatrix, distCoeffs=distCoeffs) - + # Squeeze the extra dimension out projs = projs.squeeze(axis=1) @@ -185,36 +226,41 @@ def project_3d_point(rmat, tvec, cameraMatrix, distCoeffs, obj_pts, ret_jac=Fals #--------------------------------------------------------------------------------------- -# Target Functions +# Target Functions def transform_targets(rmat, tvec, tgts): - """Transform the targets by the given transformation values + """Transform the targets by the given transformation values. + + `rmat` and `tvec` are the transformation from the camera to the object's frame. + ``tgts[i]['tvec']`` is the position of target ``i`` relative to the object frame. + ``tgts[i]['norm']`` is the normal of target ``i`` relative to the object frame. The + function returns the targets such that 'tvec' and 'norm' are relative to the camera. Parameters ---------- - rmat : np.ndarray (3, 3), float + rmat : np.ndarray, shape (3, 3), float Rotation matrix from camera to object - tvec : np.ndarray (3, 1) or (3,), float + tvec : np.ndarray, shape (3, 1), float Translation vector from camera to object - tgts : list of targets - Each target is a dict with (at a minimum) a 'tvec', and 'norm' attribute. All - other attributes will be copied to the transformed targets. The 'tvec' attribute - gives the target's location and the 'norm' attribute gives the target's normal - vector - + tgts : list of dict + Each target is a dict with (at a minimum) a 'tvec', and 'norm' attribute. 'tvec' + is the position of the target relative to the object frame. 'norm' is he normal + of the target relative to the object frame. All other attributes will be copied + to the transformed targets. + Returns ---------- - transformed targets - list of targets. Each target has the attributes of the input targets, except - the transformed targets are transformed by the given rmat and tvec transformation + tgts_tf : list of dict + List of targets. Each target has the attributes of the input targets, except + the transformed targets are transformed to be relative to the camera """ tgts_tf = [] for tgt in tgts: # Transform the target tvec and norm to be relative to the camera - tgt_tf_tvec = np.matmul(rmat, tgt['tvec']) + tvec + tgt_tf_tvec = transform_3d_point(rmat, tvec, tgt['tvec'].T).T tgt_tf_norm = np.matmul(rmat, tgt['norm']) - + tgt_copy = copy.deepcopy(tgt) tgt_copy['tvec'] = tgt_tf_tvec tgt_copy['norm'] = tgt_tf_norm @@ -229,29 +275,29 @@ def project_targets(rmat, tvec, cameraMatrix, distCoeffs, tgts, dims=None): Parameters ---------- - rmat : np.ndarray (3, 3), float + rmat : np.ndarray, shape (3, 3), float Rotation matrix from camera to object - tvec : np.ndarray (3, 1) or (3,), float + tvec : np.ndarray (3, 1), float Translation vector from camera to object - tgts : list of targets + cameraMatrix : np.ndarray, shape (3, 3), float + The (openCV formatted) camera matrix for the camera + distCoeffs : np.ndarray, shape (5, 1) or (5,), float + The (openCV formatted) distortion coefficients for the camera + tgts : list of dict Each target is a dict with (at a minimum) 'tvec' and 'target_type' attributes. The 'tvec' attribute gives the target's location and 'target_type' is a string denoting the type of target (most commonly 'dot' or 'kulite') - cameraMatrix : np.ndarray (3x3), float - The (openCV formatted) camera matrix for the camera - distCoeffs : np.ndarray (5x1) or (5,), float - The (openCV formatted) distortion coefficients for the camera - dims : np.ndarray (2, 1) or (2,), float, or equivalent or None. Optional default=None + dims : array_like, length 2, optional Dimensions of image in (width, height). If not None, any targets that get projected outside the image (x < 0, x > dims[1], y < 0, y > dims[0]) will be None rather than having a value Returns ---------- - list - list of target projections. Each target projections is a dict with the attributes - 'target_type' and 'proj'. If dims is not None, some target projections may be - None instead + list of dict + List of target projections. Each target projections is a dict with the + attributes 'target_type' and 'proj'. If `dims` is not None, some target + projections may be None instead """ if (len(tgts) == 0): return [] @@ -265,7 +311,7 @@ def project_targets(rmat, tvec, cameraMatrix, distCoeffs, tgts, dims=None): # Repackage the projected points with their target type tgt_projs = [] for proj, tgt, in zip(projs, tgts): - tgt_projs.append({'target_type': tgt['target_type'], 'proj' : proj.tolist()}) + tgt_projs.append({'target_type': tgt['target_type'], 'proj': proj}) # If dims is given, filter projections outside the image if dims is not None: @@ -273,7 +319,7 @@ def project_targets(rmat, tvec, cameraMatrix, distCoeffs, tgts, dims=None): for tgt_proj in tgt_projs: pt = tgt_proj['proj'] # If the point is within the image bounds given, add it to the temp list - if (0.0 <= pt[0] <= dims[1] - 1) and (0.0 <= pt[1] <= dims[0]): + if (0.0 <= pt[0] <= dims[1] - 1) and (0.0 <= pt[1] <= dims[0] - 1): tgt_projs_temp.append(tgt_proj) else: tgt_projs_temp.append(None) @@ -283,46 +329,44 @@ def project_targets(rmat, tvec, cameraMatrix, distCoeffs, tgts, dims=None): def get_occlusions_targets(rmat, tvec, tgts, vis_checker): - """Wrapper around visibility.py methods + """Wrapper around :mod:`~upsp.cam_cal_utils.visibility` methods Parameters ---------- - rmat : np.ndarray (3, 3), float + rmat : np.ndarray, shape (3, 3), float Rotation matrix from camera to object - tvec : np.ndarray (3, 1) or (3,), float + tvec : np.ndarray (3, 1), float Translation vector from camera to object - tgts : list of targets + tgts : list of dict Each target is a dict with (at a minimum) a 'tvec', and 'norm' attribute. The 'tvec' attribute gives the target's location and the 'norm' attribute gives the target's normal vector - vis_checker : VisibilityChecker from upsp/python/upsp/cam_cal_utils/visibility.py - VisibilityChecker object with the relevant BVH and oblique viewing angle + vis_checker : ~upsp.cam_cal_utils.visibility.VisibilityChecker + Visibility checker object with the relevant BVH and oblique viewing angle Returns ---------- - A list of (boolean, occlusion_pt) - The first element of each tuple is a boolean. This boolean denotes if there - was an occlusion or not - The second element of each tuple is the point that the occlusion happens. If - there was no occlusion (the boolean from the first element would be - False), the value [0, 0, 0] is returned. + occlusions : list of tuple + The first element of each tuple is a boolean. This boolean denotes if there was + an occlusion or not. The second element of each tuple is the point that the + occlusion happens. If there was no occlusion (the boolean from the first element + would be False), the value [0, 0, 0] is returned. """ # Get the position and orientation of the camera in the tgts frame rmat_model2camera, tvec_model2camera = invTransform(rmat, tvec) - tvec_model2camera = tvec_model2camera.ravel() # Package the tvecs and normals of the targets tvecs = [] norms = [] for tgt in tgts: - tvecs.append(np.array(tgt['tvec'])) - norms.append(np.array(tgt['norm'])) - + tvecs.append(tgt['tvec']) + norms.append(tgt['norm']) + occlusions = [] for i, node_data in enumerate(zip(tvecs, norms)): # Unpackage the data node, normal = node_data - + # Check for occlusion # Get the direction of the ray @@ -334,29 +378,30 @@ def get_occlusions_targets(rmat, tvec, tgts, vis_checker): # If it is occluded, mark it occluded and move on to the next node occlusions.append(vis_checker.does_intersect(origin, direction, return_pos=True)) - + return occlusions def get_visible_targets(rmat, tvec, tgts, vis_checker): - """Wrapper around visibility.py's is_visible + """Wrapper around :meth:`~upsp.cam_cal_utils.visibility.VisibilityChecker.is_visible` Parameters ---------- - rmat : np.ndarray (3, 3), float + rmat : np.ndarray, shape (3, 3), float Rotation matrix from camera to object - tvec : np.ndarray (3, 1) or (3,), float + tvec : np.ndarray (3, 1), float Translation vector from camera to object - tgts : list of targets + tgts : list of dict Each target is a dict with (at a minimum) a 'tvec', and 'norm' attribute. The 'tvec' attribute gives the target's location and the 'norm' attribute gives the - target's normal vector. Returned targets will have the same attributes as the - input targets - vis_checker : VisibilityChecker from upsp/python/upsp/cam_cal_utils/visibility.py - VisibilityChecker object with the relevant BVH and oblique viewing angle + target's normal vector. Both are np.ndarrays (n, 3) of floats. Returned targets + will have the same additionalk attributes as the input targets + vis_checker : ~upsp.cam_cal_utils.visibility.VisibilityChecker + Visibility checker object with the relevant BVH and oblique viewing angle Returns ---------- + list of dict List of visible targets. Returned targets are references to input targets. Returned list is the subset of the input targets that are visible to the camera """ @@ -370,6 +415,8 @@ def get_visible_targets(rmat, tvec, tgts, vis_checker): for tgt in tgts: tvecs.append(np.array(tgt['tvec'])) norms.append(np.array(tgt['norm'])) + tvecs = np.squeeze(np.array(tvecs), 2) + norms = np.squeeze(np.array(norms), 2) # Get the visible targets and return visible_indices = vis_checker.is_visible(tvec_model2camera, tvecs, norms) @@ -382,28 +429,28 @@ def reprojection_error(rmat, tvec, cameraMatrix, distCoeffs, tgts, img_targets): Parameters ---------- - rmat : np.ndarray (3, 3), float + rmat : np.ndarray, shape (3, 3), float Rotation matrix from camera to object - tvec : np.ndarray (3, 1) or (3,), float + tvec : np.ndarray, shape (3, 1), float Translation vector from camera to object - cameraMatrix : np.ndarray (3x3), float + cameraMatrix : np.ndarray, shape (3, 3), float The (openCV formatted) camera matrix for the camera - distCoeffs : np.ndarray (5x1) or (5,), float + distCoeffs : np.ndarray, shape (1, 5), float The (openCV formatted) distortion coefficients for the camera - tgts : list of targets + tgts : list of dict Each target is a dict with (at a minimum) 'tvec', 'norm', and 'target_type' attributes. The 'tvec' attribute gives the target's location and the 'norm' attribute gives the target's normal vector. 'target_type' is a string denoting the type of target (most commonly 'dot' or 'kulite') - img_targets : list of img_targets - Each img_target is a dict with (at a minimum) a 'center' attribute. The 'center' - attribute gives the pixel position of the target's center in the image. - img_targets[i] is associated with tgts[i] - + img_targets : list of dict + Each ``img_target`` is a dict with (at a minimum) a 'center' attribute. The + 'center' attribute gives the pixel position of the target's center in the image. + ``img_targets[i]`` is associated with ``tgts[i]`` + Returns ---------- - tuple (float, float) - rms distance and the maximum distance + rms, max_dist : float + RMS distance and maximum distance """ if len(tgts) == 0: @@ -423,10 +470,10 @@ def reprojection_error(rmat, tvec, cameraMatrix, distCoeffs, tgts, img_targets): # Check if this is the largest distance thus far max_dist = max(dist, max_dist) - + # Add the square of the distance to the sum rms += dist**2 - + # Divide by the number of targets to get the mean of the squares rms /= len(tgts) @@ -434,114 +481,3 @@ def reprojection_error(rmat, tvec, cameraMatrix, distCoeffs, tgts, img_targets): rms = np.sqrt(rms) return rms, max_dist - - -#--------------------------------------------------------------------------------------- -# Wind Tunnel Specific Function - -def tunnel_transform(ALPHA, BETA, PHI, STRUTZ, tvec__cor_tgts__tgts_frame): - """Calculates the transformation from the tunnel coordinate frame to the tgts frame - - Note: This is not necessarily to the tunnel origin, just some fixed point in the - tunnel (fixed within a tunnel test). If STRUTZ = STRUTZ_abs then it will be the - tunnel origin - - Parameters - ---------- - alpha : float - Tunnel alpha in degrees - beta : float - Tunnel beta in degrees - phi : float - Tunnel phi in degrees - tvec__tgts_trot__tgts_frame : np.ndarray (3, 1) or (3,), float - Translation vector from the tunnel center of rotation to the tgts frame - in the tgts frame. The tgts frame is a fixed distance from the tunnel point of - rotation from the tgts frame's point of view, that translation vector is always - along the x axis - - Returns - ---------- - tuple of (np.ndarray (3, 3) float, np.ndarray (3, 1) float) - Rotation matrix and translation vector from tgts frame to tunnel frame - """ - - # Get the component rotation matrices - - # UPWT Tunnel Coordinates are RHS Aircraft Coordinates Pitched 180 degrees - # See UPWT AIAA Coordinate Systems Training Manual For Details - # (uPSP Teams > General > 4.2 Calibration > - # Stereo Calibration > AIAA Coordinate Systems Training.pdf) - - # Positive Alpha is Positive Pitch - pitch = rot(-ALPHA, 'y') - - # Positive Beta is Negative Yaw - yaw = rot(-BETA, 'z') - - # Positive Phi is Positive Roll - roll = rot(PHI, 'x') - - # Combine into one rotation matrix - # Matrix Multiplication Order is [P][Y][R] - rotation_matrix = np.matmul(pitch, np.matmul(yaw, roll)) - - # We want the transformation from tunnel to tgts, so get the inverse - rotation_matrix = np.linalg.inv(rotation_matrix) - - # tvec__tgts_tunnel__tunnel_frame is the translation vector from the tunnel - # frame to the tgts frame, in the tunnel frame - # I.e. Since the knuckle sleeve rotates relative to the tunnel, the model - # translation vector somewhere in the cone of allowable rotation. This - # calculates that translation. Also, the strutz can move up and down in - # the z axis, so that needs to be taken into account as well - tvec__knuckle_tgts = np.matmul(rotation_matrix, tvec__cor_tgts__tgts_frame) - tvec__tunnel_tgts__tunnel_frame = tvec__knuckle_tgts + np.array([0, 0, STRUTZ]) - - return rotation_matrix, tvec__tunnel_tgts__tunnel_frame - - -# TODO: This refers specifically to a model on the sting. We will need a function for -# a floor mounted model, and ideally something in the test_config file to specify -def tf_camera_tgts_thru_tunnel(camera_cal, wtd, test_config): - """Returns the transformation from the camera to the model (tgts frame) - - Parameters - ---------- - camera_cal : list - camera calibration in the form: - [cameraMatrix, distCoeffs, rmat__camera_tunnel, tvec__camera_tunnel] - wtd : dict - wind tunnel data as a dict with (at a minimum) the keys 'ALPHA', 'BETA', 'PHI', - and 'STRUTZ'. ALPHA, BETA, and PHI are tunnel angles in degrees. STRUTZ is the - offset of the tunnel center of rotation for the z axis in inches - test_config : dict - test configuration data as a dict with (at a minimum) the key - 'tunnel-cor_to_tgts_tvec' representing the translation vector from the tunnel - center of rotation to the model frame - - Returns - ---------- - tuple (np.ndarray (3, 3) float, np.ndarray (3, 1) float) - First element is the rotation matrix, and second element is the translation - vector. - """ - - # Turn the wind tunnel data into the transformation from tunnel to targets - wtd_transform = tunnel_transform(wtd['ALPHA'], wtd['BETA'], wtd['PHI'], - wtd['STRUTZ'], test_config['tunnel-cor_to_tgts_tvec']) - rmat_tunnel_tgts, tvec_tunnel_tgts = wtd_transform - - # Transformation from tgts frame to tunnel frame - rmat_tgts_tunnel = np.linalg.inv(rmat_tunnel_tgts) - tvec_tgts_tunnel = -np.matmul(rmat_tgts_tunnel, tvec_tunnel_tgts) - - # Decompose the camera calibration into its parts - cameraMatrix, distCoeffs, rmat__camera_tunnel, tvec__camera_tunnel = camera_cal - - # Combine the transformations to get the transformation from `camera to tgts frame - rmat__camera_tgts = np.matmul(rmat__camera_tunnel, np.linalg.inv(rmat_tgts_tunnel)) - tvec__camera_tgts = tvec__camera_tunnel + np.matmul(rmat__camera_tunnel, - tvec_tunnel_tgts) - - return rmat__camera_tgts, tvec__camera_tgts diff --git a/python/upsp/cam_cal_utils/target_bumping.py b/python/upsp/cam_cal_utils/target_bumping.py index 2ab5b97..64b11f6 100644 --- a/python/upsp/cam_cal_utils/target_bumping.py +++ b/python/upsp/cam_cal_utils/target_bumping.py @@ -1,17 +1,45 @@ -import visibility -import parsers import os import numpy as np import csv import copy +from upsp.cam_cal_utils import parsers + np.set_printoptions(suppress=True) debug_print_bumping = True debug_bump_distance = True +max_bump_iterations = 10 # debug to stop infinite loops + -# Helper function that gets occlusion for target bumping def get_bumping_occlusion(tgt, vis_checker): + """Helper function to :func:`tgts_get_internals` that finds the point on the model + surface that intersects the `tgt` normal vector if the target is occluded by the + model + + Parameters + ---------- + tgt : dict + A dict and has, at a minimum, the keys 'tvec', 'target_type', 'norm'. 'tvec' has + a :class:`numpy.ndarray` (3, 1) representing the position of the target relative + to the model origin for its associated value. 'norm' has a + :class:`numpy.ndarray` (3, 1) representing the normal vector of the target + relative to the model origin for its associated value. 'target_type' has a + string representing the type of target (most commonly 'dot') for its associated + value. + vis_checker : ~upsp.cam_cal_utils.visibility.VisibilityChecker + BVH to check for visibilty of nodes + + Returns + ------- + occlusion : bool + Whether or not there is an occlusion. True means there is, False means there is + not. + location : np.ndarray, shape (3,) + Location where the `tgt` normal vector intersects the model surface + dist : float + Distance from the target position to the point of intersection + """ # Set the epsilon (occlusion check bumps) distance to 0 # This way any occlusion will be returned eps = vis_checker.epsilon @@ -24,7 +52,7 @@ def get_bumping_occlusion(tgt, vis_checker): # Check for occlusions of this target along its normal occlusion = vis_checker.does_intersect(tgt_tvec, tgt_norm, return_pos=True) - + # Reset vis_checker.epsilon to the original value vis_checker.epsilon = eps @@ -34,13 +62,35 @@ def get_bumping_occlusion(tgt, vis_checker): dist = np.linalg.norm(tgt_tvec - occlusion[1]) return (True, occlusion[1], dist) - + else: return (False, np.array([0, 0, 0]), -1) # Helper function to check if the occlusion found is outside the tolerance -def is_real_occlusion(bumping_occlusion, tgts_tol, bvh_tol): +def is_real_occlusion(bumping_occlusion, tgts_tol, grid_tol): + """Helper function to :func:`tgts_get_internals` that determines if an occlusion is + due to real geometry, or is a numerical/processing error in generating the tgts file + + Parameters + ---------- + bumping_occlusion : tuple + Return value of :func:`get_bumping_occlusion` for the target associated with the + function call + vis_checker : ~upsp.cam_cal_utils.visibility.VisibilityChecker + BVH to check for visibilty of nodes + tgts_tol : float, optional + Tolerance of the tgts file. Used to determine if target internal-ness is real, + or if it is an artifact due to the non-water tightness of the model grid + grid_tol : float, optional + Tolerance of the grid file. Used to determine if target internal-ness is real, + or if it is an artifact due to the non-water tightness of the model grid + + Returns + ------- + bool + True if the occlusion is due to real geometry + """ # The x, y, and z of the target was rounded to 1e-4 by DOTS # So the max distance a target could be rounded was 1e-4 in all 3 axes # This is our tolerance @@ -48,7 +98,7 @@ def is_real_occlusion(bumping_occlusion, tgts_tol, bvh_tol): # Tim Sanstrom's bvh seems to have a tolerance of 1e-3 built into it # The code is confusing - intersect_tol = np.sqrt(3) * bvh_tol + intersect_tol = np.sqrt(3) * grid_tol # Calculate the total tolerance tol = np.linalg.norm([dots_tol, intersect_tol]) @@ -58,25 +108,78 @@ def is_real_occlusion(bumping_occlusion, tgts_tol, bvh_tol): return (bumping_occlusion[2] >= tol) -# Helper function that returns all internal targets. -# Returns a list of internal targets. Each element in the list is a tuple -# The first item in the tuple is the target name -# The second element is a boolean to denote if it is a real occlusion or just some -# accidental occlusion due to being differentiably inside the model grid -def tgts_get_internals(tgts, vis_checker, tgts_tol=1e-4, bvh_tol=1e-3): +def tgts_get_internals(tgts, vis_checker, tgts_tol=1e-4, grid_tol=1e-3): + """Helper function to :func:`ltgt_bump_internals` that returns all internal targets + + Parameters + ----------- + tgts : list of dict + Each target is a dict and has, at a minimum, the keys 'tvec', 'target_type', + 'norm'. 'tvec' has a :class:`numpy.ndarray` (3, 1) representing the position of + the target relative to the model origin for its associated value. 'norm' has a + :class:`np.ndarray` (3, 1) representing the normal vector of the target relative + to the model origin for its associated value. 'target_type' has a string + representing the type of target (most commonly 'dot') for its associated value. + vis_checker : ~upsp.cam_cal_utils.visibility.VisibilityChecker + BVH to check for visibilty of nodes + tgts_tol : float, optional + Tolerance of the tgts file. Used to determine if target internal-ness is real, + or if it is an artifact due to the non-water tightness of the model grid + grid_tol : float, optional + Tolerance of the grid file. Used to determine if target internal-ness is real, + or if it is an artifact due to the non-water tightness of the model grid + + Returns + ------- + list of tuple + Each tuple is an occluded targets. The first item in the tuple is the target + name. The second element is a boolean to denote if it is a real occlusion (like + another part of the model is blocking the target so it should not be bumped) or + just some accidental occlusion due to being differentiably inside the model grid + """ internals = [] for tgt in tgts: bumping_occlusion = get_bumping_occlusion(tgt, vis_checker) if bumping_occlusion[0]: internals.append((tgt['name'], bumping_occlusion[2], - is_real_occlusion(bumping_occlusion, tgts_tol, bvh_tol))) + is_real_occlusion(bumping_occlusion, tgts_tol, grid_tol))) return internals -# Check if any targets are differntiably inside the model. If they are, bump them to be -# differentiably outside the model -def tgt_bump_internals(tgts, vis_checker, bump_eps=1e-5, tgts_tol=1e-1, bvh_tol=1e-3): +def tgt_bump_internals(tgts, vis_checker, bump_eps=1e-5, tgts_tol=1e-1, grid_tol=1e-3): + """Bumps all internal targets along their normal to be slightly external. Slightly + external defined by `bump_eps` + + Parameters + ---------- + tgts : list of dict + Each target is a dict and has, at a minimum, the keys 'tvec', 'target_type', + 'norm'. 'tvec' has a :class:`numpy.ndarray` (3, 1) representing the position of + the target relative to the model origin for its associated value. 'norm' has a + :class:`np.ndarray` (3, 1) representing the normal vector of the target relative + to the model origin for its associated value. 'target_type' has a string + representing the type of target (most commonly 'dot') for its associated value. + vis_checker : ~upsp.cam_cal_utils.visibility.VisibilityChecker + BVH to check for visibilty of nodes + bump_eps : float, optional + Distance to bump the targets outside the model. Should be small so targets are + just barely external to the model + tgts_tol : float, optional + Tolerance of the tgts file. Used to determine if target internal-ness is real, + or if it is an artifact due to the non-water tightness of the model grid + grid_tol : float, optional + Tolerance of the grid file. Used to determine if target internal-ness is real, + or if it is an artifact due to the non-water tightness of the model grid + + Returns + ------- + tgts_bumped : list of dict + Copy of `tgts` with some target positions bumped along their normal to be + slightly external. + was_bumped : bool + Denotes if any targets were bumped + """ # Create a new list to store the bumped targets tgts_bumped = [] @@ -89,14 +192,14 @@ def tgt_bump_internals(tgts, vis_checker, bump_eps=1e-5, tgts_tol=1e-1, bvh_tol= # If there was an occlusion, check the status if bumping_occlusion[0]: - + if debug_bump_distance: dist = np.linalg.norm(np.array(bumping_occlusion[1]) - np.array(tgt['tvec'])) print('\tInternal Target:', tgt['name'], 'Dist:', dist) # If the occlusion is artificial (differentiably inside the mode), bump the # target to be just outside the model - if not is_real_occlusion(bumping_occlusion, tgts_tol, bvh_tol): + if not is_real_occlusion(bumping_occlusion, tgts_tol, grid_tol): # Set the was_bumped flag to True was_bumped = True @@ -109,17 +212,17 @@ def tgt_bump_internals(tgts, vis_checker, bump_eps=1e-5, tgts_tol=1e-1, bvh_tol= # Bump the target to the occlusion point plus some small tolerance along the normal bumped_tgt['tvec'] = np.array(bumping_occlusion[1]) + bump_eps * bumped_tgt['norm'] - + # Add the bumped target to the new list of targets tgts_bumped.append(bumped_tgt) - + if debug_print_bumping: print('\tInternal Target:', tgt['name']) print('\t\tOriginal tvec:', tgt['tvec']) print('\t\tNew tvec:', bumped_tgt['tvec']) # If the occlusion is real (truely occluded by some model surface), - # then just add the original target to the new list of targets + # then just add the original target to the new list of targets else: tgts_bumped.append(tgt) @@ -130,38 +233,60 @@ def tgt_bump_internals(tgts, vis_checker, bump_eps=1e-5, tgts_tol=1e-1, bvh_tol= return tgts_bumped, was_bumped -# Assumes all targets are outside the model. Returns a list of bumped targets such that -# each one is differntiably outside the model (by a distance of bump_eps def tgt_bump_externals(tgts, vis_checker, bump_eps=1e-5): + """Bumps very external targets to be slightly external. Slightly external defined by + `bump_eps` + + Parameters + ---------- + tgts : list of dict + Each target is a dict and has, at a minimum, the keys 'tvec', 'target_type', + 'norm'. 'tvec' has a :class:`numpy.ndarray` (3, 1) representing the position of + the target relative to the model origin for its associated value. 'norm' has a + :class:`np.ndarray` (3, 1) representing the normal vector of the target relative + to the model origin for its associated value. 'target_type' has a string + representing the type of target (most commonly 'dot') for its associated value. + vis_checker : ~upsp.cam_cal_utils.visibility.VisibilityChecker + BVH to check for visibilty of nodes + bump_eps : float, optional + Distance to bump the targets outside the model. Should be small so targets are + just barely external to the model + + Returns + ------- + tgts_bumped : list of dict + Copy of tgts with some target positions bumped along their normal to be slightly + external + """ tgts_bumped = [] for tgt in tgts: # Create a copy of the target, but with an inverted normal - tgt_inv_norm = {'tvec' : np.array(tgt['tvec']) + 100 * bump_eps * np.array(tgt['norm']), - 'norm' : -np.array(tgt['norm'])} - + tgt_inv_norm = {'tvec': np.array(tgt['tvec']) + 100 * bump_eps * np.array(tgt['norm']), + 'norm': -np.array(tgt['norm'])} + # Get the point on the model surface just behind the target bumping_occlusion = get_bumping_occlusion(tgt_inv_norm, vis_checker) - + # Create a copy of the original target bumped_tgt = copy.copy(tgt) - # Normalize the target normal vector + # Normalize the target normal vector bumped_tgt['norm'] = np.array(tgt['norm'], dtype=np.float64) bumped_tgt['norm'] /= np.linalg.norm(tgt['norm']) # Bump the target to the occlusion point plus some small tolerance along the normal bumped_tgt['tvec'] = np.array(bumping_occlusion[1]) + bump_eps * bumped_tgt['norm'] - + # Add the bumped target to the new list of targets tgts_bumped.append(bumped_tgt) - + if debug_print_bumping: print('External Bump:', tgt['name']) print('\tOriginal tvec:', tgt['tvec']) print('\tOcclusion Site:', bumping_occlusion[1]) print('\tNew tvec:', bumped_tgt['tvec']) - + if debug_bump_distance: dist = np.linalg.norm(np.array(bumped_tgt['tvec']) - np.array(tgt['tvec'])) print('\tExternal Target:', tgt['name'], 'Dist:', dist) @@ -169,46 +294,118 @@ def tgt_bump_externals(tgts, vis_checker, bump_eps=1e-5): return tgts_bumped -# Bump the internal targets, then bump all targets to be just at the surface of the model -def tgts_bumper(tgts, vis_checker, bump_eps=1e-5, tgts_tol=1e-1, bvh_tol=1e-3): +def tgts_bumper(tgts, vis_checker, bump_eps=1e-5, tgts_tol=1e-1, grid_tol=1e-3): + """Bumps all internal targets along their normal to be slightly external. Bumps any + targets very external to be slightly external + + Parameters + ---------- + tgts : list of dict + Each target is a dict and has, at a minimum, the keys 'tvec', 'target_type', + 'norm'. 'tvec' has a :class:`numpy.ndarray` (3, 1) representing the position of + the target relative to the model origin for its associated value. 'norm' has a + :class:`np.ndarray` (3, 1) representing the normal vector of the target relative + to the model origin for its associated value. 'target_type' has a string + representing the type of target (most commonly 'dot') for its associated value. + vis_checker : ~upsp.cam_cal_utils.visibility.VisibilityChecker + BVH to check for visibilty of nodes + bump_eps : float, optional + Distance to bump the targets outside the model. Should be small so targets are + just barely external to the model + tgts_tol : float, optional + Tolerance of the tgts file. Used to determine if target internal-ness is real, + or if it is an artifact due to the non-water tightness of the model grid + grid_tol : float, optional + Tolerance of the grid file. Used to determine if target internal-ness is real, + or if it is an artifact due to the non-water tightness of the model grid + + Returns + ------- + bumped_external_targets : list of dict + Copy of tgts input with the 'tvec' values modified so that each tgt is outside + the grid of `vis_checker` + """ # Bump the internal targets until nothing needs to be bumped # Python doesn't have a Do-While Loop so I have to run it once then throw it into # the loop was_bumped = True - internal_bump_count = 1 + internal_bump_count = 0 bumped_internal_targets = tgts if debug_print_bumping: print('Internal Bump Iteration', internal_bump_count) while was_bumped: + if internal_bump_count > max_bump_iterations: + print('Number of internal bumps exceeds max allowed. Skiping to external bump.' + + 'Targets are still be internal. Please modify bump_eps, tgts_tol, or grid_tol') + break + bumped_internal_targets, was_bumped = tgt_bump_internals( - bumped_internal_targets, vis_checker, bump_eps, tgts_tol, bvh_tol + bumped_internal_targets, vis_checker, bump_eps, tgts_tol, grid_tol ) internal_bump_count += 1 if was_bumped and debug_print_bumping: print('Internal Bump Iteration', internal_bump_count) - # Uncomment this when tgt_bump_externals is working properly - # Bump the external targets - bumped_targets = tgt_bump_externals(bumped_internal_targets, vis_checker, bump_eps) - - return bumped_targets - - -# Takes a targets file and generates a 'bumped' version where all targets are moved -# a differential distance along their normal such that they are 1e-5 inches outside -# the model (+/- some for numerical tolerances) -def tgts_file_bumper(tgts_file_path, vis_checker, bump_eps=1e-5, tgts_tol=1e-1, bvh_tol=1e-3): + # Bump the external targets and ensure all targets are external + are_internal = True + external_bump_count = 0 + while are_internal: + if external_bump_count > max_bump_iterations: + print('Number of external bumps exceeds max allowed. Writing bumped file' + + 'anyway. Targets are still be internal. Please modify bump_eps, ' + + 'tgts_tol, or grid_tol') + break + + # Bump the external targets + bumped_external_targets = tgt_bump_externals(bumped_internal_targets, vis_checker, bump_eps) + bumped_internal_targets = bumped_external_targets + + _, was_bumped = tgt_bump_internals( + bumped_internal_targets, vis_checker, bump_eps, tgts_tol, grid_tol) + + # If there are no internal targets, set are_interal to False + if not was_bumped: + are_internal = False + + external_bump_count += 1 + + return bumped_external_targets + + +def tgts_file_bumper(tgts_file_path, vis_checker, bump_eps=1e-5, tgts_tol=1e-1, grid_tol=1e-3): + """Creates a tgts file with all internal targets bumped along their normal to be + slightly external and very external targets to be slightly external + + Creates a new tgts file in the same directory as tgts_file_path with the same name, + but with the suffix '_bumped' attached to the filename + + Parameters + ---------- + tgts_file_path : path-like + Path to the tgts file + vis_checker : ~upsp.cam_cal_utils.visibility.VisibilityChecker + BVH to check for visibilty of nodes + bump_eps : float, optional + Distance to bump the targets outside the model. Should be small so targets are + just barely external to the model + tgts_tol : float, optional + Tolerance of the tgts file. Used to determine if target internal-ness is real, + or if it is an artifact due to the non-water tightness of the model grid + grid_tol : float, optional + Tolerance of the grid file. Used to determine if target internal-ness is real, + or if it is an artifact due to the non-water tightness of the model grid + """ # Read in the targets - tgts = parsers.read_tgts(tgts_file_path, read_all_fields=True, read_pK=True) - + tgts = parsers.read_tgts(tgts_file_path) + # Bump the targets - bumped_targets = tgts_bumper(tgts, vis_checker, bump_eps, tgts_tol, bvh_tol) - + bumped_targets = tgts_bumper(tgts, vis_checker, bump_eps, tgts_tol, grid_tol) + # Get the filepath of the new tgts file filename, file_ext = os.path.splitext(os.path.basename(tgts_file_path)) new_tgts_file_path = os.path.join(os.path.dirname(tgts_file_path), filename + '_bumped' + file_ext) - + # Open the new tgts file with open(new_tgts_file_path, 'w') as f_write: csv_writer = csv.writer(f_write, delimiter=' ', quoting=csv.QUOTE_MINIMAL) @@ -218,36 +415,36 @@ def tgts_file_bumper(tgts_file_path, vis_checker, bump_eps=1e-5, tgts_tol=1e-1, # Read the old tgts file csv_reader = csv.reader(f_read, delimiter=' ') is_in_targets_section = False - + # For each line in the old tgts file for row in csv_reader: line = [] for item in row: if (item != ''): line.append(item) - + # Check for when we enter the targets section (denoted with *Targets) # When we do, write all the bumped targets if not is_in_targets_section and (line[0] == '*Targets'): # Write the *Targets header csv_writer.writerow(line) is_in_targets_section = True - + # Write the bumped targets for tgt in bumped_targets: tgt_line = [str(tgt['idx']).ljust(5, ' '), - "{:>14}".format("{:3.9f}".format(tgt['tvec'][0])), - "{:>14}".format("{:3.9f}".format(tgt['tvec'][1])), - "{:>14}".format("{:3.9f}".format(tgt['tvec'][2])), - "{:>14}".format("{:3.9f}".format(tgt['norm'][0])), - "{:>14}".format("{:3.9f}".format(tgt['norm'][1])), - "{:>14}".format("{:3.9f}".format(tgt['norm'][2])) + ' ', + "{:>14}".format("{:3.9f}".format(tgt['tvec'][0][0])), + "{:>14}".format("{:3.9f}".format(tgt['tvec'][1][0])), + "{:>14}".format("{:3.9f}".format(tgt['tvec'][2][0])), + "{:>14}".format("{:3.9f}".format(tgt['norm'][0][0])), + "{:>14}".format("{:3.9f}".format(tgt['norm'][1][0])), + "{:>14}".format("{:3.9f}".format(tgt['norm'][2][0])) + ' ', str(tgt['size']).ljust(6, ' '), str(tgt['zones'][0]).ljust(5, ' '), str(tgt['zones'][1]).ljust(5, ' '), str(tgt['zones'][2]).ljust(5, ' '), str(tgt['name']).ljust(6, ' ')] - + for item in tgt_line: f_write.write(item) f_write.write('\n') @@ -263,4 +460,3 @@ def tgts_file_bumper(tgts_file_path, vis_checker, bump_eps=1e-5, tgts_tol=1e-1, if not is_in_targets_section: csv_writer.writerow(row) return - diff --git a/python/upsp/cam_cal_utils/visibility.py b/python/upsp/cam_cal_utils/visibility.py index 9458856..f496d8b 100644 --- a/python/upsp/cam_cal_utils/visibility.py +++ b/python/upsp/cam_cal_utils/visibility.py @@ -1,117 +1,63 @@ import logging import os -import sys import numpy as np -import time + +import upsp.processing.p3d_utilities as p3d +import upsp.processing.p3d_conversions as p2g +import upsp.raycast log = logging.getLogger(__name__) -THIS_DIRECTORY = os.path.dirname(os.path.abspath(__file__)) - -python_dir = os.path.dirname(os.path.dirname(THIS_DIRECTORY)) - -sys.path.append(THIS_DIRECTORY) -sys.path.append(python_dir) - -# Normally, 'upsp' will throw a warning indicating -# that the pybind11 modules are not available when -# importing from the source tree. -import warnings # noqa -warnings.simplefilter("ignore") -import upsp.processing.p3d_utilities as p3d # noqa -import upsp.processing.p3d_conversions as p2g # noqa -import upsp # noqa - -warnings.resetwarnings() - -# todo-mshawlec: clean up how we handle pybind11 modules -# between two uses cases of a) dev tree, vs. b) installed sw. -# -# If we're running this module out of a development -# tree, then we have to import pybind11 modules from the -# project build/ directory. If we hit a ModuleNotFoundError, -# then we're likely running out of an install, where the -# pybind11 modules are installed right in the python/upsp package -# folder (and should have been imported automatically above during -# the 'import upsp' call). -try: - upsp_dir = os.path.dirname(python_dir) - UPSP_PYBIND11_BUILD_PATH = os.path.abspath(os.path.join(upsp_dir, "build")) - sys.path.append(UPSP_PYBIND11_BUILD_PATH) - # Unorthodox "patch" of module into the upsp package namespace - import raycast # noqa - upsp.raycast = raycast -except ModuleNotFoundError: - pass - - -class VisibilityChecker(): + +class VisibilityChecker: """Visibilty object class for occlusion and viewing angle checking - + This class creates an object for checking visibility of a list of points with - associated normals. The intended use is for the external calibration to - check target visibility, and to check visibility of the the model mesh nodes - for node-pixel mapping. - - For the visibility check, it is assumes that the entire model is within the field - of view of the camera. This simplifies the check for 2 reasons - 1) We don't need to check if a node is within the field of view - 2) We don't need to check if an intersection is behind the camera - - A ray is drawn from a given point to the camera. If that ray intersects a - node, it is deemed not visible. However, if there is a node behind the - camera that intersects with the ray, this will still be seen as an - intersection and flagged as not visible. Even though the intersection - behind the camera does not truely occlude the given point. - - Methods - ------- - init - creates object and populates scene - load_mesh - reads grid file to populate scene - package_primitves - turns p3d outputs into primitives for scene - is_back_facing - inputs 'angle', vector from camera to node, and node normal - outputs True if node is viewing angle is less than the input angle - does_intersect - inputs a ray via origin and direction. Returns True if - there is an intersection between that ray and the primitives of the - grid of the grid path file - is_visible - inputs nodes and their normals (optionally a viewing angle) + associated normals. The intended use is for the external calibration to check target + visibility, and to check visibility of the the model mesh nodes for node-pixel + mapping. + + For the visibility check, it is assumes that the entire model is within the field of + view of the camera. This simplifies the check for 2 reasons: + + 1. We don't need to check if a node is within the field of view + 2. We don't need to check if an intersection is behind the camera + + - A ray is drawn from a given point to the camera. If that ray intersects a + node, it is deemed not visible. However, if there is a node behind the camera + that intersects with the ray, this will still be seen as an intersection and + flagged as not visible. Even though the intersection behind the camera does + not truly occlude the given point. + + Parameters + ---------- + grid_path : str + Filepath to the grid file + oblique_angle : float, optional + Maximum allowable oblique viewing angle. Viewing angle for a node pointed + directly at the camera and in the center of the field of view of the camera + is 0 degrees + epsilon : float, optional + Intersection tolerance. For the ray tracing intersection check, the origin of + the ray is offset from the model by a distance of epsilon in the direction of + the normal + debug : bool, optional + Debug parameter to speed up grid loading. It is suggested to use this debug for + all development, and to leave it False for all non-development case. If True, + looks for ``'{{filename}}_primitives.npy'`` in '.cache', where ``filename`` + is the filename from `grid_path`. If it finds it, loads from the ``.npy`` file + rather than reading and processing the grid file. If it doesn't find it, it will + read and process the grid file as normal, and save the numpy array as + ``'{{filename}}_primitives.npy'`` in '.cache' + debug_nogrid : bool, optional + Debug parameter to speed up computation. It is suggested to use this debug for + development if occlusions are not needed for development work. There is no + reason to leave this parameter for any non-development work. If True, instead of + loading a real grid file, it loads a fake one with a single, extremely small + primitive (effectively having 'no grid'). This makes BVH operations much faster. """ def __init__(self, grid_path, oblique_angle=70, epsilon=1e-4, debug=False, debug_nogrid=False): - """Initializes the visibility check object - - Parameters - ---------- - grid_path : str - Filepath to the grid file - oblique_angle : float, optional default=70.0 - Maximum allowable oblique viewing angle. Viewing angle for a node pointed - directly at the camera and in the center of the field of view of the camera - is 0 degrees - epsilon : float, optional default=1e-4) - Intersection tolerance. For the ray tracing intersection check, the origin - of the ray is offset from the model by a distance of epsilon in the - direction of the normal - debug : boolean, optional default=False - Debug parameter to speed up grid loading. It is suggested to use this debug - for all development, and to leave it False for all non-development case. - If True, looks for filename + '_primitives.npy' in '.cache', where filename - is the filename from grid_path. If it finds it, loads from the .npy file - rather than reading and processing the grid file. If it doesn't fine it, - it will read and process the grid file as normal, and save the numpy - array as filename + '_primitives.npy' in '.cache' - debug_nogrid : boolean, optional default=False - Debug parameter to speed up computation. It is suggested to use this debug - for development if occlusions are not needed for development work. There - is no reason to leave this parameter for any non-development work - If True, instead of loading a real grid file, it loads a fake one with a - single, extremely small primitive (effectively having 'no grid'). This - is so the BVH operations much faster. - - Returns - ---------- - VisibilityChecker with internal objects based on given parameters - """ - # If debug_nogrid is used, create a fake BVH with one very small primitive if debug_nogrid: primitives = np.array([0.00001, 0.0, 0.0, 0.0, 0.00001, 0.0, 0.0, 0.0, 0.00001]) @@ -148,6 +94,9 @@ def __init__(self, grid_path, oblique_angle=70, epsilon=1e-4, debug=False, debug t = self.load_mesh(grid_path) primitives = self.package_primitives(t) + # Store the grid path used for this object + self.grid_path = grid_path + # Convert primitives to a list (potentially from a np array) primitives = primitives.tolist() @@ -157,7 +106,7 @@ def __init__(self, grid_path, oblique_angle=70, epsilon=1e-4, debug=False, debug # Save the square of the cosine of the oblique angle # This will be used for back face culling self.update_oblique_angle(oblique_angle) - + # Save the BVH to the instance # Simple scene with several triangle primitives # [t0p0x, t0p0y, t0p0z, t0p1x, t0p1y, t0p1z, t0p2x, t0p2y, t0p2z, ...] @@ -168,38 +117,33 @@ def __init__(self, grid_path, oblique_angle=70, epsilon=1e-4, debug=False, debug def update_oblique_angle(self, oblique_angle): """Updates object elements related to the oblique viewing angle - + Parameters - ---------- + ---------- oblique_angle : float Maximum allowable oblique viewing angle. Viewing angle for a node pointed directly at the camera and in the center of the field of view of the camera is 0 degrees - - Returns - ---------- - None """ self.oblique_angle = oblique_angle self.squared_cos_angle = np.cos(np.deg2rad(oblique_angle))**2 def load_mesh(self, grid_path): """Loads the grid file into vertices and indices - + Parameters ---------- - grid_path : str + grid_path : string Filepath to the grid file - + Returns - ---------- - dict - dict with keys "vertices" and "indices". t["vertices"] is a (N, 3) array of - the model vertices. t["indices"] is a (N, 3) array of ints. t["indices"][i] - refers to model face i. The vertices that make up face i are t["vertices"][n] - where n is (3, 1) from t["indices"][i] + ------- + t : dict + Dict with keys "vertices" and "indices". ``t["vertices"]`` is a (N, 3) array + of the model vertices. ``t["indices"]`` is a (N, 3) array of ints where each + row refers to a model face ``i``. The vertices that make up face ``i`` are + ``t["vertices"][n]``, where ``n`` is (3, 1) from ``t["indices"][i]`` """ - grd = p3d.read_p3d_grid(grid_path) t = p2g.p3d_to_gltf_triangles(grd) t["vertices"] = np.array(t["vertices"]) @@ -208,21 +152,20 @@ def load_mesh(self, grid_path): def package_primitives(self, t): """Packages the primitives of the grid file into the BVH format - - Converts the vertices and indices into a list of primitives in the form: + + Converts the vertices and indices into a list of primitives in the form:: + [t0p0x, t0p0y, t0p0z, t0p1x, t0p1y, t0p1z, t0p2x, t0p2y, t0p2z, ...] - + Parameters ---------- t : dict - dict with keys "vertices" and "indices". t["vertices"] is a (N, 3) array of - the model vertices. t["indices"] is a (N, 3) array of ints. t["indices"][i] - refers to model face i. The vertices that make up face i are t["vertices"][n] - where n is (3, 1) from t["indices"][i] - + Dict with keys "vertices" and "indices" from :meth:`load_mesh`. + Returns - ---------- - primitives array + ------- + primitives : np.ndarray + Packaged primitives """ # Get vertex and face information @@ -256,77 +199,77 @@ def package_primitives(self, t): def unit_vector(self, vector): """Returns the unit vector of the vector - - Helper function for angle_between + + Helper function for :meth:`angle_between` Parameters ---------- - vector : np.ndarray (3,) or (3, 1), float - vector whose unit vector is desired + vector : np.ndarray, shape (n, 3) float + Array of vectors Returns - ---------- - unit vector of input vector - + ------- + unit_vector : np.ndarray + Unit vector(s) corresponding to `vector` + See Also - -------- + -------- angle_between : Returns the angle in radians between vectors 'v1' and 'v2' """ - return vector / np.linalg.norm(vector) + return np.divide(vector, np.expand_dims(np.linalg.norm(vector, axis=1), 1)) def angle_between(self, v1, v2): - """Returns the angle in radians between vectors 'v1' and 'v2' - + """Returns the angle in radians between vectors `v1` and `v2` + Parameters ---------- - v1 : np.ndarray (3,) or (3, 1), float + v1 : np.ndarray, shape (n, 3), float Vector 1 - v2 : np.ndarray (3,) or (3, 1), float + v2 : np.ndarray, shape (n, 3), float Vector 2 Returns - ---------- - float - Angle (in radians) between v1 and v2 - + ------- + angle : float + Angle (in radians) between `v1` and `v2` + See Also - -------- + -------- unit_vector : Returns the unit vector of the vector is_back_facing : This is the 'slow' version of the back face culling check """ v1_u = self.unit_vector(v1) v2_u = self.unit_vector(v2) - return np.arccos(np.clip(np.dot(v1_u, v2_u), -1.0, 1.0)) + return np.rad2deg(np.arccos(np.clip(np.sum(np.multiply(v1_u, v2_u), axis=1), -1.0, 1.0))) def is_back_facing(self, t, n): """This is the 'slow' version of the back face culling check - + This function is mostly for legacy/regression purposes - TODO: This does not filter points on the horizon of the model (~90 degrees - oblique viewing angle). It does seem to filter those above 70 degrees (might - not though, very little testing was done). It might have something to do - with the clip in angle between - + TODO: This does not filter points on the horizon of the model (~90 degrees + oblique viewing angle). It does seem to filter those above 70 degrees (might not + though, very little testing was done). It might have something to do with the + clip in angle between + Parameters ---------- - t : np.ndarray (3,1) or (3,), float + t : np.ndarray, shape (3,) or (3, n), float Translation vector from camera to node - n : np.ndarray (3,1) or (3,), float + n : np.ndarray, shape (3,) or (3, n), float Normal of the node Returns - ---------- - True if the node is back facing. False if it is not back facing + ------- + back_facing : bool + True if the node is back facing. False if it is not back facing See Also - -------- - angle_between : - Returns the angle in radians between vectors 'v1' and 'v2' - is_back_facing : - This is the 'slow' version of the back face culling check - is_back_facing_fast_vectorized : - Returns array of booleans for which nodes are back facing + -------- + angle_between : Returns the angle in radians between vectors 'v1' and 'v2' + is_back_facing : This is the 'slow' version of the back face culling check + is_back_facing_fast_vectorized : Returns array of booleans for which nodes are + back facing """ if self.angle_between(t, n) > self.oblique_angle: return True @@ -334,53 +277,59 @@ def is_back_facing(self, t, n): return False def is_back_facing_fast(self, t, n): - """Returns True if the node is back facing, otherwise returns False - + r"""Returns True if the node is back facing, otherwise returns False + This re-write is significantly faster than the naive approach, but not as fast - as the vectorized approach. This function is mostly for legacy/regression purposes + as the vectorized approach. This function is mostly for legacy/regression + purposes - Calculations: - 1) The node is back facing if: - angle between vectors < oblique angle + The node is back facing if angle between vectors < oblique angle - 2) Mathematically this is done as: - arccos(dot(t, n) / (||t|| * ||n||) < oblique_angle + Mathematically this is done as: - 3) arccos is an expensive operation. Re-write as follows: - dot(t, n) / (||t|| * ||n||) < cos(oblique_angle) + .. math:: + \arccos\left(\frac{t \cdot n}{\|t\| \|n\|}\right) < \textrm{oblique angle} - 4) Division is more expensive than multiplication - dot(t, n) < cos(oblique_angle) * ||t|| * ||n|| + Since ``arccos`` is an expensive operation, re-write as: - 5) magnitude(a) can be re-written as sqrt(dot(a, a)) - dot(t, n) < cos(oblique_angle) * sqrt(dot(t, t)) * sqrt(dot(n, n)) + .. math:: + \frac{t \cdot n}{\|t\| \|n\|} < \cos(\textrm{oblique angle}) - 6) sqrt is an expensive function, square both sides and re-write as follows: - abs is required on LHS to preserve sign. All elements of RHS are positive - since t and n contain only real numbers - dot(t, n) * abs(dot(t, n)) < cos^2(oblique_angle) * dot(t, t) * dot(n, n) + Since division is more expensive than multiplication, re-write as: - 7) cos^2(oblique_angle) is given as an input to additional speedup + .. math:: + t \cdot n < \|t\| \|n\| \cos(\textrm{oblique angle}) + + Using :math:`\|a\|^2 = a \cdot a`, we get: + + .. math:: + (t \cdot n) (|t \cdot n|) < + (t \cdot t) (n \cdot n) \cos^2(\textrm{oblique angle}) + + Where the absolute value is required on the LHS to preserve sign. All elements + of RHS are positive since ``t`` and ``n`` contain only real numbers + + :math:`\cos^2(\textrm{oblique angle})` is calculated and cached when the oblique + angle changes, giving an additional speedup Parameters ---------- - t : np.ndarray (3,1) or (3,), float + t : np.ndarray, shape (3, 1) or (3,), float Translation vector from camera to node - n : np.ndarray (3,1) or (3,), float + n : np.ndarray, shape (3, 1) or (3,), float Normal of the node Returns - ---------- - Boolean - True if input is back facing (viewing angle > maximum oblique angle) - False if input is not back facing (viewing angle <= maximum oblique angle) - + ------- + back_facing : bool + True if input is back facing (viewing angle > maximum oblique angle). False + if input is not back facing (viewing angle <= maximum oblique angle) + See Also - -------- - is_back_facing : - This is the 'slow' version of the back face culling check - is_back_facing_fast_vectorized : - Returns array of booleans for which nodes are back facing + -------- + is_back_facing : This is the 'slow' version of the back face culling check + is_back_facing_fast_vectorized : Returns array of booleans for which nodes are + back facing """ proj = np.dot(t, n) @@ -397,30 +346,29 @@ def is_back_facing_fast(self, t, n): def is_back_facing_fast_vectorized(self, t, n): """Returns array of booleans for which nodes are back facing - - See is_back_facing_fast for explaination of math. To vectorize dot(a, b) we will - use np.sum(a*b, axis=1) - + + See :meth:`is_back_facing_fast` for explaination of math. To vectorize + :math:`a \\cdot b` we use ``np.sum(a*b, axis=1)`` + Parameters ---------- - t : np.ndarray (N,3) or (3,), float + t : np.ndarray, shape (N, 3), float Array of translation vectors from camera to nodes - n : np.ndarray (N,3) or (3,), float + n : np.ndarray, shape (N, 3), float Array of normal vectors of the nodes Returns - ---------- - Boolean Array - output[i] is True if node[i] is backfacing (viewing angle > maximum oblique - angle). output[i] is False if node[i] is not backfacing + ------- + back_facing : np.ndarray, bool + ``output[i]`` is True if ``node[i]`` is backfacing (viewing angle > maximum + oblique angle). ``output[i]`` is False if ``node[i]`` is not backfacing (viewing angle <= maximum oblique angle). See Also - -------- - is_back_facing : - This is the 'slow' version of the back face culling check - is_back_facing_fast : - Returns True if the node is back facing, otherwise returns False + -------- + is_back_facing : This is the 'slow' version of the back face culling check + is_back_facing_fast : Returns True if the node is back facing, otherwise returns + False """ proj = np.sum(t*n, axis=-1) @@ -428,75 +376,84 @@ def is_back_facing_fast_vectorized(self, t, n): def does_intersect(self, origin, direction, return_pos=False): """Function that determines if a point is occluded by the object mesh - + Creates a ray from origin with given direction. Checks for intersection of ray with the BVH Parameters ---------- - origin : np.ndarray (3,), float - start of ray - direction : np.ndarray (3,), float - direction of ray + origin : np.ndarray, shape (3, 1), float + start of ray + direction : np.ndarray, shape (3, 1), float + direction of ray - Output - ---------- - Boolean + Returns + ------- + result : bool True if node is occluded and False if node is not occluded """ r = upsp.raycast.Ray(*origin, *direction) h = upsp.raycast.Hit() result = self.scene.intersect(r, h) - + # If the position was requested, return it if return_pos: - return (result, np.array(h.pos)) - + return (result, np.expand_dims(h.pos, 1)) + # Otherwise just return the boolean else: return result - def is_visible(self, tvec_model_to_camera, nodes, normals): + def is_visible(self, tvec_model_to_camera, nodes, normals, return_angles=False): """Returns list of nodes that are visible - + Currently only checks for oblique viewing angle and occlusion, assumes all nodes - are within FOV of camera + are within FOV of camera Parameters ---------- - tvec_model_to_camera : np.ndarray (3,1) or (3,), float - translation vector from model to camera - nodes : np.ndarray (N, 3), float - X, Y, Z position of nodes to be checked. nodes[i] is associated with - normals[i] - normals : np.ndarray (N, 3), float - Normal vectors. normals[i] is associated with nodes[i] - + tvec_model_to_camera : np.ndarray, shape (3, 1), float + translation vector from model to camera + nodes : np.ndarray, shape (N, 3), float + X, Y, Z position of nodes to be checked. ``nodes[i]`` is associated with + ``normals[i]`` + normals : np.ndarray, shape (N, 3), float + Normal vectors. ``normals[i]`` is associated with ``nodes[i]`` + return_angles : bool, optional + If True, return angles for each visible node as well. + Returns ---------- + visible : np.ndarray Numpy array of the indices of the nodes that are visible - + angles : np.ndarray + Angles of visible nodes. Only returned when ``return_angles=True`` + See Also ---------- - is_back_facing_fast_vectorized : - Returns array of booleans for which nodes are back facing - does_intersect : - Function that determines if a point is occluded by the object mesh + is_back_facing_fast_vectorized : Returns array of booleans for which nodes are + back facing + does_intersect : Function that determines if a point is occluded by the object + mesh """ - # Get the tvecs from the camera to each node - tvec_model_to_camera = tvec_model_to_camera.ravel() - tvecs = tvec_model_to_camera - nodes - + tvecs = tvec_model_to_camera.T - nodes + # Get the unit_tvecs and unit_normals - tvec_norms = np.linalg.norm(tvecs, axis=1) - unit_tvecs = tvecs / tvec_norms.reshape(-1, 1) + tvec_norms = np.linalg.norm(tvecs, axis=-1) + unit_tvecs = tvecs / np.expand_dims(tvec_norms, 1) normal_norms = np.linalg.norm(normals, axis=1) - unit_normals = normals / normal_norms.reshape(-1, 1) - + unit_normals = normals / np.expand_dims(normal_norms, 1) + # Determine what nodes are back-facing # or nearly back-facing based on oblique_angle - back_facings = self.is_back_facing_fast_vectorized(unit_tvecs, unit_normals) + # If the angles values are not needed, use is_back_facing_fast_vectorized + # If angles are needed, use the slower + if not return_angles: + back_facings = self.is_back_facing_fast_vectorized(unit_tvecs, unit_normals) + else: + angles = self.angle_between(unit_tvecs, unit_normals) + back_facings = np.where(angles > self.oblique_angle, True, False) # Get the offset node positions epsilon_normals = self.epsilon * unit_normals @@ -505,37 +462,112 @@ def is_visible(self, tvec_model_to_camera, nodes, normals): visible = [] for i in range(len(nodes)): # If it is back-facing, move on to the next node - if back_facings[i]: + if back_facings[i]: continue - direction = self.unit_vector(tvecs[i]) - # If there is an occlusion, move on to the next node # origin is the ray origin offset so it is close to, but not on the model # tvecs[i] is the ray direction vector that goes from the camera to node i if self.does_intersect(origins[i], unit_tvecs[i]): continue - + # If the node was not back-facing, and was not occluded, mark it visible visible.append(i) - + + if return_angles: + return np.array(visible), np.array(angles[visible]) + return np.array(visible) def tri_normal(self, face): """Returns the normal of a face with vertices ordered counter-clockwise - + Parameters ---------- - face : np.ndarray (3, 3), float - X, Y, Z positions of the fae vertices. face[i] contains x, y, z of vertex i + face : np.ndarray, shape (3, 3), float + X, Y, Z positions of the face vertices. ``face[i]`` contains x, y, z of + vertex ``i`` Returns ---------- - Normal vector of input node + normal : np.ndarray, shape (3,) + Normal vector of input face """ # https://www.scratchapixel.com/lessons/3d-basic-rendering/ray-tracing-rendering-a-triangle/geometry-of-a-triangle A = face[1] - face[0] # Edge 1 B = face[2] - face[1] # Edge 2 C = np.cross(A, B) - return C + return C.reshape(3, 1) + def get_faces_and_face_normals(self): + """Returns the faces and face normals of all grid nodes + + Returns + ------- + faces : np.ndarray, shape (n, 3, 3) + Each face is a (3, 3) with (i, 3) corresponding to a node, and each node + having an (x, y, z). Faces are ordered counter-clockwise. + face_normals : np.ndarray, shape (n, 3) + The face normal is (x, y, z). ``face_normals[i]`` corresponds to ``face[i]`` + """ + # Get the mesh + t = self.load_mesh(self.grid_path) + + # Get vertex and face information + nverts = int(t["vertices"].size / 3) + nfaces = int(t["indices"].size / 3) + verts = np.reshape(t["vertices"], (nverts, 3)) + + # This is the faces, with the indexes corresponding to the elements of verts + # Note: faces are counter-clockwise ordered so normals are implicit in faces + faces_v_idx = np.reshape(t["indices"], (nfaces, 3)) + + # Get all the faces and the face normals + faces = [] + face_normals = [] + for face_v_idx in faces_v_idx: #[np.random.choice(range(len(faces_v_idx)), 25000)]: + face = np.array([verts[face_v_idx[0]], + verts[face_v_idx[1]], + verts[face_v_idx[2]]]) + + if ((face[0] == face[1]).all() or (face[0] == face[2]).all() + or (face[1] == face[2]).all()): + continue + + faces.append(face) + face_normals.append(self.tri_normal(face)) + + return np.array(faces), np.array(face_normals).reshape(-1, 3) + + def get_tvecs_and_norms(self): + """Returns the tvecs and tvec normals of all grid nodes + + Returns + ------- + tvecs : np.ndarray, shape (n, 3) + Each tvec is corresponds to the (x, y, z) of a node. + norms : np.ndarray, shape (n, 3) + Each norm is an (x, y, z) of the node's normal vector ``norms[i]`` + corresponds to ``tvecs[i]``. The norm is the normal of the first face to + contain the tvec. Each tvec can appear in multiple faces, so the first is + used. In practice, each face that contains the node will have a different + normal vector, but in general the difference is a relatively small angle. + """ + # Get the faces and face normals from the vis_checker + faces, face_normals = self.get_faces_and_face_normals() + + # Calculate a normal for every node. If a node is on several faces, assume the + # normals of all those faces are close enough and pick any one of them + tvecs, norms = [], [] + node_set = set() + for face, normal in zip(faces, face_normals): + for node in face: + if not tuple(node.tolist()) in node_set: + node_set.add(tuple(node.tolist())) + tvecs.append(np.array(node)) + norms.append(np.array(normal)) + + # Save the nodes and normals as np arrays + tvecs = np.array(tvecs) + norms = np.array(norms) + return tvecs, norms diff --git a/python/upsp/cam_cal_utils/visualization.py b/python/upsp/cam_cal_utils/visualization.py index 7242555..86bbabe 100644 --- a/python/upsp/cam_cal_utils/visualization.py +++ b/python/upsp/cam_cal_utils/visualization.py @@ -24,12 +24,18 @@ def plot_coord_sys(rmat, tvec, ax, scale=10, text=None): In the plot, the x axis is blue, the y axis is orange, and the z axis is green - Inputs: - rmat - Rotation Matrix. Columns represent the basis of the coordinate system - tvec - Origin of the coordinate system - ax - matplotlib axis - scale - Size of the vcoordinate system vectors - text - label of the coordinate system + Parameters + ---------- + rmat : array_like, shape (3, 3) + Rotation Matrix. Columns represent the basis of the coordinate system + tvec : array_like, , shape (3,) + Origin of the coordinate system + ax : mpl_toolkits.mplot3d.axes3d.Axes3D + matplotlib axis + scale : int or float, optional + Size of the vcoordinate system vectors + text : str, optional + Label of the coordinate system. If ``None`` (default), no text is added. """ s_rmat = scale * rmat @@ -37,18 +43,18 @@ def plot_coord_sys(rmat, tvec, ax, scale=10, text=None): # TODO: these should be drawn in the order of distance to the virtual camera # Drawing them in this order by default can cause the debug outputs to look incorrect # due to occlusions - ax.plot3D([tvec[0], tvec[0] + s_rmat[0][0]], [tvec[1], tvec[1] + s_rmat[1][0]], [tvec[2], tvec[2] + s_rmat[2][0]], 'b') - ax.plot3D([tvec[0], tvec[0] + s_rmat[0][1]], [tvec[1], tvec[1] + s_rmat[1][1]], [tvec[2], tvec[2] + s_rmat[2][1]], 'orange') - ax.plot3D([tvec[0], tvec[0] + s_rmat[0][2]], [tvec[1], tvec[1] + s_rmat[1][2]], [tvec[2], tvec[2] + s_rmat[2][2]], 'g') + ax.plot3D([tvec[0][0], tvec[0][0] + s_rmat[0][0]], [tvec[1][0], tvec[1][0] + s_rmat[1][0]], [tvec[2][0], tvec[2][0] + s_rmat[2][0]], 'b') + ax.plot3D([tvec[0][0], tvec[0][0] + s_rmat[0][1]], [tvec[1][0], tvec[1][0] + s_rmat[1][1]], [tvec[2][0], tvec[2][0] + s_rmat[2][1]], 'orange') + ax.plot3D([tvec[0][0], tvec[0][0] + s_rmat[0][2]], [tvec[1][0], tvec[1][0] + s_rmat[1][2]], [tvec[2][0], tvec[2][0] + s_rmat[2][2]], 'g') if (text is not None) and debug_text_displays: - ax.text(tvec[0], tvec[1], tvec[2], text) + ax.text(tvec[0][0], tvec[1][0], tvec[2][0], text) def show_image_locations(img, img_locations, fig_name, scale=5, c='w'): """ Displays an image with the given image locations indicated with a marker of the - given color (white by default) + given color (white by default) """ plt.figure(fig_name) @@ -62,11 +68,12 @@ def show_coord_transforms(cs1_rmat, cs1_tvec, cs2_rmat, cs2_tvec, figname=None, texts=[None, None, None], compares=[False, False, False]): """ Shows the transformation from coordinate system 1 (cs1) to coordinate system 2 (cs2) - Additionally, shows the origin coordinate system. Draws a line from cs1 to cs2 - Texts is the label to be shown for the origin, cs1, and cs2 respectively - compares is a list of booleans that determines if a line is drawn from the origin to - cs1 (compares[0]), from cs1 to cs2 (compares[1]), and from cs2 to the origin - (compares[2]) + + Additionally, shows the origin coordinate system. Draws a line from cs1 to cs2 Texts + is the label to be shown for the origin, cs1, and cs2 respectively compares is a + list of booleans that determines if a line is drawn from the origin to cs1 + (compares[0]), from cs1 to cs2 (compares[1]), and from cs2 to the origin + (compares[2]) """ if figname is None: @@ -114,27 +121,30 @@ def show_coord_transforms(cs1_rmat, cs1_tvec, cs2_rmat, cs2_tvec, figname=None, def plot_pts_and_norms(pts_and_norms, ax, scale=5, c='r'): - """ - Helper function to plot points with normals - - Input: - pts_and_norms - list of dictionaries. Each dictionary has keys 'tvec' and 'norm' - tvec refers to the translation vector from the origin to the point - norm refers to the point's normal vector - ax - matplotlib axis - scale - size of the normal vectors - c - color of points and normals + """Helper function to plot points with normals + + Parameters + ---------- + pts_and_norms : list of dict + Each dictionary has keys 'tvec' and 'norm' tvec refers to the translation vector + from the origin to the point norm refers to the point's normal vector + ax : mpl_toolkits.mplot3d.axes3d.Axes3D + matplotlib axis + scale : int or float, optional + Size of the normal vectors + c : str, optional + Color of points and normals """ for pt_and_norm in pts_and_norms: # Calculate translation vector relative to camera - tvec = pt_and_norm['tvec'] + tvec = pt_and_norm['tvec'][:, 0] # Plot point ax.scatter([tvec[0]], [tvec[1]], [tvec[2]], marker='o', c='r', s=scale*2) # Calculate scaled normal vector in camera frame - s_norm = scale * np.array(pt_and_norm['norm']) + s_norm = scale * pt_and_norm['norm'][:, 0] # Plot normal vector ax.plot3D([tvec[0], tvec[0] + s_norm[0]], @@ -143,23 +153,30 @@ def plot_pts_and_norms(pts_and_norms, ax, scale=5, c='r'): c=c) -def show_pts_and_norms(rmat, tvec, pts_and_norms, ax=None, c='r', texts=[None, None]): +def show_pts_and_norms(rmat, tvec, pts_and_norms, ax=None, c='r', texts=(None, None)): """ Helper function to plot transformation from the origin to given coordinate system, - and show a set of points. Originally intended to plot the tgts frame and the - targets with their normals - - Input: - rmat - basis of the coordinate system to be shown - tvec - origin of coordinate system to be shown - pts_and_norms - list of dictionaries. Each dictionary has keys 'tvec' and 'norm' - tvec refers to the translation vector from the origin to the point - norm refers to the point's normal vector. Points and normals are in the - coordinate frame of the origin - ax - matplotlib axis - scale - size of the normal vectors - c - color of points and normals - texts - labels for the origin and transformed coordinate system + and show a set of points. Originally intended to plot the tgts frame and the targets + with their normals + + Parameters + ---------- + rmat : array_like, shape (3, 3) + Basis of the coordinate system to be shown + tvec : array_like, shape (3,) + Origin of coordinate system to be shown + pts_and_norms : list of dict + Each dictionary has keys 'tvec' and 'norm' tvec refers to the translation vector + from the origin to the point norm refers to the point's normal vector. Points + and normals are in the coordinate frame of the origin + ax : matplotlib.Axes, optional + matplotlib axis. If not provided, a figure and 3D axes are created. + scale : int or float + Size of the normal vectors + c : str, optional + Color of points and normals + texts : tuple, optional + Labels for the origin and transformed coordinate system. """ # If no axis was given, make a new one @@ -184,19 +201,25 @@ def show_pts_and_norms(rmat, tvec, pts_and_norms, ax=None, c='r', texts=[None, N def show_projection_matching(img, proj_pts, matching_points, num_matches=None, name='', bonus_pt=None, scale=10., ax=None): - """ - Show the projected target matching. - Projected points are labeled in red - Matching points are labeled in white - Line connecting point to match is drawn in black - - Input - img - Display image - proj_pts - projected location of 3D point - matching_points - image location of point - name - figure name prefix - bonus_pt - optional input. Point to be shown in blue - scale - scale of point labels + """Show the projected target matching. + + Projected points are labeled in red. Matching points are labeled in white. Line + connecting point to match is drawn in black + + Parameters + ---------- + img : array_like + Display image + proj_pts : array_like, shape (n, 2) + :rojected locations of 3D points + matching_points : array_like, shape (n, 2) + Image location of point + name : str, optinoal + Figure name prefix + bonus_pt : array_like, optional + Optional input. Point to be shown in blue + scale : int or float, optional + Scale of point labels """ if num_matches is None: @@ -217,4 +240,4 @@ def show_projection_matching(img, proj_pts, matching_points, num_matches=None, plt.scatter([bonus_pt[0]], [bonus_pt[1]], s=scale, c='b') plt.savefig(name + '_Matched_Points.png', dpi=400) - plt.close() \ No newline at end of file + plt.close() diff --git a/python/upsp/intensity_mapping/node_pixel_mapping.py b/python/upsp/intensity_mapping/node_pixel_mapping.py index 2b8c24b..4d8bfcf 100644 --- a/python/upsp/intensity_mapping/node_pixel_mapping.py +++ b/python/upsp/intensity_mapping/node_pixel_mapping.py @@ -1,69 +1,62 @@ import numpy as np -import os -import sys import cv2 -np.set_printoptions(linewidth=np.inf, precision=4, threshold=np.inf) - -current_dir = os.path.dirname(os.path.abspath(__file__)) -parent_dir = os.path.dirname(current_dir) - -utils = os.path.join(parent_dir, 'cam_cal_utils') -sys.path.append(utils) +from upsp.cam_cal_utils import photogrammetry -import visualization -import visibility -import photogrammetry -import patching +np.set_printoptions(linewidth=np.inf, precision=4, threshold=np.inf) def node_to_pixel_mapping_keyframe(rmat, tvec, cameraMatrix, distCoeffs, vis_checker, nodes, normals): """Returns data on the projected visible nodes for a keyframe - + Returns a numpy array of the pixel positions of the projected locations of the visible nodes. Additionally returns the gradient of the pixel position with respect - to the rotation vector and translation vector. Non-visible nodes are given a pixel + to the rotation vector and translation vector. Non-visible nodes are given a pixel position and gradient value of NAN. Additionally returns a sorted numpy array of the indices of the nodes that are visible. Parameters ---------- - rmat : np.ndarray (3x3), float + rmat : np.ndarray, shape (3, 3), float The rotation matrix from the camera to the model - tvec : np.ndarray (3x1) or (3,), float + tvec : np.ndarray, shape (3, 1), float The translation vector from the camera to the model - cameraMatrix : np.ndarray (3x3), float + cameraMatrix : np.ndarray, shape (3, 3), float The (openCV formatted) camera matrix for the camera - distCoeffs : np.ndarray (5x1) or (5,), float + distCoeffs : np.ndarray, shape (5, 1) or (5,), float The (openCV formatted) distortion coefficients for the camera - vis_checker : VisibilityChecker from upsp/python/upsp/cam_cal_utils/visibility.py - VisibilityChecker object with the relevant BVH and oblique viewing angle - nodes : np.ndarray (N, 3), float - A numpy array of the X, Y, and Z values of the nodes. nodes[n] is associated - with normals[n] - normals : np.ndarray (N, 3), float - A numpy array of the i, j, and k values of the node normals. normals[n] is - associated with nodes[n] - + vis_checker : ~upsp.cam_cal_utils.visibility.VisibilityChecker + ``VisibilityChecker`` object with the relevant BVH and oblique viewing angle + nodes : np.ndarray, shape (N, 3), float + A numpy array of the X, Y, and Z values of the nodes. ``nodes[n]`` is associated + with ``normals[n]`` + normals : np.ndarray, shape (N, 3), float + A numpy array of the i, j, and k values of the node normals. ``normals[n]`` is + associated with ``nodes[n]`` + Returns - ---------- - tuple of length 3 : (projections, jacobians, visible_indices) - projections : np.ndarray (N, 2), float - pixel locations of the projected nodes. Non-visibles nodes will be NAN. - projections[i] is associated with nodes[i] and jacobian[i] - jacobians : np.ndarray (N, 2, 6), float - Jacobian of pixel locations of the projected nodes. Non-visibles nodes will - be NAN. Jacobian axis 2 refers to (rvec | tvec). jacobian[i] is associated - with nodes[i] and projections[i] - visible_indices : np.ndarray (V,), int - sorted numpy array of the indices of the visibles nodes + ------- + projections : np.ndarray, shape (N, 2), float + Pixel locations of the projected nodes. Non-visibles nodes will be NAN. + ``projections[i]`` is associated with ``nodes[i]`` and ``jacobian[i]`` + jacobians : np.ndarray, shape (N, 2, 6), float + Jacobian of pixel locations of the projected nodes. Non-visibles nodes will + be NAN. Jacobian axis 2 refers to (`rvec` | `tvec`). ``jacobian[i]`` is + associated with ``nodes[i]`` and ``projections[i]`` + visible_indices : np.ndarray, shape (V,), int + Sorted numpy array of the indices of the visibles nodes + + See Also + -------- + node_to_pixel_mapping_non_keyframe : use outputs to quickly map non-keyframes """ # Get the visible nodes rmat_model2camera, tvec_model2camera = photogrammetry.invTransform(rmat, tvec) vis_idxs = vis_checker.is_visible(tvec_model2camera, nodes, normals) + vis_idxs = np.sort(vis_idxs) vis_nodes = nodes[vis_idxs] - + # Project all visible nodes projs, jacs = photogrammetry.project_3d_point(rmat, tvec, cameraMatrix, distCoeffs, vis_nodes, ret_jac=True) @@ -72,44 +65,48 @@ def node_to_pixel_mapping_keyframe(rmat, tvec, cameraMatrix, distCoeffs, vis_che # Initialize an output of all NANs projs_fin = np.full((len(nodes), 2), np.NAN) jacs_fin = np.full((len(nodes), 2, 6), np.NAN) - + # For visible nodes, replace the NANs with data projs_fin[vis_idxs] = projs jacs_fin[vis_idxs] = jacs - return projs_fin, jacs_fin, np.sort(vis_idxs) + return projs_fin, jacs_fin, vis_idxs def node_to_pixel_mapping_non_keyframe(rmat_key, tvec_key, rmat_curr, tvec_curr, projs, jacs, vis_idxs): """Returns data on the projected visible nodes for a non-keyframe - + Returns a numpy array of the pixel positions of the projected locations of the visible nodes. Non-visible nodes are given a pixel position of NAN. Assumes set of - nodes visible in keyframe is same for nearby non-keyframes. Uses the jacobian to + nodes visible in keyframe is same for nearby non-keyframes. Uses the Jacobian to quickly approximate the projected location Parameters ---------- - rmat_key : np.ndarray (3x3), float + rmat_key : np.ndarray, shape (3, 3), float The rotation matrix from the camera to the model of the keyframe - tvec_key : np.ndarray (3x1) or (3,), float + tvec_key : np.ndarray, shape (3, 1), float The translation vector from the camera to the model of the keyframe - rmat_curr : np.ndarray (3x3), float + rmat_curr : np.ndarray, shape (3, 3), float The rotation matrix from the camera to the model of the current non-keyframe - tvec_curr : np.ndarray (3x1) or (3,), float + tvec_curr : np.ndarray, shape (3, 1) or (3,), float The translation vector from the camera to the model of the current non-keyframe - projs : np.ndarray (N, 2), float + projs : np.ndarray, shape (N, 2), float Projected locations of the nodes in the keyframe - jacs : np.ndarray (N, 2, 6), float + jacs : np.ndarray, shape (N, 2, 6), float Jacobians of the projected locations - vis_idxs : np.ndarray (V,), float - Indices of the visible nodes. + vis_idxs : np.ndarray, shape (V,), float + Indices of the visible nodes. Returns - ---------- - updated_projections : np.ndarray (N, 2), float + ------- + updated_projections : np.ndarray, shape (N, 2), float Approximate pixel locations of the projected nodes. Non-visibles nodes will be - NAN. updated_projections[i] is associated with projs[i] + NAN. ``updated_projections[i]`` is associated with ``projs[i]`` + + See Also + -------- + node_to_pixel_mapping_keyframe : create keyframe inputs """ # Convert the rotation matrix to a rotation vector @@ -117,15 +114,15 @@ def node_to_pixel_mapping_non_keyframe(rmat_key, tvec_key, rmat_curr, tvec_curr, rvec_curr = np.array(cv2.Rodrigues(rmat_curr)[0]) # Get the delta Transformation - dr = np.squeeze(rvec_curr - rvec_key, axis=1) + dr = rvec_curr - rvec_key dt = tvec_curr - tvec_key dT = np.concatenate((dr, dt)) - + projs_partial = projs[vis_idxs] jacs_partial = jacs[vis_idxs] # Get the pixel updates - projs_updated = projs_partial + np.sum(jacs_partial * dT, axis=2) + projs_updated = projs_partial + np.sum(jacs_partial * dT.T, axis=2) projs_fin = np.full((len(projs), 2), np.NAN) projs_fin[vis_idxs] = projs_updated @@ -135,7 +132,7 @@ def node_to_pixel_mapping_non_keyframe(rmat_key, tvec_key, rmat_curr, tvec_curr, def node_to_pixel_mapping_non_keyframe_full(rmat, tvec, cameraMatrix, distCoeffs, nodes, vis_idxs): """Returns data on the projected visible nodes for a non-keyframe - + Returns a numpy array of the pixel positions of the projected locations of the visible nodes. Non-visible nodes are given a pixel position of NAN. Assumes set of nodes visible in keyframe is same for nearby non-keyframes. Fully computes the @@ -143,25 +140,29 @@ def node_to_pixel_mapping_non_keyframe_full(rmat, tvec, cameraMatrix, distCoeffs Parameters ---------- - rmat : np.ndarray (3x3), float + rmat : np.ndarray, shape (3, 3), float The rotation matrix from the camera to the model of the current non-keyframe - tvec : np.ndarray (3x1) or (3,), float + tvec : np.ndarray, shape (3, 1), float The translation vector from the camera to the model of the current non-keyframe - cameraMatrix : np.ndarray (3x3), float + cameraMatrix : np.ndarray, shape (3, 3), float The (openCV formatted) camera matrix for the camera - distCoeffs : np.ndarray (5x1) or (5,), float + distCoeffs : np.ndarray, shape (5, 1) or (5,), float The (openCV formatted) distortion coefficients for the camera - nodes : np.ndarray (N, 3), float - A numpy array of the X, Y, and Z values of the nodes. nodes[n] is associated - with normals[n] - vis_idxs : np.ndarray (V,), float + nodes : np.ndarray, shape (N, 3), float + A numpy array of the X, Y, and Z values of the nodes. ``nodes[n]`` is associated + with ``normals[n]`` + vis_idxs : np.ndarray, shape (V,), float Indices of the visible nodes Returns - ---------- - updated_projections : np.ndarray (N, 2), float + ------- + updated_projections : np.ndarray, shape (N, 2), float Pixel locations of the projected nodes. Non-visibles nodes will be NAN. - updated_projections[i] is associated with nodes[i] + ``updated_projections[i]`` is associated with ``nodes[i]`` + + See Also + -------- + node_to_pixel_mapping_keyframe : create `vis_idxs` based on a keyframe """ # Get just the visible nodes vis_nodes = nodes[vis_idxs] @@ -174,4 +175,3 @@ def node_to_pixel_mapping_non_keyframe_full(rmat, tvec, cameraMatrix, distCoeffs projs_fin[vis_idxs] = projs return projs_fin - diff --git a/python/upsp/intensity_mapping/patching.py b/python/upsp/intensity_mapping/patching.py index 25a70db..3c30805 100644 --- a/python/upsp/intensity_mapping/patching.py +++ b/python/upsp/intensity_mapping/patching.py @@ -1,17 +1,9 @@ import numpy as np import copy import cv2 -import time -import os -import sys -current_dir = os.path.dirname(os.path.abspath(__file__)) -parent_dir = os.path.dirname(current_dir) -utils = os.path.join(parent_dir, 'cam_cal_utils') -sys.path.append(utils) +from upsp.cam_cal_utils import photogrammetry -import photogrammetry -import visibility np.set_printoptions(linewidth=180, precision=4, threshold=np.inf) @@ -37,103 +29,103 @@ def get_target_node_idxs(nodes, tgts, buffer_thickness_in): Returns list of indices of nodes that fall within the patch of a target. This is calculated by finding the distance from every node to every target. All nodes that - are a distance less than tgt['size'] from the target are marked as 'invalid' and the - indices of those nodes are returned + are a distance less than ``tgt['size']`` from the target are marked as 'invalid' and + the indices of those nodes are returned This is calculated in a two step procedure for a speedup. It is based on the fact - that euclidean distance <= manhattan distance <= sqrt(3) * euclidean distance. And - that manhattan distance is much faster to calculate than euclidean distance since + that Euclidean distance <= Manhattan distance <= sqrt(3) * euclidean distance, and + that Manhattan distance is much faster to calculate than Euclidean distance since there is no square or square root operation. - Step 1 calculates the manhattan distance (L1 norm) from every node to every target. - If the manhattan distance is greater than sqrt(3) * tgt['size'], we know for certain - that the node is at least tgt['size'] units of euclidean distance from the target. - This will be the vast majority of nodes (all but ~300 for example launch vehicle - w/ ~1 million nodes). + Step 1 calculates the Manhattan distance (L1 norm) from every node to every target. + If the Manhattan distance is greater than ``sqrt(3) * tgt['size']``, we know for + certain that the node is at least ``tgt['size']`` units of Euclidean distance from + the target. This will be the vast majority of nodes (all but ~300 for example launch + vehicle w/ ~1 million nodes). - Step 2 checks the euclidean distance of each node that failed the manhattan distance - check. Nodes that fall within tgt['size'] of any target are marked as invalid and - their index is returned. + Step 2 checks the Euclidean distance of each node that failed the Manhattan distance + check. Nodes that fall within ``tgt['size']`` of any target are marked as invalid + and their index is returned. If a large portion of the nodes are near a target, this process would be slow since it is effectively double calculating any node near a target. However, since the - targets are relatively small compared to the surface area of the, it results in a - roughly 2X speedup. + targets are relatively small compared to the surface area of the model, it results + in a roughly 2X speedup. Parameters ---------- - nodes : np.ndarray (N, 3), float + nodes : np.ndarray, shape (N, 3), float A numpy array of the X, Y, and Z values of the nodes - tgts : list of targets + tgts : list Each target is a dictionary with (at a minimum) a 'tvec' and 'size' attribute. - The 'tvec' attribute gives the target's location and the 'size' gives the - euclidean distance from the center of the target to the perimeter + The 'tvec' attribute gives the target's location and the 'size' gives the + Euclidean distance from the center of the target to the perimeter buffer_thickness_in : float - Buffer (in inches) to add to fiducials when determining internals applied radially - (increases effective radius of fiducial by buffer_thickness_in) + Buffer (in inches) to add to fiducials when determining internals applied + radially (increases effective radius of fiducial by `buffer_thickness_in`) Returns - ---------- - sorted numpy array (np.int32) of the indices of the nodes that are inside of a target + ------- + nodes_in_targets : np.ndarray + Sorted array of the indices (``np.int32``) of the nodes that are inside of a + target """ # Heuristic - Nodes within np.sqrt(3) * tgt['size'] units of manhattan distance are # flagged as potential invalid nodes heuristic_invalid_nodes = set() for tgt in tgts: - manhattan_dist = np.linalg.norm(np.absolute(nodes - tgt['tvec']), ord=1, axis=1) + manhattan_dist = np.linalg.norm(np.absolute(nodes - tgt['tvec'].T), ord=1, axis=1) heuristic_invalid_nodes.update(np.squeeze(np.argwhere(manhattan_dist < (np.sqrt(3) * (tgt['size'] / 2 + buffer_thickness_in))), axis=1).tolist()) heuristic_invalid_nodes = list(heuristic_invalid_nodes) # Final Check - Of the nodes flagged, check if any are within of euclidean distance invalid_nodes = set() for tgt in tgts: - dist = np.linalg.norm(nodes[heuristic_invalid_nodes] - tgt['tvec'], axis=1) + dist = np.linalg.norm(nodes[heuristic_invalid_nodes] - tgt['tvec'].T, axis=1) invalid_nodes.update(set(heuristic_invalid_nodes[i] for i in np.squeeze(np.argwhere(dist < (tgt['size'] / 2 + buffer_thickness_in)), axis=1))) - + return np.array(sorted(invalid_nodes), dtype=np.int32) def patchFiducials(fiduals_visible, inp_img, rmat, tvec, cameraMatrix, distCoeffs, boundary_thickness, buffer_thickness_in): - """ Patches clusters in the inp_img + """Patches clusters in the `inp_img` - Parameters: - ----------- - fiduals_visible : list of fiducials + Parameters + ---------- + fiduals_visible : list Each fiducial is a dict with (at a minimum) 'tvec' and 'target_type' attributes. The 'tvec' attribute gives the fiducial's location and 'target_type' is a string denoting the type of fiducial (most commonly 'dot' or 'kulite') - inp_img : np.uint8 np.ndarray (image) - Input image to be patched. rmat and tvec should be aligned to image - rmat : np.ndarray (3, 3), float + inp_img : np.ndarray, np.uint8 + Input image to be patched. `rmat` and `tvec` should be aligned to image + rmat : np.ndarray, shape (3, 3), float Rotation matrix from camera to object - tvec : np.ndarray (3, 1) or (3,), float + tvec : np.ndarray, shape (3, 1), float Translation vector from camera to object - cameraMatrix : np.ndarray (3x3), float + cameraMatrix : np.ndarray, shape (3x3), float The (openCV formatted) camera matrix for the camera - distCoeffs : np.ndarray (5x1) or (5,), float + distCoeffs : np.ndarray, shape (5, 1) or (5,), float The (openCV formatted) distortion coefficients for the camera boundary_thickness : int - thickness of boundary (in pixels) + Thickness of boundary (in pixels) buffer_thickness_in : float - Buffer (in inches) to add to fiducials when determining internals applied radially - (increases effective radius of fiducial by buffer_thickness_in) + Buffer (in inches) to add to fiducials when determining internals applied + radially (increases effective radius of fiducial by `buffer_thickness_in`) - Returns: - -------------- - float np.ndarray (image) + Returns + ------- + out_img : np.ndarray, float Image with patched fiducials - See Also: - ----------- - get_fiducial_internal_and_boundary : - Return internal and boundary pixel positions for the input fiducial - get_cluster_internal_and_boundary : - Returns a list of internal and boundary pixels for the input cluster - polyfit2D : - Finds the polynomial fit using the boundary pixels - polyval2D : - Finds the value for the internal pixels using the polynomial fit + See Also + -------- + get_fiducial_internal_and_boundary : Return internal and boundary pixel positions + for the input fiducial + get_cluster_internal_and_boundary : Returns a list of internal and boundary pixels + for the input cluster + polyfit2D : Finds the polynomial fit using the boundary pixels + polyval2D : Finds the value for the internal pixels using the polynomial fit """ # Create an output copy of the input image to work with out_img = copy.deepcopy(inp_img).astype(np.float32) @@ -163,7 +155,7 @@ def patchFiducials(fiduals_visible, inp_img, rmat, tvec, cameraMatrix, distCoeff # Fit a 2D polynomial to the boundary intensities coeffs = polyfit2D(local_bounds, Is) - + # Using the fit, find values for the internal intensities new_internal_intensities = polyval2D(local_internals, coeffs) @@ -175,35 +167,41 @@ def patchFiducials(fiduals_visible, inp_img, rmat, tvec, cameraMatrix, distCoeff def clusterFiducials(fiduals_visible, rmat, tvec, cameraMatrix, distCoeffs, boundary_thickness, buffer_thickness_in): - """ Clusters input fiducials based on image location and image size + """Clusters input fiducials based on image location and image size - A cluster is made of all fiducials with a path of overlap between them. I.e. A is - overlapping B which is overlapping C and D. Therefore A, B, C, and D are all in the + A cluster is made of all fiducials with a path of overlap between them, i.e. A is + overlapping B which is overlapping C and D, therefore A, B, C, and D are all in the same cluster. It is considered overlap if the internal pixels to one fiducial overlap the internal or boundary pixels of another fiducial. A cluster can be a single fiducial if there is no overlap. This function clusters targets, then returns the clusters. - Parameters: - ----------- - fiduals_visible : list of fiducials + Parameters + ---------- + fiduals_visible : list Each fiducial is a dict with (at a minimum) 'tvec' and 'target_type' attributes. The 'tvec' attribute gives the fiducial's location and 'target_type' is a string denoting the type of fiducial (most commonly 'dot' or 'kulite') - rmat : np.ndarray (3, 3), float + rmat : np.ndarray, shape (3, 3), float Rotation matrix from camera to object - tvec : np.ndarray (3, 1) or (3,), float + tvec : np.ndarray, shape (3, 1), float Translation vector from camera to object - cameraMatrix : np.ndarray (3x3), float + cameraMatrix : np.ndarray, shape (3, 3), float The (openCV formatted) camera matrix for the camera - distCoeffs : np.ndarray (5x1) or (5,), float + distCoeffs : np.ndarray, shape (5, 1) or (5,), float The (openCV formatted) distortion coefficients for the camera boundary_thickness : int - thickness of boundary (in pixels) + Thickness of boundary (in pixels) buffer_thickness_in : float - Buffer (in inches) to add to fiducials when determining internals applied radially - (increases effective radius of fiducial by buffer_thickness_in) + Buffer (in inches) to add to fiducials when determining internals applied + radially (increases effective radius of fiducial by `buffer_thickness_in`) + + Returns + ------- + clusters : list + List of clusters, where each cluster is a list of fiducials from + `fiducials_visible`. """ # Represent fiducials as circles with center and radius unclustered_fiducial_sets = [] #deleteme? @@ -226,7 +224,7 @@ def clusterFiducials(fiduals_visible, rmat, tvec, cameraMatrix, distCoeffs, boun # set of internal or set of boundary pixels of another # Clusters are connected components of that graph clusters = [] - + # 1) Add the an unclustered fiducal to a cluster. Remove it from the # unclustered list # 2) For each unclusted fiducial, find the distance to each fiducial in the @@ -255,7 +253,7 @@ def clusterFiducials(fiduals_visible, rmat, tvec, cameraMatrix, distCoeffs, boun unclustered_fiducial_sets.remove(unclustered) was_added = True break - + # If the boundaries of one overlap the internals of thee other, add # the unclustered to the cluster if clustered[1].intersection(unclustered[0]) or clustered[0].intersection(unclustered[1]): @@ -263,21 +261,21 @@ def clusterFiducials(fiduals_visible, rmat, tvec, cameraMatrix, distCoeffs, boun unclustered_fiducial_sets.remove(unclustered) was_added = True break - + if was_added: break - + clusters.append(cluster) # Repackage as fiducial cluster fiducial_clusters = [] for cluster in clusters: fiducial_cluster = [] - + for image_fiducial in cluster: i = unclustered_fiducial_sets_orig.index(image_fiducial) fiducial_cluster.append(fiduals_visible[i]) - + fiducial_clusters.append(fiducial_cluster) return fiducial_clusters @@ -285,45 +283,45 @@ def clusterFiducials(fiduals_visible, rmat, tvec, cameraMatrix, distCoeffs, boun # TODO: automatically find fiducials that are close together to group into clusters def get_cluster_internal_and_boundary(cluster, rmat, tvec, cameraMatrix, distCoeffs, boundary_thickness, buffer_thickness_in): - """ Returns a list of internal and boundary pixels for the input cluster + """Returns a list of internal and boundary pixels for the input cluster - An internal pixel is either directly a part of the fiducials or in between fiducials. - A boundary pixel is any pixel within buffer pixels of an internal pixel and is not - an internal itself - - Parameters: - ----------- - cluster : list of fiducials - Each fiducial is a dict with (at a minimum) 'tvec' and 'target_type' attributes. - The 'tvec' attribute gives the fiducial's location and 'target_type' is a string - denoting the type of fiducial (most commonly 'dot' or 'kulite') - rmat : np.ndarray (3, 3), float + An internal pixel is either directly a part of the fiducials or in between + fiducials. A boundary pixel is any pixel within buffer pixels of an internal pixel + and is not an internal itself + + Parameters + ---------- + cluster : list + List of fiducials in the cluster. Each fiducial is a dict with (at a minimum) + 'tvec' and 'target_type' attributes. The 'tvec' attribute gives the fiducial's + location and 'target_type' is a string denoting the type of fiducial (most + commonly 'dot' or 'kulite') + rmat : np.ndarray, shape (3, 3), float Rotation matrix from camera to object - tvec : np.ndarray (3, 1) or (3,), float + tvec : np.ndarray, shape (3, 1), float Translation vector from camera to object - cameraMatrix : np.ndarray (3x3), float + cameraMatrix : np.ndarray, shape (3x3), float The (openCV formatted) camera matrix for the camera - distCoeffs : np.ndarray (5x1) or (5,), float + distCoeffs : np.ndarray, shape (5, 1) or (5,), float The (openCV formatted) distortion coefficients for the camera boundary_thickness : int thickness of boundary (in pixels) buffer_thickness_in : float - Buffer (in inches) to add to fiducials when determining internals applied radially - (increases effective radius of fiducial by buffer_thickness_in) - - Returns: - ----------- - tuple - Both items are np.ndarrays. First is the internals with shape (n, 2) for the - position (x, y) of n internal pixels. Second is the boundary pixels with shape - (m, 2) for the positin (x, y) of m boundary pixels - - See Also: - ----------- - get_fiducial_boundary_map_from_internal_map : - Determines boundary pixels from bit mask of internal pixels - get_fiducial_pixel_properties : - Returns the pixel properties of the input fiducial + Buffer (in inches) to add to fiducials when determining internals applied + radially (increases effective radius of fiducial by `buffer_thickness_in`) + + Returns + ------- + internals : np.ndarray, shape (n, 2) + Positions (x, y) of the ``n`` internal points + bounds : np.ndarray, shape (m, 2) + Positions (x, y) of the ``m`` boundary pixels + + See Also + -------- + get_fiducial_boundary_map_from_internal_map : Determines boundary pixels from bit + mask of internal pixels + get_fiducial_pixel_properties : Returns the pixel properties of the input fiducial """ # First stage: Get the minimum, axis aligned rectangle that contains all fiducials # of this cluster @@ -339,7 +337,7 @@ def get_cluster_internal_and_boundary(cluster, rmat, tvec, cameraMatrix, distCoe # Update t_min and t_max t_min = np.minimum(t_min, t_min_temp) t_max = np.maximum(t_max, t_max_temp) - + # Second Stage: make a mini images the size of that minimum axis aligned rectangle # and mark internal and boundary pixels. An internal pixel is either directly a # part of the fiducials or in between fiducials. A boundary pixel is any pixel within @@ -347,17 +345,17 @@ def get_cluster_internal_and_boundary(cluster, rmat, tvec, cameraMatrix, distCoe # Mini image to contain the internal points internal_map = np.zeros((t_max[1] - t_min[1], t_max[0] - t_min[0])) - + # Mark all points that are directly a part of the fiducials as internal for tgt in cluster: internal_points, __ = get_fiducial_internal_and_boundary(tgt, rmat, tvec, cameraMatrix, distCoeffs, boundary_thickness, buffer_thickness_in) - + # Mark every internal point as internal for x, y in internal_points: internal_map[y - t_min[1]][x - t_min[0]] = 1 # Mark all points between fiducials as internal - + # Start with a dilation then erosion to fill in any small gaps between fiducials # that might not otherwise get filled. If there are no gaps, this is an identity # operation @@ -376,13 +374,13 @@ def get_cluster_internal_and_boundary(cluster, rmat, tvec, cameraMatrix, distCoe if internal_map[y][x]: min_y = y break - + # Find the bottommost internal pixel of this row for y in reversed(range(t_max[1] - t_min[1])): if internal_map[y][x]: max_y = y break - + # Fill in everything in between for y in range(min_y, max_y): internal_map[y][x] = 1 @@ -398,13 +396,13 @@ def get_cluster_internal_and_boundary(cluster, rmat, tvec, cameraMatrix, distCoe if internal_map[y][x]: min_x = x break - + # Find the bottommost internal pixel of this row for x in reversed(range(t_max[0] - t_min[0])): if internal_map[y][x]: max_x = x break - + # Fill in everything in between for x in range(min_x, max_x): internal_map[y][x] = 1 @@ -423,44 +421,44 @@ def get_cluster_internal_and_boundary(cluster, rmat, tvec, cameraMatrix, distCoe bounds += (t_min[0], t_min[1]) return internals, bounds - + def get_fiducial_internal_and_boundary(tgt, rmat, tvec, cameraMatrix, distCoeffs, boundary_thickness, buffer_thickness_in): - """ Return internal and boundary pixel positions for the input fiducial + """Return internal and boundary pixel positions for the input fiducial An internal pixel a part of the fiducials (minimum axis aligned bounding rectangle). A boundary pixel is any pixel within buffer pixels of an internal pixel and is not an internal itself - Parameters: - ----------- + Parameters + ---------- tgt : dict dict with (at a minimum) 'tvec' and 'target_type' attributes. The 'tvec' - attribute gives the fiducial's location and 'target_type' is a string denoting the - type of fiducial (most commonly 'dot' or 'kulite') - rmat : np.ndarray (3, 3), float + attribute gives the fiducial's location and 'target_type' is a string denoting + the type of fiducial (most commonly 'dot' or 'kulite') + rmat : np.ndarray, shape (3, 3), float Rotation matrix from camera to object - tvec : np.ndarray (3, 1) or (3,), float + tvec : np.ndarray, shape (3, 1), float Translation vector from camera to object - cameraMatrix : np.ndarray (3x3), float + cameraMatrix : np.ndarray, shape (3x3), float The (openCV formatted) camera matrix for the camera - distCoeffs : np.ndarray (5x1) or (5,), float + distCoeffs : np.ndarray, shape (5, 1) or (5,), float The (openCV formatted) distortion coefficients for the camera boundary_thickness : int thickness of boundary (in pixels) buffer_thickness_in : float - Buffer (in inches) to add to fiducials when determining internals applied radially - (increases effective radius of fiducial by buffer_thickness_in) - - Returns: - ----------- - tuple - Both items are np.ndarrays. First is the internals with shape (n, 2) for the - position (x, y) of n internal pixels. Second is the boundary pixels with shape - (m, 2) for the positin (x, y) of m boundary pixels - - See Also: - ----------- + Buffer (in inches) to add to fiducials when determining internals applied + radially (increases effective radius of fiducial by `buffer_thickness_in`) + + Returns + ------- + internals : np.ndarray, shape (n, 2) + Positions (x, y) of the ``n`` internal points + bounds : np.ndarray, shape (m, 2) + Positions (x, y) of the ``m`` boundary pixels + + See Also + -------- get_fiducial_boundary_map_from_internal_map : Determines boundary pixels from bit mask of internal pixels get_fiducial_pixel_properties : @@ -477,7 +475,7 @@ def get_fiducial_internal_and_boundary(tgt, rmat, tvec, cameraMatrix, distCoeffs # Get the fiducial center location within the local map tgt_proj_local = [tgt_proj['proj'][0]-t_min[0], tgt_proj['proj'][1]-t_min[1]] - + # For each pixel, find the point on the pixel closest to the center of the fiducial # The expression to find the x coordinate of that point is max(X1, min(Xc, X2)) # where X1 is the left edge of the pixel, X2 is the right edge, and Xc is the @@ -495,7 +493,7 @@ def get_fiducial_internal_and_boundary(tgt, rmat, tvec, cameraMatrix, distCoeffs xs, ys = xs.ravel(), ys.ravel() Xns = np.maximum(xs - 0.5, np.minimum(tgt_proj_local[0], xs + 0.5)) Yns = np.maximum(ys - 0.5, np.minimum(tgt_proj_local[1], ys + 0.5)) - + # For each of the closest points, if it is within the radius of the center it is # internal and needs to be marked as such in the internal_map closest_distances = np.linalg.norm([Xns - tgt_proj_local[0], Yns - tgt_proj_local[1]], axis=0) @@ -513,41 +511,41 @@ def get_fiducial_internal_and_boundary(tgt, rmat, tvec, cameraMatrix, distCoeffs # Offset by the region corner to get absolute image coordinates internals += (t_min[0], t_min[1]) bounds += (t_min[0], t_min[1]) - + return internals, bounds def get_fiducial_boundary_map_from_internal_map(internal_map, boundary_thickness): - """ Determines boundary pixels from bit mask of internal pixels + """Determines boundary pixels from bit mask of internal pixels - internal_map is a bitwise mask (image) where 1 is an internal and 0 is not. - internal_map needs to be big enough to contain all boundary pixels. That means it - needs to have a buffer of 0's around it that is boundary_thickness thick on all + `internal_map` is a bitwise mask (image) where 1 is an internal and 0 is not. + `internal_map` needs to be big enough to contain all boundary pixels. That means it + needs to have a buffer of 0's around it that is `boundary_thickness` thick on all sides (a number of columns of 0's left of leftmost internal equal to - boundary_thickness. Same for rightmost. And similarly rows above and below) + `boundary_thickness`. Same for rightmost. And similarly rows above and below) - Performs n dilation operations on the internal_map with a 3x3 square kernel where - n is equal to boundary_thickness. boundary_map (return value) is the result minus - internal_map + Performs ``n`` dilation operations on the internal_map with a 3x3 square kernel + where ``n`` is equal to boundary_thickness. `boundary_map` (return value) is the + result minus `internal_map` - Parameters: - ----------- - internal_map : np.ndarray (image) - bitwise mask of internal pixels + Parameters + ---------- + internal_map : np.ndarray + Bitwise mask of internal pixels boundary_thickness : int thickness of boundary (in pixels) - Returns: - ----------- - np.ndarray (image) - Bitmask of boundary pixels. Same dimensions of internal_map input - - See Also: - ----------- - get_fiducial_internal_and_boundary : - Return internal and boundary pixel positions for the input fiducial - get_cluster_internal_and_boundary : - Returns a list of internal and boundary pixels for the input cluster + Returns + ------- + boundary_map : np.ndarray + Bitmask of boundary pixels. Same shape as `internal_map` input + + See Also + -------- + get_fiducial_internal_and_boundary : Return internal and boundary pixel positions + for the input fiducial + get_cluster_internal_and_boundary : Returns a list of internal and boundary pixels + for the input cluster """ # Dilate the internals image to get an image of the internals and boundary kernel = np.full((3, 3), 1) @@ -561,50 +559,53 @@ def get_fiducial_boundary_map_from_internal_map(internal_map, boundary_thickness def get_fiducial_pixel_properties(tgt, rmat, tvec, cameraMatrix, distCoeffs, boundary_thickness, buffer_thickness_in): - """ Returns the pixel properties of the input fiducial + """Returns the pixel properties of the input fiducial Pixel properties refers to projected location, pixel size (adjusted for focal length, distance to camera, and diameter), and minimum axis-aligned bounding box - Parameters: - ----------- + Parameters + ---------- tgt : dict dict with (at a minimum) 'tvec' and 'target_type' attributes. The 'tvec' - attribute gives the fiducial's location and 'target_type' is a string denoting the - type of fiducial (most commonly 'dot' or 'kulite') - rmat : np.ndarray (3, 3), float + attribute gives the fiducial's location and 'target_type' is a string denoting + the type of fiducial (most commonly 'dot' or 'kulite') + rmat : np.ndarray, shape (3, 3), float Rotation matrix from camera to object - tvec : np.ndarray (3, 1) or (3,), float + tvec : np.ndarray, shape (3, 1), float Translation vector from camera to object - cameraMatrix : np.ndarray (3x3), float + cameraMatrix : np.ndarray, shape (3, 3), float The (openCV formatted) camera matrix for the camera - distCoeffs : np.ndarray (5x1) or (5,), float + distCoeffs : np.ndarray, shape (5, 1) or (5,), float The (openCV formatted) distortion coefficients for the camera boundary_thickness : int thickness of boundary (in pixels) buffer_thickness_in : float - Buffer (in inches) to add to fiducials when determining internals applied radially - (increases effective radius of fiducial by buffer_thickness_in) - - Returns: - ----------- - tuple - First item is fiducial projection which is a dict with keys 'target_type', and - 'proj' which map to a string and list of positions (length 2, (x, y)) - respectfully. 'target_type' matches the input 'target_type'. Second item is - a float for the fiducial size in pixels accounting for the focal length, distance - to camera, and diameter. Does not take model geometry into account, and is - either exactly accurate or an over estimate (likely a mild overestimate). Last - two items are t_min and t_max. t_min is the upper left corner of the minimum - axis aligned bounding box of the fiducial plus boundary pixels. t_max is the - bottom right corner for that bounding box. Both are int np.ndarrays of (2,) - - See Also: - ----------- - get_fiducial_internal_and_boundary : - Return internal and boundary pixel positions for the input fiducial - get_cluster_internal_and_boundary : - Returns a list of internal and boundary pixels for the input cluster + Buffer (in inches) to add to fiducials when determining internals applied + radially (increases effective radius of fiducial by `buffer_thickness_in`) + + Returns + ------- + tgt_proj : dict + Fiducial projection which is a dict with keys 'target_type', and 'proj' which + map to a string and list of positions (length 2, (x, y)) respectively. + 'target_type' matches the input 'target_type'. + targ_size_px : float + Fiducial size in pixels accounting for the focal length, distance to camera, and + diameter. Does not take model geometry into account, and is either exactly + accurate or an over estimate (likely a mild overestimate). + t_min : np.ndarray, shape (2,), int + Upper left corner of the minimum axis aligned bounding box of the fiducial plus + boundary pixels. + t_max : np.ndarray, shape (2,), int + Bottom right corner for the bounding box. + + See Also + -------- + get_fiducial_internal_and_boundary : Return internal and boundary pixel positions + for the input fiducial + get_cluster_internal_and_boundary : Returns a list of internal and boundary pixels + for the input cluster """ # Effective size of fiducial in inches targ_size_in = tgt['size'] + 2 * buffer_thickness_in @@ -627,33 +628,34 @@ def get_fiducial_pixel_properties(tgt, rmat, tvec, cameraMatrix, distCoeffs, bou # Take the floor and ceiling respectively to convert to an integer t_min = np.floor(t_min).astype(np.int32) t_max = np.ceil(t_max).astype(np.int32) - + return tgt_proj, targ_size_px, t_min, t_max def polyfit2D(bounds, Is): - """ Finds the polynomial fit using the boundary pixels - - Parameters: - ----------- - bounds : np.ndarray vector (n, 2) of floats + """Finds the polynomial fit using the boundary pixels + + Parameters + ---------- + bounds : np.ndarray, shape (n, 2) of floats (x, y) position of boundary pixels in local coordinates (i.e. leftmost boundary - pixel has x coordinate of 0. Topmost has coordinate of 0). bounds[n] corresponds - to Is[n] - Is : np.ndarray vector (n,) of floats - Intensity value of boundary pixels pixels. Is[n] corresponds to bounds[n] - - Returns: - ----------- - np.ndarray vector (n, 1) - coeffs polynomial fit coefficients - - See Also: - ----------- + pixel has x coordinate of 0. Topmost has coordinate of 0). ``bounds[n]`` + corresponds to ``Is[n]`` + Is : np.ndarray, shape (n,) of floats + Intensity value of boundary pixels pixels. ``Is[n]`` corresponds to + ``bounds[n]`` + + Returns + ------- + coeffs : np.ndarray, shape (n, 1) + Polynomial fit coefficients + + See Also + -------- polyval2D : Finds the value for the internal pixels using the polynomial fit """ assert((len(bounds) == len(Is))) - + # Determine the number of coefficients # Terms are 1 constant, 2 linear (x, y), 3 quadratic (xx, xy, yy), ... # Equivalent to 1 + 2 + ... + degree + (degree+1) = (degree+2)*(degree+1)/2 @@ -674,37 +676,37 @@ def polyfit2D(bounds, Is): if (i + j) <= degree: A[:, count] = pow(bounds[:, 1], i) * pow(bounds[:, 0], j) count += 1 - + # Solution to linear equation is just pixel values b = np.array(Is) # Initialize polynomial coefficient output vector poly = np.zeros(num_coeffs, dtype=np.float32) - + # Solve least squares problem coeffs = np.linalg.lstsq(A, b, rcond=None)[0] - + return np.array(coeffs) def polyval2D(internals, coeffs): """ Finds the value for the internal pixels using the polynomial fit - - Parameters: - ----------- - internals : np.ndarray vector (n, 2) of floats + + Parameters + ---------- + internals : np.ndarray, shape (n, 2) of floats (x, y) position of internal pixels in local coordinates (i.e. leftmost boundary pixel has x coordinate of 0. Topmost has y coordinate of 0) - coeffs : np.ndarray vector (n,) + coeffs : np.ndarray, shape (n,) Polynomial fit coefficients - - Returns: - ----------- - float np.ndarray (n,) + + Returns + ------- + Is : np.ndarray, shape (n,) Estimated intensity for internals - See Also: - ----------- + See Also + -------- polyfit2D : Finds the polynomial fit using the boundary pixels """ # Determine the number of coefficients @@ -727,4 +729,3 @@ def polyval2D(internals, coeffs): Is = np.matmul(A, coeffs) return Is - diff --git a/python/upsp/kulite_comparison/plotting.py b/python/upsp/kulite_comparison/plotting.py index 8e9dcb9..d9e93c3 100644 --- a/python/upsp/kulite_comparison/plotting.py +++ b/python/upsp/kulite_comparison/plotting.py @@ -44,17 +44,17 @@ def make_selection_area_images(df: pd.DataFrame, out_dir=None): Parameters ---------- df : pandas.DataFrame - Records with the following columns: - 'Pipeline Directory': data pipeline top-level directory - 'Datapoint': datapoint identifier - 'Kulite Nearest Vertex': index of vertex nearest to Kulite - 'Kulite Name': name of kulite - 'Selection Vertices': string repr of list of indices of selected vertices, - e.g., "[124,231,101]" + Records with the following columns: + + - 'Pipeline Directory' : data pipeline top-level directory + - 'Datapoint' : datapoint identifier + - 'Kulite Nearest Vertex' : index of vertex nearest to Kulite + - 'Kulite Name' : name of kulite + - 'Selection Vertices' : string repr of list of indices of selected vertices, + e.g., "[124,231,101]" out_dir : str or None - Directory for writing output images. If None, defaults to - subdirectories of the current working directory named according to - the 'Pipeline Directory' names. + Directory for writing output images. If None, defaults to subdirectories of the + current working directory named according to the 'Pipeline Directory' names. """ if out_dir is None: out_dir = os.getcwd() diff --git a/python/upsp/processing/grids.py b/python/upsp/processing/grids.py index dade993..ff304a4 100644 --- a/python/upsp/processing/grids.py +++ b/python/upsp/processing/grids.py @@ -6,17 +6,21 @@ class StructGrid: """Manage plot3d-style structured grid and write formatted to file - Attributes: - sz : array-like, sizes of each grid [3xN] - x : array-like, x-position [N] - y : array-like, y-position [N] - z : array-like, z-position [N] - zones : list of vertices ordered by zones [N] + Attributes + ---------- + sz : array_like + sizes of each grid [3xN] + x : array_like + x-position [N] + y : array_like, + y-position [N] + z : array_like, + z-position [N] + zones : np.ndarray + vertices ordered by zones [N] """ def __init__(self): - """ Create empty StructGrid """ - self.sz = [] self.x = [] self.y = [] @@ -26,13 +30,11 @@ def __init__(self): def load_grid(self, grid_file): """Read a formatted p3d file - Args: - grid_file (str) : formatted plot3d file - - Returns: - None + Parameters + ---------- + grid_file : str + formatted plot3d file """ - with open(grid_file, "r") as f: n_zones = int(f.readline()) @@ -64,13 +66,11 @@ def load_grid(self, grid_file): self.zones = np.array(zone_list, dtype=np.float32) def num_zones(self): - """ Return the number of grids (or zones) """ - + """Return the number of grids (or zones)""" return len(self.sz) def size(self): - """ Return the number of grid nodes """ - + """Return the number of grid nodes""" total_size = 0 for i in range(0, self.num_zones()): total_size += np.product(self.sz[i]) @@ -80,17 +80,21 @@ def size(self): def num_faces(self, zone=None): """Return the number of faces in a zone - Args: - zone (int): zone number (0-based) + Parameters + ---------- + zone : int, optional + zone number (0-based) - Returns: - (int) if zone = None, number of faces in grid - else, number of faces in zone + Returns + ------- + n_faces : int + Number of faces in the given `zone` (if provided), otherwise the total + number of faces in the grid. - Raises: - RuntimeError: invalid zone number + Raises + ------ + RuntimeError: invalid zone number """ - n_faces = 0 if zone is None: @@ -113,13 +117,11 @@ def num_faces(self, zone=None): def write_p3d(self, fileout): """Write formatted p3d file - Args: - fileout (str) : output file - - Returns: - None + Parameters + ---------- + fileout : str + output file """ - with open(fileout, "w") as f: f.write("{}\n".format(len(self.sz))) for z in range(0, len(self.sz)): @@ -139,31 +141,34 @@ def write_p3d(self, fileout): def write_zones_mapping(self, fileout): """Write out the binary vertex zones mapping from a plot3d grid - Args: - fileout (str) : output file - - Returns: - None + Parameters + ---------- + fileout : str + output file """ - self.zones.tofile(fileout) class UnstructGrid: """Manages triangulated unstructured grid - Attributes: - n_comps : number of components - tris : array-like, node ids in each triangle [3,T] - comps : array-like, component id for each node [N] - x : array-like, x-position of each node [N] - y : array-like, y-position of each node [N] - z : array-like, z-position of each node [N] + Attributes + ---------- + n_comps : int + number of components + tris :array_like + node ids in each triangle [3,T] + comps : array_like + component id for each node [N] + x : array_like + x-position of each node [N] + y : array_like + y-position of each node [N] + z : array_like + z-position of each node [N] """ def __init__(self): - """ Create an empty UnstructGrid """ - self.x = [] self.y = [] self.z = [] @@ -174,24 +179,27 @@ def __init__(self): self.n_comps = 0 def num_comps(self): - """ Return the number of components """ + """Return the number of components""" return self.n_comps def num_nodes(self): - """ Return the number of nodes """ + """Return the number of nodes""" return len(self.x) def num_faces(self, comp=None): """Return the number of faces in a component - Args: - comp (int): component number (id) + Parameters + ---------- + comp : int, optional + component number (id) - Returns: - (int) if comp = None, number of faces in grid, else number of - faces in component + Returns + ------- + n_faces : int + Number of faces in the given `comp` (if provided), otherwise the total + number of faces in the grid. """ - n_faces = 0 if comp is None: @@ -206,14 +214,18 @@ def num_faces(self, comp=None): def extract_comp(self, comp): """Extract a sub-grid containing just the component of interest - Args: - comp (int): component id - - Returns: - (UnstructGrid) with just the selected component and mapping of - old nodes to new nodes + Parameters + ---------- + comp : int + component id + + Returns + ------- + g : UnstructGrid + New unstructured grid with just the selected component + n2n : list + mapping of old nodes to new nodes """ - # Initialize the new grid g = UnstructGrid() @@ -257,14 +269,19 @@ def extract_comp(self, comp): def get_area(self, t): """Return the area of a triangle - Args: - t (int) : triangle index + Parameters + ---------- + t : int + triangle index - Returns: - (float) area of the triangle + Returns + ------- + float + area of the triangle - Raises: - RuntimeError : The triangle index is invalid + Raises + ------ + RuntimeError : The triangle index is invalid """ if t < 0 or t >= self.num_faces(): diff --git a/python/upsp/processing/kulite_utilities.py b/python/upsp/processing/kulite_utilities.py index 5401f1b..975cb9e 100644 --- a/python/upsp/processing/kulite_utilities.py +++ b/python/upsp/processing/kulite_utilities.py @@ -1,12 +1,4 @@ -''' -Kulites class and helpers taken and pared-down from Jessie Powell code present -in branch uPSP-40. -Modified to return all kulites if no kulite list is passed in. -Modified to generate rms field with optional filter option or generate from psd -with start freq -Modified to allow global application of sos filter - -''' +"""Kulites class and helpers""" import numpy as np import os import re @@ -26,28 +18,34 @@ class Kulites: - """ Load and Manage kulite data. - Data is in PSI unless optional psf argument is true - - Attributes: - data : dict - keys : kulite names [K] - values : numpy array of pressure-time history [P] + """Load and manage kulite data + + Data is in PSI unless optional psf argument is True + + Parameters + ---------- + data_dir : str + directory with ``*.info``, ``*.slow``, ``*.fast`` files + run : int + run number + seq : int + sequence number + kulites : list + kulites to load + data_type : numpy.dtype + data type of array elements + f_type : str + 'slow' or 'fast' + + Attributes + ---------- + data : dict + Dictionary with Kulite names as keys and numnpy arrays with pressure time + histoires as values. """ def __init__(self, data_dir, run, seq, kulites='all', data_type=np.float32, psf=False, f_type='slow'): - """ Load the kulite data into a dictionary struct - - Args: - data_dir (str) : directory with *.info, *.slow, *.fast files - run (int) : run number - seq (int) : sequence number - kulites (List) : kulites to load - data_type (np.dtype) : data type of array elements - f_type (str) : 'slow' or 'fast' - """ - # Create dictionary to hold data self.raw, raw = dict(), dict() self.native, native = dict(), dict() @@ -361,14 +359,20 @@ def try_load_cal(): ############################################################################### def read_tgts(tgts_file, kulites='all'): - """ Read in the tgts file data and return the coordinates of the kulites - - Args: - tgts_file (str) : targets data file - kulites (List) : list of kulite strings - - Returns: - (Dict) of (x,y,z) positions for each kulite keyed by name + """Read in the tgts file data and return the coordinates of the kulites + + Parameters + ---------- + tgts_file : str + targets data file + kulites : list, optional + list of kulite strings. If "all" (default), all targets identified as Kulites + are returned. + + Returns + ------- + pos : dict of list + (x,y,z) positions for each kulite keyed by name """ pos = dict() @@ -389,10 +393,10 @@ def read_tgts(tgts_file, kulites='all'): def compute_delta_rms(kulites, sosfilter=None): - ''' + """ calculate rms values for each kulite by subtracting off the mean, and return a dict. if sosfilter is provided, applies it to the timeseries before calculating rms - ''' + """ def calc_rms(a): return np.sqrt(np.mean(a ** 2)) @@ -411,10 +415,10 @@ def calc_rms(a): def compute_rms_from_psd(kulites, startfreq=None): - ''' + """ calculate rms values for each kulite in the input collection by integrating its psd if startfreq is provided, integrates from startfreq upwards - ''' + """ rms_dict = dict() psds = compute_psd(kulites) freqs = psds['freq'] @@ -433,12 +437,12 @@ def compute_rms_from_psd(kulites, startfreq=None): def apply_filter(kulites, sosfilter): - ''' - Apply an sos filter to the given kulite collection, and return a new - kulites object. + """ + Apply an sos filter to the given kulite collection, and return a new kulites object. + Creates a deep copy in case you want to use this for comparing filtered vs unfiltered collections, and want to hang on to an unmodified version too - ''' + """ filtered_kulites = copy.deepcopy(kulites) for name, timeseries in kulites.data.items(): filtered_kulites.data[name] = signal.sosfilt(sosfilter, timeseries) @@ -446,14 +450,20 @@ def apply_filter(kulites, sosfilter): def compute_psd(kulites, w_len=1024): - """ Compute psds for kulite data - - Args: - kulites (Kulite) : Kulite or VKulite object - w_len (int) : window length - - Returns: - (Dict) of psds for each kulite (plus frequency) + """Compute PSDs for kulite data + + Parameters + ---------- + kulites : Kulites + Kulites object + w_len : int, optional + window length + + Returns + ------- + dict + Dictionary of PSDs for each kulite, with an additional entry "freq" for the + frequency bins """ data = dict() @@ -477,16 +487,22 @@ def compute_psd(kulites, w_len=1024): return data +# FIXME this is broken? def create_kulite_grid(tgts_file, kul_strs): - """ Generate a structGrid of kulite locations - Args: - tgts_file (str) : TGTS kulite location file - kul_strs (List) : kulite names to include in plots - - Returns: - kul_grid : kulite positions in cartesion and cylindrical form + """Generate a structured grid of kulite locations + + Parameters + ---------- + tgts_file : str + TGTS kulite location file + kul_strs : list + kulite names to include in plots + + Returns + ------- + kul_grid : StructGrid + kulite positions in cartesion and cylindrical form """ - # Load the kulite positions kulite_pos = kul_strs.read_tgts(tgts_file, kul_strs) @@ -505,30 +521,35 @@ def create_kulite_grid(tgts_file, kul_strs): def write_kulite_positions_p3d(output_file, tgts_file, kul_strs): - """ Generate a 1D plot3d grid with kulite locations - - Args: - output_file (str) : output p3d file - tgts_file (str) : TGTS kulite location file - kul_strs (List) : kulite names to include in plots - - Returns: - None + """Generate a 1D plot3d grid with kulite locations + + Parameters + ---------- + output_file : str + output p3d file + tgts_file : str + TGTS kulite location file + kul_strs : list + kulite names to include in plots """ kul_grid = create_kulite_grid(tgts_file, kul_strs) kul_grid.write_p3d(output_file) def read_targets_matrix(tgts_file, kul_strs): - """ Generate a list of kulite data - - Args: - tgts_file (str) : TGTS kulite location file - kul_strs (List) : kulite names to include in plots - - Returns: - List of Lists of kulite data - name, x, y, z, r, theta, i, j, k, diam, zone - + """Generate a list of kulite data + + Parameters + ---------- + tgts_file : str + TGTS kulite location file + kul_strs : list + kulite names to include in plots. + + Returns + ------- + list of list + Kulite data of the form: ``name, x, y, z, r, theta, i, j, k, diam, zone`` """ if kul_strs != 'all': mat = np.empty([len(kul_strs), 11], dtype=object) @@ -573,14 +594,19 @@ def read_targets_matrix(tgts_file, kul_strs): def create_pressure_array(kuls, data_type=np.float32): - """ Generate a numpy array with kulite pressure-time histories - - Args: - kuls (dict) : kulite names linked to array of pressure values - data_type (np.dtype) : data type of array elements - - Returns: - numpy array of kulite data + """Generate a numpy array with kulite pressure-time histories + + Parameters + ---------- + kuls : dict + kulite names linked to array of pressure values + data_type : numpy.dtype + data type of array elements + + Returns + ------- + np.ndarray + kulite data """ size = len(kuls.data) * len(kuls.data[list(kuls.data.keys())[0]]) leng = len(kuls.data[list(kuls.data.keys())[0]]) diff --git a/python/upsp/processing/p3d_utilities.py b/python/upsp/processing/p3d_utilities.py index 5c104bf..599a21b 100644 --- a/python/upsp/processing/p3d_utilities.py +++ b/python/upsp/processing/p3d_utilities.py @@ -1,8 +1,3 @@ -''' -pared-down mashup of Jessie Powell's grids.py and plot3d.py from UPSP-40. -used for grabbing of grid points and data stored in -p3d files (typically both P_ss and Cp_rms) -''' import math import numpy as np @@ -26,19 +21,17 @@ def to_cartesian(r,theta): return (y, z) class StructGrid: - """ Manage plot3d-style structured grid and write formatted to file - - Attributes: - x : array-like, x-position [N] - y : array-like, y-position [N] - z : array-like, z-position [N] - r : array-like, cylindrical coords [N] - theta : array-like, cylindrical coords [N] + """Manage plot3d-style structured grid and write formatted to file + + Attributes + ---------- + x, y, z : array_like + Cartesian coordinates [N] + r, theta : array_like + Cylindrical coordinates [N] """ def __init__(self): - """ Create empty StructGrid """ - self.sz = [] self.x = [] self.y = [] @@ -59,15 +52,13 @@ def reduce_grid(self,good_indices): def load_grid(self, grid_file): - """ Read a formatted p3d file - - Args: - grid_file (str) : formatted plot3d file + """ Read a formatted p3d file - Returns: - None + Parameters + ---------- + grid_file : str + formatted plot3d file """ - with open(grid_file, 'r') as f: sz = [] n_zones = int(f.readline()) @@ -94,13 +85,17 @@ def load_grid(self, grid_file): self.r, self.theta = to_cylindrical(self.y,self.z) def read_p3d_grid(filename): - """ Read an unformatted plot3d grid + """Read an unformatted plot3d grid - Args: - filename (str) : unformatted plot3d grid file + Parameters + ---------- + filename : str + unformatted plot3d grid file - Returns: - (StructGrid) + Returns + ------- + StructGrid + grid """ grid = StructGrid() @@ -144,13 +139,17 @@ def read_p3d_grid(filename): return grid def read_p3d_function(filename): - """ Read in the plot3d function file (first function) + """Read in the plot3d function file (first function) - Args: - filename (str) : plot3d binary function file + Parameters + ---------- + filename : str + plot3d binary function file - Return: - (np.array) first function in the file (Cp) + Returns + ------- + np.ndarray + first function in the file (Cp) """ # Open the file for reading diff --git a/python/upsp/processing/plot3d.py b/python/upsp/processing/plot3d.py index 629e345..91bd8f7 100644 --- a/python/upsp/processing/plot3d.py +++ b/python/upsp/processing/plot3d.py @@ -7,11 +7,15 @@ def read_p3d_grid(filename): """Read an unformatted plot3d grid - Args: - filename (str) : unformatted plot3d grid file - - Returns: - (StructGrid) + Parameters + ---------- + filename : str + unformatted plot3d grid file + + Returns + ------- + ~upsp.processing.grids.StructGrid + grid """ grid = grids.StructGrid() @@ -64,11 +68,15 @@ def read_p3d_grid(filename): def read_p3d_function(filename): """Read in the plot3d function file (first function) - Args: - filename (str) : plot3d binary function file + Parameters + ---------- + filename : str + plot3d binary function file - Return: - (np.array) first function in the file (Cp) + Returns + ------- + np.ndarray + first function in the file (Cp) """ # Open the file for reading diff --git a/python/upsp/processing/tree.py b/python/upsp/processing/tree.py index c413e20..1d7effa 100644 --- a/python/upsp/processing/tree.py +++ b/python/upsp/processing/tree.py @@ -4,11 +4,13 @@ import logging import os import re +import shutil import string -import subprocess import sys import textwrap +import numpy as np +import upsp from . import io log = logging.getLogger(__name__) @@ -59,7 +61,7 @@ def _camera_name(filename): return "cam%02d" % _camera_number_from_filename(filename) -_ADD_FIELD_EXE = "add_field" +_ADD_FIELD_EXE = shutil.which("add_field") _DEFAULT_TASK_EXE_NAME = "task.sh" @@ -78,6 +80,32 @@ class Error(Exception): pass +def ensure_unique(values: list[str], prefixes=None): + # Ensure a list of strings is unique. If not, add prefixes to non-unique elements. + # If prefixes is not specified, the list index (0, 1, 2, ...) is used. + prefixes = list(range(len(values))) if prefixes is None else prefixes + prefixes = [str(s) for s in prefixes] + + add_prefix = np.array([False, False, False, False], dtype=bool) + uniq_values, inv_idxs = np.unique(values, return_inverse=True) + for ii in range(len(uniq_values)): + duplicate_check = inv_idxs == ii + if np.count_nonzero(duplicate_check) > 1: + add_prefix[duplicate_check] = True + new_values = np.where( + add_prefix, list(map(''.join, zip(prefixes, values))), values + ) + return new_values + + +def copy_files_ensure_unique_names(src_filenames, dst_dir, src_prefixes=None): + src_basenames = [os.path.basename(fn) for fn in src_filenames] + dst_basenames = ensure_unique(src_basenames, prefixes=src_prefixes) + for src_fn, dst_bn in zip(src_filenames, dst_basenames): + dst_fn = os.path.join(dst_dir, dst_bn) + shutil.copy(src_fn, dst_fn) + print("Copied:", src_fn, '->', dst_fn) + # Create a processing tree for uPSP raw data. # # The processing tree is a hierarchy to contain @@ -125,6 +153,20 @@ def create( ctx_filename = os.path.join(dirname, "context.json") io.json_write_or_die(cfg, ctx_filename, indent=2) + # Write the origin JSON files to the config directory + # ... if their basenames aren't unique, MAKE them unique + # (to avoid overwriting eachother. It's possible they have + # the same basename but are located in different folders) + _cfgs = { + "data-": data_config_filename, "user-": user_config_filename, + "proc-": proc_config_filename, "plot-": plot_config_filename + } + copy_files_ensure_unique_names( + list(_cfgs.values()), + _configuration_path(cfg), + src_prefixes=list(_cfgs.keys()) + ) + def _resolve_nas_config(nas: dict): # Resolve NAS launch parameters for each pipeline step @@ -247,155 +289,51 @@ def _validate_re(s): return proc -def _assert_valid_code_version(cfg): - _launcher_env_sh(cfg) - - -# Cache the outputs to minimize the number of times a subprocess is launched -_git_top_level_dir_cache = {} -_code_version_str_cache = {} -_launcher_env_sh_cache = {} - - -def _git_top_level_dir(path): - """Return top-level directory of git repo containing 'path' - Returns '' if path is not a child of a valid git repository. - """ - realpath = os.path.realpath(path) - if realpath in _git_top_level_dir_cache: - return _git_top_level_dir_cache[realpath] - try: - if os.path.isdir(realpath): - cwd = realpath - else: - cwd = os.path.dirname(realpath) - result = subprocess.run( - ["git", "rev-parse", "--show-toplevel"], - cwd=cwd, - capture_output=True, - text=True, - ) - if result.returncode != 0: - return "" - else: - cachepath = os.path.realpath(result.stdout.strip()) - _git_top_level_dir_cache[realpath] = cachepath - return cachepath - except FileNotFoundError: # cwd=path fails, ! path.isdir... - return "" - except PermissionError: # cwd=path fails, user cannot 'cd' - return "" - - -def _install_dir(): - return os.environ.get("_UPSP_RUNTIME_ROOT", "") - - -def _code_version_str(): - """Return uPSP code version for this module file""" - if __file__ in _code_version_str_cache: - return _code_version_str_cache[__file__] - gtl = _git_top_level_dir(__file__) - if gtl: - version_exe = "%s/build/version" % gtl - else: - version_exe = "%s/version" % _install_dir() - result = subprocess.run([version_exe], capture_output=True, text=True) - if result.returncode != 0: - return "" - else: - version = result.stdout.strip() - _code_version_str_cache[__file__] = version - return version - - def _launcher_env_sh(cfg: dict): """Returns string containing sh environment setup commands - The launcher environment activates the appropriate - uPSP software version using the following strategy: - - If this Python module lives under a valid git repository, - then assume we'd like to run using a development build of - the software. By default, binaries are built in the "build/" - subdirectory, scripts are under the "scripts/" subdirectory, - and python modules are under the "python/" subdirectory. - - Otherwise, we assume the user has activated an installed version - of the uPSP software (either via the NAS module system or by - directly sourcing the 'activate.sh' script shipped with the install). - In this case, the _UPSP_EXEC_ROOT environment variable is populated - with the path to the installed software. + - Load any required system libraries that aren't available + by default... for example, system-provided MPI libraries. + + - Prefix the PATH with the directory of our current python + interpreter. Ensures any scripts keying off "/usr/bin/env python" + resolve the correct interpreter at runtime. """ - code_version = _code_version_str() - if code_version in _launcher_env_sh_cache: - return _launcher_env_sh_cache[code_version] - - dev_dir = _git_top_level_dir(__file__) - install_dir = _install_dir() - - code_version_is_dev = os.path.isdir(dev_dir) - code_version_is_install = os.path.isdir(install_dir) - env_sh_lines = [ "source /usr/local/lib/global.profile", "module purge", "module load mpi-hpe/mpt.2.25", + "export PATH=%s:$PATH" % (os.path.dirname(shutil.which("python"))) ] - - if code_version_is_dev: - log.info("Launchers will use uPSP software build from '%s'", dev_dir) - env_sh_lines.append("source %s/scripts/activate.sh" % dev_dir) - dev_python_path = os.path.join(dev_dir, "python") - dev_build_path = os.path.join(dev_dir, "build") - dev_scripts_path = os.path.join(dev_dir, "scripts") - env_sh_lines.append("export PYTHONPATH=%s:$PYTHONPATH" % dev_python_path) - env_sh_lines.append("export PATH=%s:$PATH" % dev_build_path) - env_sh_lines.append("export PATH=%s:$PATH" % dev_scripts_path) - elif code_version_is_install: - log.info("Launchers will use uPSP software install in '%s'", install_dir) - env_sh_lines.append("source %s/activate.sh" % install_dir) - else: - raise Error( - "\n".join( - [ - "Invalid code_version '%s'" % code_version, - "Must be one of the following:", - "1. A module from /nobackupp11/uPSP/modulefiles, e.g., 'upsp/v2.0'", - "2. A directory containing an install of the uPSP software", - "3. A local git repository working directory (for developers)", - ] - ) - ) - s = "\n".join(env_sh_lines) - _launcher_env_sh_cache[code_version] = s - return s + return "\n".join(env_sh_lines) def _create_qsub_step_launcher(cfg: dict): filename = _launchers_path(cfg, "qsub-step") env_sh = _launcher_env_sh(cfg) exe_sh = textwrap.dedent( - """ - SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + f""" + SCRIPT_DIR=$( cd -- "$( dirname -- "${{BASH_SOURCE[0]}}" )" &> /dev/null && pwd ) STEP_LAUNCHER_FILENAME=$(realpath $1) STEP_LAUNCHER_BASENAME=$(basename $STEP_LAUNCHER_FILENAME) - STEP_NAME="$(echo ${STEP_LAUNCHER_BASENAME} | cut -d+ -f2)" + STEP_NAME="$(echo ${{STEP_LAUNCHER_BASENAME}} | cut -d+ -f2)" D=$SCRIPT_DIR/../04_processing/01_exec/$STEP_NAME ALL_DATAPOINTS=($(ls -1 $D)) - INP_DATAPOINTS=("${@:2}") + INP_DATAPOINTS=("${{@:2}}") SEL_DATAPOINTS=() - if (( ${#INP_DATAPOINTS[@]} )); then - SEL_DATAPOINTS+=( "${INP_DATAPOINTS[@]}" ) + if (( ${{#INP_DATAPOINTS[@]}} )); then + SEL_DATAPOINTS+=( "${{INP_DATAPOINTS[@]}}" ) else - SEL_DATAPOINTS+=( "${ALL_DATAPOINTS[@]}" ) + SEL_DATAPOINTS+=( "${{ALL_DATAPOINTS[@]}}" ) fi readarray -t UPSP_QSUB_ARGS_OUT < \ - <(upsp-qsub-args "$SCRIPT_DIR/.." $STEP_NAME ${SEL_DATAPOINTS[@]}) - NLINES="${#UPSP_QSUB_ARGS_OUT[@]}" + <({shutil.which("upsp-qsub-args")} "$SCRIPT_DIR/.." $STEP_NAME ${{SEL_DATAPOINTS[@]}}) + NLINES="${{#UPSP_QSUB_ARGS_OUT[@]}}" IDX=0 while [ $IDX -lt $NLINES ]; do - THIS_QSUB_ARGS=${UPSP_QSUB_ARGS_OUT[$IDX]} + THIS_QSUB_ARGS=${{UPSP_QSUB_ARGS_OUT[$IDX]}} IDX=$(( $IDX + 1 )) - THIS_DATAPOINTS=${UPSP_QSUB_ARGS_OUT[$IDX]} + THIS_DATAPOINTS=${{UPSP_QSUB_ARGS_OUT[$IDX]}} IDX=$(( $IDX + 1 )) CMD="$STEP_LAUNCHER_FILENAME $THIS_DATAPOINTS" EX="qsub $THIS_QSUB_ARGS -- $CMD" @@ -517,14 +455,14 @@ def _create_pbs_file(cfg: dict, step_name: str, run_number: str, exe_name: str): "# directory of the process. We want these core dumps to be written", "# to the process logs directory.", "cd %s" % _app_logs_path(cfg, step_name, run_number), - "mpiexec psp_process \\", + "mpiexec %s \\" % (shutil.which("psp_process"),), " -cutoff_x_max=%s \\" % (pcfg["cutoff_x_max"]), " -input_file=%s \\" % (_inp_filename(cfg, step_name, run_number)), " -h5_out=%s \\" % (h5_filename), " -paint_cal=%s \\" % (this_run["paint_calibration_file"]), " -steady_p3d=%s \\" % (this_run["steady_psp_file"]), " -frames=%s \\" % (pcfg["number_frames"]), - " -code_version=%s \\" % (_code_version_str()), + " -code_version=%s \\" % (upsp.__version__), " > %s 2>&1\n" % (log_filename), ] exe_sh = "\n".join(exe_rows) @@ -548,7 +486,7 @@ def _create_input_file( frame_rate = 10000 fstop = 2.8 input_rows = [ - "%%Version %s" % _code_version_str(), + "%%Version %s" % upsp.__version__, "%%Date_Created: %s" % cfg["__meta__"]["__date__"], "", "@general", @@ -618,7 +556,7 @@ def _configuration_name(cfg: dict): def _version_configuration_name(cfg: dict): # Combine software version and configuration name into a single string. - return "+".join([_code_version_str(), _configuration_name(cfg)]) + return "+".join([upsp.__version__, _configuration_name(cfg)]) def _root_path(cfg: dict, *args): @@ -706,7 +644,9 @@ def _create_task_render_images(cfg: dict, step_name: str, run_number: str): # TODO This file will be overwritten every time this is run per-datapoint. # Inefficient but not a huge deal. cfg_filename = _create_plotting_config_file(cfg, "render-images") - exe_sh = "upsp-plotting render-images %s %s" % (cfg_filename, run_number) + exe_sh = "%s render-images %s %s" % ( + shutil.which("upsp-plotting"), cfg_filename, run_number + ) env_sh = _launcher_env_sh(cfg) exe_filename = _app_exec_path(cfg, step_name, run_number, _DEFAULT_TASK_EXE_NAME) _create_dir_and_log(_app_logs_path(cfg, step_name, run_number)) @@ -754,7 +694,7 @@ def _first_frames_info(cfg: dict, run_number: str): log_filename = _app_logs_path(cfg, step_name, run_number, "%s.out" % src_name) exe_sh_lines.extend( [ - "upsp-extract-frames \\", + "%s \\" % (shutil.which("upsp-extract-frames"),), " -input=%s \\" % src_filename, " -output=%s.%s \\" % (img_prefix, img_ext), " -start=%d \\" % (img_frame_number), @@ -789,7 +729,7 @@ def _create_task_external_calibration( "# directory of the process. We want core dumps to be written", "# to the process logs directory.", "cd %s" % _app_logs_path(cfg, step_name, run_number), - "upsp-external-calibration \\", + "%s \\" % (shutil.which("upsp-external-calibration"),), " --tgts %s \\" % this_run["targets_file"], " --grd %s \\" % this_run["grid_file"], " --wtd %s \\" % this_run["wtd_file"], @@ -828,29 +768,6 @@ def _create_task_psp_process( return None, ["run-psp-process.pbs", "run-add-field.sh"] -def _create_step_unity_export(cfg: dict): - step_name = "unity-export" - log_filename = _app_logs_path(cfg, step_name, "%s.out" % step_name) - exe_filename = _launchers_path(cfg, "run-%s" % step_name) - out_dir = _app_products_path(cfg, step_name) - exe_sh_lines = [ - "# By default, core dumps are written out to the current working", - "# directory of the process. We want core dumps to be written", - "# to the process logs directory.", - "cd %s" % _app_logs_path(cfg, step_name), - "upsp-unity-export \\", - " --pipeline_dir '%s' \\" % _root_path(cfg), - " --output_dir '%s' \\" % out_dir, - " > %s 2>&1\n" % log_filename, - ] - exe_sh = "\n".join(exe_sh_lines) - env_sh = _launcher_env_sh(cfg) - _create_dir_and_log(os.path.dirname(log_filename)) - _create_dir_and_log(out_dir) - _create_dir_and_log(_app_logs_path(cfg, step_name, "jobs")) - _create_launcher(exe_filename, exe_sh=exe_sh, env_sh=env_sh) - - def _create_per_datapoint_step(cfg: dict, step_name: str, fn, inputs=None): output = {} for run_number, _ in cfg["datapoints"].items(): @@ -867,7 +784,6 @@ def _create_per_datapoint_step(cfg: dict, step_name: str, fn, inputs=None): def _create(cfg: dict): - _assert_valid_code_version(cfg) _create_root_dir(cfg) _create_dir_and_log(_configuration_path(cfg)) _create_dir_and_log(_launchers_path(cfg)) @@ -892,19 +808,20 @@ def _create(cfg: dict): {dp: [[v], {}] for dp, v in external_calibration_info.items()}, ) - # TODO: pull scalar filenames as output fom psp_process step, similar - # to first_frame_info and external_calibration_info. - _create_per_datapoint_step( - cfg, - "render-images", - _create_task_render_images, - ) + # TODO: re-enable this task, which is a step to render views of the + # model colored by certain scalars output by psp_process (the + # demo application made use of the NAS-provided TecPlot install, + # but has not been made general enough to include in the batch + # processing autogeneration tools). We will likely want to rewrite + # the step to make use of the PyTecplot interface, which is a + # lightweight Python package that connects to a Tecplot backend + # provided it is running and then provides an API for plotting. + # _create_per_datapoint_step( + # cfg, + # "render-images", + # _create_task_render_images, + # ) _create_qsub_step_launcher(cfg) - # TODO: unity-export operates on 1+ datapoints at a time (as opposed to - # other steps that are all per-datapoint), so it fit into the - # current per-datapoint templated launch framework. - _create_step_unity_export(cfg) - return _root_path(cfg) diff --git a/python/upsp/processing/unity_conversions.py b/python/upsp/processing/unity_conversions.py deleted file mode 100644 index 7fdd33e..0000000 --- a/python/upsp/processing/unity_conversions.py +++ /dev/null @@ -1,75 +0,0 @@ -import numpy as np -import os -from pathlib import Path - -from . import io -from . import plot3d -from . import p3d_conversions - - -# NOTE: Unity uses a left hand coordinate system, and knows that obj files use -# the right-hand coord convention. It will therefore negate the x values -# of the obj file to convert the handedness open loading the obj mesh -# into the 3D space. Between Unity and wind tunnel coord systems, it is -# the z axis that is inverted; therefore to make Unity's representation -# of the obj mesh align with the experimental setup, the obj file needs -# both z and x axes to be negated. -def convert_to_unity_obj(p3d_file, obj_file, zones_map_file): - grd = plot3d.read_p3d_grid(p3d_file) - grd.write_zones_mapping(zones_map_file) - tris = p3d_to_unity_obj_triangles(grd) - - v = tris["vertices"] - f = tris["faces"] - - obj_path = os.path.dirname(obj_file) - Path(obj_path).mkdir(parents=True, exist_ok=True) - - # Write obj file by listing vertices, vertex normals then faces - with open(obj_file, "w+") as fp: - fp.write("o Mesh_0\n") - vList = map(lambda x: x + "\n", v) - fp.writelines(vList) - fp.write("usemtl None\n") - fp.write("s off\n") - fList = map(lambda x: x + "\n", f) - fp.writelines(fList) - - -def p3d_to_unity_obj_triangles(grd): - obj = p3d_conversions.p3d_to_obj_triangles(grd) - - vertices = obj["vertices"] - unity_vertices = [] - for vertex in vertices: - line = vertex.split(" ") - # NOTE: the x- and z-values are negated to be Unity-compatible - unity_line = ( - line[0] - + " " - + str(-1.0 * float(line[1])) - + " " - + line[2] - + " " - + str(-1.0 * float(line[3])) - ) - unity_vertices.append(unity_line) - - obj["vertices"] = unity_vertices - return obj - - -# NOTE: Unity uses a left hand coordinate system, so all data files that are -# written for Unity consumption must negate the z axis in order to align -# with the right handed experimental system setup -def convert_kulite_locations(kul_mat): - unity_kul_mat = np.empty([len(kul_mat), len(kul_mat[0])], dtype=object) - for i in range(0, len(kul_mat)): - row = kul_mat[i] - unity_z = str(-1.0 * float(row[3])) - unity_k = str(-1.0 * float(row[8])) - unity_row = [row[0], row[1], row[2], unity_z, row[4], row[5], row[6], - row[7], unity_k, row[9], row[10]] - unity_kul_mat[i] = unity_row - - return unity_kul_mat diff --git a/python/upsp/processing/unity_stats.py b/python/upsp/processing/unity_stats.py deleted file mode 100644 index bd561b4..0000000 --- a/python/upsp/processing/unity_stats.py +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env python -# -# Read in the data files needed by Unity. Calculate and write out -# relevant statistics for visualization. -import glob -import logging -import os -import struct -import math -from . import io - -import upsp.processing.context as context # noqa - -log = logging.getLogger(__name__) - - -def write_unity_dataset_with_stats( - vertex_file, src_dataset_name, dst_dataset_name, - max_number_frames=100 -): - """Visualization: data and stats files from binary data and write to disk""" - - FLOAT_SIZE_BYTES = 4 - - # 1. Figure out the number of model nodes from vertex file size - number_nodes = os.path.getsize(vertex_file) / FLOAT_SIZE_BYTES - log.info("Inspected %s to find %d model nodes", vertex_file, number_nodes) - - # 2. Given a number of frames, compute total number of bytes we want - total_number_bytes = os.path.getsize(src_dataset_name) - total_number_frames = total_number_bytes / (number_nodes * FLOAT_SIZE_BYTES) - number_frames = max_number_frames \ - if total_number_frames > max_number_frames \ - else total_number_frames - number_bytes = int(number_frames * (number_nodes * FLOAT_SIZE_BYTES)) - - log.info("[%d frames] x [%d nodes] x [%d bytes per float] = [%d bytes] ~ [%d MB]", - number_frames, number_nodes, FLOAT_SIZE_BYTES, number_bytes, - int(number_bytes / 1024.0 / 1024.0)) - - # 3. Read, process and dump those bytes to a new file - input_filename = glob.glob(src_dataset_name)[0] - output_data_filename = dst_dataset_name + ".bytes" - output_stats_filename = dst_dataset_name + "_stats.txt" - - log.info("Reading %d bytes from %s, writing to %s", number_bytes, - input_filename, output_data_filename) - - # chunk_size = 100 * 1024 * 1024 - nchunks, remainder = divmod(number_bytes, FLOAT_SIZE_BYTES) - chunks = [FLOAT_SIZE_BYTES] * nchunks - if remainder != 0: - log.warning("Extra remainder data size: " + remainder) - - max = 0 - min = float("inf") - total = 0 - number_floats = 0 - mean_est = 0 - M2 = 0 - - # Iterate once to get maximum, minimum and total value of raw data - # and write it out to the output file - with open(input_filename, "rb") as fi: - with open(output_data_filename, "wb") as fdo: - for ch in chunks: - byte_val = fi.read(ch) - float_val = struct.unpack("f", byte_val)[0] - - if float_val < min: - min = float_val - if float_val > max: - max = float_val - - if not math.isnan(float_val): - # log.info("float_val: %f" % float_val) - total = total + float_val - number_floats = number_floats + 1 - - (mean_est, M2) = update(mean_est, M2, number_floats, - float_val) - - fdo.write(byte_val) - - if number_floats == 0: - os.remove(output_data_filename) - return - - stdev = math.sqrt(M2 / number_floats) - mean = total / number_floats - - # Write statistics results to text file - with open(output_stats_filename, "w") as fso: - fso.write(str(max) + "\n") - fso.write(str(min) + "\n") - fso.write(str(mean) + "\n") - fso.write(str(stdev) + "\n") - - os.chmod(output_data_filename, io._CHMOD_URW_GR__O__) - os.chmod(output_stats_filename, io._CHMOD_URW_GR__O__) - - -def update(mean, M2, count, newValue): - delta = newValue - mean - mean += delta / count - delta2 = newValue - mean - M2 += delta * delta2 - return (mean, M2) diff --git a/python/upsp/target_localization/blob_detector_methods.py b/python/upsp/target_localization/blob_detector_methods.py index 1752921..cabc42c 100644 --- a/python/upsp/target_localization/blob_detector_methods.py +++ b/python/upsp/target_localization/blob_detector_methods.py @@ -3,12 +3,7 @@ import os import sys -current_dir = os.path.dirname(__file__) -parent_dir = os.path.dirname(current_dir) -cam_cal_utils = os.path.join(parent_dir, 'cam_cal_utils') -sys.path.append(cam_cal_utils) - -import img_utils +from upsp.cam_cal_utils import img_utils #--------------------------------------------------------------------------------------- @@ -50,39 +45,30 @@ def blob_func(detection_method, decide_method=None): """Returns a function that acts as a wrapper around openCV's blob detection function to act as a target center localizer - + Parameters ---------- - detection_method : str + detection_method : string String must be a key in the global dict target_detectors. The key selects the blob detection parameters to use - decide_method : optional {'biggest', 'smallest', 'first', 'last', 'random', None}, default=None - decision method if more than 1 blob is detected in the image. Biggest and - smallest selects he largest and smallest (respectfully) based on the keypoint + decide_method : {'biggest', 'smallest', 'first', 'last', 'random', None}, optional + decision method if more than 1 blob is detected in the image. Biggest and + smallest selects the largest and smallest (respectfully) based on the keypoint size parameter. First and last select the first and last (respectfully) based on - the order returned by the blob detector. Random selects a random blob. None - uses the default method, which is last + the order returned by the blob detector. Random selects a random blob. None uses + the default method, which is last Returns - Function + ------- + callable Blob detector localization function - Wrapper around a blob detector to act as a - localization function + localization function. The function has the signature:: - Parameters - ---------- - img : np.ndarray, 2D, uint8 or uint16 - Image containing one target - return_keypoint : optional, boolean. - If True returns openCV keypoint object. If False, returns only estimated - target center location - - Returns - ---------- - Tuple if return_keypoint is False, and keypoint is return_keypoint is True - Tuple's first index is the center. The center is a length 2 tuple of floats. - The first item of the center is the x coordiante and the second item is the - y coordinate + func(img, target_type=None, return_keypoint=False) -> keypoint + where ``img`` is a 2D image array containing one target, ``target_type`` does + nothing, and ``return_keypoint`` specifies whether the keypoint itself is + returned (``True``) or just the center position (x, y). """ if (detection_method in list(target_detectors.keys())): diff --git a/python/upsp/target_localization/gaussian_fitting_methods.py b/python/upsp/target_localization/gaussian_fitting_methods.py index 1cf86b0..f480ee4 100644 --- a/python/upsp/target_localization/gaussian_fitting_methods.py +++ b/python/upsp/target_localization/gaussian_fitting_methods.py @@ -1,12 +1,12 @@ import matplotlib.pyplot as plt import scipy.optimize as opt import numpy as np -import cv2 import warnings -from blob_detector_methods import blob_func - from scipy.optimize import OptimizeWarning +from upsp.target_localization.blob_detector_methods import blob_func + + np.set_printoptions(linewidth=180, precision=8, threshold=np.inf) super_gauss_power_upper_bound = 20 @@ -17,24 +17,19 @@ def super_twoD_Gaussian(pt, amplitude, xo, yo, sx, sy, theta, offset, p): """Returns the value of a super 2D Gaussian function at the given point - - Ref: https://en.wikipedia.org/wiki/Gaussian_function#Higher-order_Gaussian_or_super-Gaussian_function - + + See https://en.wikipedia.org/wiki/Gaussian_function#Higher-order_Gaussian_or_super-Gaussian_function + Parameters ---------- - pt : np.ndarray (N, 2) or (2, ), float - 2D Gaussian function point input - Length of super_twoD_Gaussian is equal to length of pt + pt : np.ndarray, shape (N, 2), float + 2D Gaussian function point input. Length of output is equal to length of pt amplitude : float Amplitude of super 2D Gaussian - xo : float - x mean of super 2D Gaussian (x center of target) - yo : float - y mean of super 2D Gaussian (y center of target) - sx : float - x standard deviation of super 2D Gaussian - sy : float - y standard deviation of super 2D Gaussian + xo, yo : float + mean position of super 2D Gaussian (center of target) + sx, sy: float + standard deviations of super 2D Gaussian theta : float Angle of super 2D Gaussian ellipse offset : float @@ -43,10 +38,10 @@ def super_twoD_Gaussian(pt, amplitude, xo, yo, sx, sy, theta, offset, p): Power of super 2D Gaussian. Higher power is more platykurtic Returns - ---------- - float or (N,) np.ndarray - Returns value of super 2D Gaussian function with given parameters at given point(s). - Length of return is equal to length of the pt input + ------- + float or np.ndarray, shape (N, 1) + Value of super 2D Gaussian function with given parameters at given point(s). + Length of return is equal to length of the pt input """ # Seperate the point into x an y @@ -66,7 +61,7 @@ def super_twoD_Gaussian(pt, amplitude, xo, yo, sx, sy, theta, offset, p): a = (cos_sq / (2 * sx_sq)) + (sin_sq / (2 * sy_sq)) b = - (sin2 / (4 * sx_sq)) + (sin2 / (4 * sy_sq)) c = (sin_sq / (2 * sx_sq)) + (cos_sq / (2 * sy_sq)) - + # Calculate the Gaussian value and scale by the amplitude quad = a * (xval**2) + 2 * b * (xval * yval) + c * (yval**2) @@ -81,84 +76,52 @@ def super_twoD_Gaussian(pt, amplitude, xo, yo, sx, sy, theta, offset, p): def twoD_Gaussian(pt, amplitude, xo, yo, sx, sy, theta, offset): - """Wrapper to call super_twoD_Gaussian with a power of 1 - - Parameters - ---------- - pt : np.ndarray (N, 2) or (2, ), float - 2D Gaussian function point input - Length of super_twoD_Gaussian is equal to length of pt - amplitude : float - Amplitude of super 2D Gaussian - xo : float - x mean of super 2D Gaussian (x center of target) - yo : float - y mean of super 2D Gaussian (y center of target) - sx : float - x standard deviation of super 2D Gaussian - sy : float - y standard deviation of super 2D Gaussian - theta : float - Angle of super 2D Gaussian ellipse - offset : float - Floor of super 2D Gaussian - - Returns - ---------- - float or (N,) np.ndarray - Returns value of super 2D Gaussian function with given parameters at given point(s). - Length of return is equal to length of the pt input - + """Wrapper to call :func:`super_twoD_Gaussian` with a power of 1 + See Also ---------- super_twoD_Gaussian : Returns the value of a super 2D Gaussian function at the given point """ - return super_twoD_Gaussian(pt, amplitude, xo, yo, sx, sy, theta, offset, 1) def super_twoD_Gaussian_nobounds(pt, ln_amplitude, xo, yo, ln_sx, ln_sy, theta, offset, ln_p): - """Wrapper to call super_twoD_Gaussian with no bounds on inputs - - For super_twoD_Gaussian, the function does not make physical sense if amplitude, the - standard deviations, or power are non-positive. (power must be > 1 for the - function to be platykurtic) + """Wrapper to call :func:`super_twoD_Gaussian` with no bounds on inputs - super_twoD_Gaussian_nobounds is for optimizers that require no bounds. The natural - logarithm of those values are passed to this function, and those values are passed - through exp() before being passed to super_twoD_Gaussian. Those parameters can then - vary from -inf to +inf, and will be mapped to (0, np.inf] + For :func:`super_twoD_Gaussian`, the function does not make physical sense if + amplitude, the standard deviations, or power are non-positive. (power must be > 1 + for the function to be platykurtic) + + :func:`super_twoD_Gaussian_nobounds` is for optimizers that require no bounds. The + natural logarithm of those values are passed to this function, and those values are + passed through ``exp()`` before being passed to super_twoD_Gaussian. Those + parameters can then vary from -inf to +inf, and will be mapped to (0, np.inf] Parameters ---------- - pt : np.ndarray (N, 2) or (2, ), float - 2D Gaussian function point input - Length of super_twoD_Gaussian is equal to length of pt + pt : np.ndarray, shape (N, 2), float + 2D Gaussian function point input. Length of output is equal to length of pt ln_amplitude : float Natural log of the amplitude of super 2D Gaussian - xo : float - x mean of super 2D Gaussian (x center of target) - yo : float - y mean of super 2D Gaussian (y center of target) - ln_sx : float - Natural log of the x standard deviation of super 2D Gaussian - ln_sy : float - Natural log of the y standard deviation of super 2D Gaussian + xo, yo : float + mean position of super 2D Gaussian (center of target) + ln_sx, ln_sy: float + Natural log of standard deviations of super 2D Gaussian theta : float Angle of super 2D Gaussian ellipse offset : float Floor of super 2D Gaussian ln_p : float - Natural log of the power of super 2D Gaussian minus 1. p = exp(ln_p) + 1 - Higher power is more platykurtic. + Natural log of the power of super 2D Gaussian minus 1. ``p = exp(ln_p) + 1`` + Higher power is more platykurtic. Returns ---------- - float or (N,) np.ndarray - Returns value of super 2D Gaussian function with given parameters at given point(s). - Length of return is equal to length of the pt input - + float or np.ndarray, shape (N, 1) + Value of super 2D Gaussian function with given parameters at given point(s). + Length of return is equal to length of the pt input + See Also ---------- super_twoD_Gaussian : @@ -171,61 +134,57 @@ def super_twoD_Gaussian_nobounds(pt, ln_amplitude, xo, yo, ln_sx, ln_sy, theta, # Gaussian Target Localization -def gauss_fitter_func(fit, get_rms=False, curve_fit_kwds=None, bit_depth=12, debug=False): +def gauss_fitter_func(fit, get_rms=False, curve_fit_kwds=None, debug=False): """Returns a function that acts as a wrapper around a 2D Gaussian function to act as a target center localizer - + Parameters ---------- - fit : {'super', 'normal'} - Specifies fitting function to use, super or standard 2D Gaussian - get_rms - optional, boolean - If True, adds rms to output. If False does not - curve_fit_kwds - optional, dict or None, default=None - Keywords for scipy curve fit optimizer. If None, the following keywords are - used: {'maxfev' : 25000, 'ftol' : 1e-4, 'xtol' : 1e-4} - debug - optional, boolean - If True, adds full optimization parameters and covariance matrix to output. - If False does not + fit : {'super', 'normal'} + Specifies fitting function to use, super or standard 2D Gaussian + get_rms : bool, optional + If True, adds rms to output. If False does not + curve_fit_kwds : dict, optional + Keyword arguments for scipy curve fit optimizer. If None, the following keywords + are used: ``{'maxfev' : 25000, 'ftol' : 1e-4, 'xtol' : 1e-4}`` + debug : bool, optional + If True, adds full optimization parameters and covariance matrix to output. If + False does not Returns - ---------- - Function + ------- + callable 2D Gaussian Fitting Function - returns center of target given an image - + Wrapper around optimizer of 2D Gaussian functions to act as target localization function. - Parameters - ---------- - img : np.ndarray, 2 dimensional (grayscale), unint8 or uint16 - Image with one target. Typically small - target_type : optional, {'dot', 'kulite', None}, default=None - Type of target in image. Used to initialize target finding parameters - keypoint : optional, blob detector keypoint or None, default=None - If given a blob detector keypoint, initializes target finding parameters - from the blob detector keypoint parameters. If None uses default parameters - and assumes starting position for target is dead center of image - img_offset : optional, stuple or None, default=None - Since the blob detector keypoint is from the whole image, and the localizer - is a local function, the offset tells where the blob detector keypoint is in - the local image. Additionall, output is offset by img_offset to give the - target's location in the whole image. If None, do none of that. + Arguments: - Returns - ---------- - tuple - If gauss_fitter_func's debug input is False, return tuple contains center as - first item. Center is a tuple of 2 floats denoting the target x and y - position. If gauss_fitter_func's get_rms is True, return tuple contains - rms error as second item. If get_rms is False, return tuple has no second - item. - If gauss_fitter_func's debug input is True, return tuple contains an inner - tuple as the first itme. The inner tuple contains the full set of optimizer - parameters as the first item, and the covariance matrix as the second item. - If gauss_fitter_func's get_rms is True, return tuple contains rms error as - the second item. If get_rms is False, return tuple has no second item. - If there is an error in the optimization, all -1's will be returned for the + - ``img`` (:class:`numpy.ndarray`, 2 dimensional (grayscale), unint8) : Image + with one target. Typically small + - ``center`` (tuple of length 2, float): Initialization for target localization + - ``target_type`` (``{'dot', 'kulite', None}``): Type of target in image. Used + to initialize target finding parameters + - ``img_offset`` (tuple or None): Since the center location is from the whole + image, and the localizer is a local function, the offset tells where the + center location is in the local image. Additionally, output is offset by + `img_offset` to give the target's location in the whole image. If None, do + none of that. + + Returns: + + If `debug` input is False, return tuple contains center as first item. + Center is a tuple of 2 floats denoting the target x and y position. + + If `get_rms` is True, return tuple contains rms error as second item. If + `get_rms` is False, return tuple has no second item. + + If `debug` input is True, return tuple contains an inner tuple as the first + item. The inner tuple contains the full set of optimizer parameters as the + first item, and the covariance matrix as the second item. + + If there is an error in the optimization, all None's will be returned for the center, optimizer parameters, covatiance matrix, and rms as needed. """ @@ -242,11 +201,11 @@ def gauss_fitter_func(fit, get_rms=False, curve_fit_kwds=None, bit_depth=12, deb warnings.filterwarnings("ignore", category=OptimizeWarning, message=".*Covariance of the parameters could not be estimated*.") if curve_fit_kwds is None: - curve_fit_kwds = {'maxfev' : 25000, 'ftol' : 1e-4, 'xtol' : 1e-4} + curve_fit_kwds = {'maxfev': 25000, 'ftol': 1e-4, 'xtol': 1e-4} - def fit_gaussian(img, target_type=None, keypoint=None, img_offset=None): + def fit_gaussian(img, center=None, target_type=None, img_offset=None): """Returns center of target given an image - + Wrapper around optimizer of 2D Gaussian functions to act as target localization function. @@ -254,16 +213,15 @@ def fit_gaussian(img, target_type=None, keypoint=None, img_offset=None): ---------- img : np.ndarray, 2 dimensional (grayscale), unint8 Image with one target. Typically small + center : tuple of length 2, float or None. Optional, default=None + Initialization for target localization. If None, blob detection is performed + to initalize the target position target_type : {'dot', 'kulite', None}, default=None Type of target in image. Used to initialize target finding parameters - keypoint : blob detector keypoint or None, default=None - If given a blob detector keypoint, initializes target finding parameters - from the blob detector keypoint parameters. If None uses default parameters - and assumes starting position for target is dead center of image img_offset : tuple or None, default=None - Since the blob detector keypoint is from the whole image, and the localizer - is a local function, the offset tells where the blob detector keypoint is in - the local image. Additionall, output is offset by img_offset to give the + Since the center location is from the whole image, and the localizer + is a local function, the offset tells where the center location is in + the local image. Additionally, output is offset by img_offset to give the target's location in the whole image. If None, do none of that. Returns @@ -279,34 +237,34 @@ def fit_gaussian(img, target_type=None, keypoint=None, img_offset=None): parameters as the first item, and the covariance matrix as the second item. If gauss_fitter_func's get_rms is True, return tuple contains rms error as the second item. If get_rms is False, return tuple has no second item. - If there is an error in the optimization, all -1's will be returned for the + If there is an error in the optimization, all None's will be returned for the center, optimizer parameters, covatiance matrix, and rms as needed. """ - # If a keypoint was not given as input, find one with a blob detector - if keypoint is None: + if center is None: keypoint = blob_detector(img, return_keypoint=True) # If the blob detector didn't find a keypoint, initialize one based # on a reasonable guess if keypoint is None: - keypoint = cv2.KeyPoint((img.shape[1] - 1) / 2, (img.shape[0] - 1) / 2, - _size=4.4) + center = ((img.shape[1] - 1) / 2, (img.shape[0] - 1) / 2) + else: + center = keypoint.pt - # Populate initial guess information based on the keypoint input - center_x = keypoint.pt[0] - center_y = keypoint.pt[1] + # Populate initial guess information based on the center input + center_x = center[0] + center_y = center[1] - # img_offset serves to help if the input keypoint is in different coordinates - # than the input image (as is the case with cropping) + # img_offset serves to help if the center input than the input image + # (as is the case with cropping) if img_offset is not None: center_x -= img_offset[0] center_y -= img_offset[1] # Populate the initial guess for size_x and size_y based on target type if target_type is None: - size_x = keypoint.size / 4 - size_y = keypoint.size / 4 + size_x = 0.75 + size_y = 0.75 else: if (target_type == 'kulite'): size_x = 0.8 @@ -326,7 +284,7 @@ def fit_gaussian(img, target_type=None, keypoint=None, img_offset=None): # so they are now bright dots relative to the background (raises in # intensity relative to background) img = -img - + # Create the grid of pixel locations as a two 2D numpy arrays x = np.arange(0, img.shape[1]) y = np.arange(0, img.shape[0]) @@ -341,13 +299,13 @@ def fit_gaussian(img, target_type=None, keypoint=None, img_offset=None): center_x, center_y, size_x, size_y, angle, np.average(img)] - + # Set the bounds for the inputs bounds = [(0.0, np.inf), # Amplitude bounds (0.0, img.shape[1]), (0.0, img.shape[0]), # center location bounds (np.finfo(float).eps, np.inf), (np.finfo(float).eps, np.inf), # std dev bounds (-np.inf, np.inf), # angle bounds - (-(np.power(2, bit_depth) - 1), 0)] # img is 12 bit, and (-1 *img) is passed to function + (min(img.flatten()), max(img.flatten()))] # If this is using a super gauss add the additional p term for the initial # guess and bounds @@ -373,20 +331,20 @@ def fit_gaussian(img, target_type=None, keypoint=None, img_offset=None): try: popt, pcov = opt.curve_fit(fitter, pts, z, p0=initial_guess, method='trf', bounds=bounds, **curve_fit_kwds) - + # If debug is on, append the full popt and pconv if debug: retval.append((popt, pcov)) - + # If debug is off, append the center location as a tuple to the return value else: x_opt, y_opt = popt[1], popt[2] - + if img_offset is not None: x_opt += img_offset[0] y_opt += img_offset[1] - - retval.append((x_opt, y_opt)) + + retval.append(np.array((x_opt, y_opt))) # If retval is on, calculate the rms and append if get_rms or debug: @@ -410,43 +368,41 @@ def fit_gaussian(img, target_type=None, keypoint=None, img_offset=None): error = z - z_hat rms = np.sqrt(np.sum(np.power(error, 2)) / error.size) retval.append(rms) - + except RuntimeError as e: - # Since there was a runtime error, throw a warning and return the -1 flags + # Since there was a runtime error, throw a warning and return the None flags warnings.warn(str(e), RuntimeWarning) # If debug is on, append arrays of the appropriate size for popt and pconv if debug: - retval.append((np.full(len(initial_guess), -1), - np.full((len(popt), len(popt)), -1))) + retval.append((np.full(len(initial_guess), None), + np.full((len(popt), len(popt)), None))) - # If debug is off, just append (-1, -1) as the center location + # If debug is off, just append (None, None) as the center location else: - retval.append((-1, -1)) + retval.append(np.array((None, None))) - # If get_rms is on, append -1 + # If get_rms is on, append None if get_rms: - retval.append(-1) - - return tuple(retval) + retval.append(None) except ValueError as e: - # Since there was a runtime error, throw a warning and return the -1 flags + # Since there was a runtime error, throw a warning and return the None flags warnings.warn(str(e), RuntimeWarning) # If debug is on, append arrays of the appropriate size for popt and pconv if debug: - retval.append(np.full(len(initial_guess), -1)) - retval.append(np.full((len(initial_guess), len(initial_guess)), -1)) + retval.append(np.full(len(initial_guess), None)) + retval.append(np.full((len(initial_guess), len(initial_guess)), None)) - # If debug is off, just append (-1, -1) as the center location + # If debug is off, just append (None, None) as the center location else: - retval.append((-1, -1)) + retval.append((None, None)) - # If get_rms is on, append -1 + # If get_rms is on, append None if get_rms: - retval.append(-1) - + retval.append(None) + return tuple(retval) # Return a tuple of the return value (converting from list) diff --git a/scripts/upsp-external-calibration b/scripts/upsp-external-calibration index bb92cc9..3d92a59 100755 --- a/scripts/upsp-external-calibration +++ b/scripts/upsp-external-calibration @@ -4,27 +4,14 @@ import cv2 import json import logging import os -import sys import matplotlib matplotlib.use("Agg") -sys.path.append( - os.path.realpath( - os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "python") - ) -) - -import warnings # noqa - -warnings.filterwarnings("ignore") -import upsp.cam_cal_utils.parsers as parsers # noqa -import upsp.cam_cal_utils.visibility as visibility # noqa -import upsp.cam_cal_utils.external_calibrate as external_calibrate # noqa -import upsp.cam_cal_utils.photogrammetry as photogrammetry # noqa - -warnings.filterwarnings("default") - +import upsp.cam_cal_utils.parsers as parsers +import upsp.cam_cal_utils.visibility as visibility +import upsp.cam_cal_utils.external_calibrate as external_calibrate +import upsp.cam_cal_utils.photogrammetry as photogrammetry DEFAULT_OUTPUT_FILENAME = "external_calibration.json" @@ -93,8 +80,9 @@ def main(): for camera_number, img_filename, img in imgs: log.info("Calibrating camera %02d based on %s", camera_number, img_filename) - camera_cal = parsers.read_camera_params( - camera_number, args.cal_dir, img.shape + camera_cal_path = os.path.join(args.cal_dir, 'camera' + str(camera_number).rjust(2, '0') + '.json') + camera_cal = parsers.read_camera_tunnel_cal( + camera_cal_path, img.shape ) try: @@ -109,8 +97,8 @@ def main(): with open(output_filename, "w") as fp: json.dump( { - "cameraMatrix": camera_cal[0].squeeze().tolist(), - "distCoeffs": camera_cal[1].squeeze().tolist(), + "cameraMatrix": camera_cal[2].squeeze().tolist(), + "distCoeffs": camera_cal[3].squeeze().tolist(), "rmat": rmat.squeeze().tolist(), "tvec": tvec.squeeze().tolist(), "imageSize": [img.shape[1], img.shape[0]] diff --git a/scripts/upsp-unity-export b/scripts/upsp-unity-export deleted file mode 100755 index b6b2458..0000000 --- a/scripts/upsp-unity-export +++ /dev/null @@ -1,372 +0,0 @@ -#!/usr/bin/env python -# -# Parse raw wind tunnel test data and export resources -# used by the Unity3D visualization tool. -import argparse -import logging -import os -import sys -import glob -import pprint -import numpy as np -import math - -sys.path.append( - os.path.realpath( - os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "python") - ) -) - -import warnings # noqa - -warnings.filterwarnings("ignore") -import upsp.processing.context as context # noqa - -# todo-mshawlec: importing "tree" to access helper methods -# for resolving paths in the processing pipeline directory. -# These should be migrated into the context.Pipeline interface. -import upsp.processing.io as io # noqa -import upsp.processing.tree as tree # noqa -import upsp.processing.kulite_processing as kulite_processing # noqa -import upsp.processing.kulite_utilities as kulite_utilities # noqa -import upsp.processing.unity_conversions as unity_conversions # noqa -import upsp.processing.unity_stats as unity_stats # noqa - -warnings.filterwarnings("default") - -log = logging.getLogger(__name__) - - -# Variable declarations -unity_tree = {} -UNITY_GRID_ASSETS_SUBDIR = "Assets/Grids" -UNITY_STREAMING_ASSETS_SUBDIR = "Assets/StreamingAssets" -UNITY_TEXT_ASSETS_SUBDIR = "Assets/Text" - - -def model_config_name(datapoint): - config_ranges = {"config54": [3142, 3173], "config55": [3003, 3141]} - run_number = int(datapoint[:4]) - for config, v in config_ranges.items(): - if run_number >= v[0] and run_number <= v[1]: - return config - log.warning( - "%s: no model config. Expected mapping: %s" - % (datapoint, pprint.pformat(config_ranges)) - ) - - -def check_unity_tree(cfg_name, datapoint): - dp_idx = np.nan - cfg_idx = np.nan - for i, sub_dict in enumerate(unity_tree["Configs"]): - if cfg_name in sub_dict.values(): - cfg_idx = i - for j, sub_dp_dict in enumerate(sub_dict["Datapoints"]): - if datapoint in sub_dp_dict.values(): - dp_idx = j - - return cfg_idx, dp_idx - - -def write_empty_config_entry(cfg_name): - return { - "Name": cfg_name, - "FileList": [], - "Datapoints": [] - } - - -def add_config_entry_to_unity_tree(cfg_name, cfg_entry): - # Check for config and append entry - for sub_dict in unity_tree["Configs"]: - if cfg_name in sub_dict.values(): - sub_dict["FileList"].append(cfg_entry) - cfg_flag = True - - # If config not found, write config entry - if not cfg_flag: - config_contents = { - "Name": cfg_name, - "FileList": [cfg_entry], - "Datapoints": [] - } - unity_tree["Configs"].append(config_contents) - - -def add_dp_entry_to_unity_tree(cfg_name, dp_name, dp_entry): - # Check for config and datapoint - cfg_idx, dp_idx = check_unity_tree(cfg_name, dp_name) - - # If datapoint not found, write datapoint entry - if math.isnan(dp_idx): - dp_contents = { - "Name": dp_name, - "FileList": [dp_entry] - } - - # If config not found, write config entry - if math.isnan(cfg_idx): - config_contents = write_empty_config_entry(cfg_name) - config_contents["Datapoints"].append(dp_contents) - unity_tree["Configs"].append(config_contents) - # Otherwise enter config entry - else: - unity_tree["Configs"][cfg_idx]["Datapoints"].append(dp_contents) - - # Otherwise enter datapoint entry - else: - unity_tree["Configs"][cfg_idx]["Datapoints"][dp_idx]["FileList"].append( - dp_entry - ) - - -def _create_data_and_stats_files(ctx: context.Pipeline, output_dir): - """Visualization: calculate data and stats files from binary data""" - - for dp_ctx in ctx.datapoints: - datapoint = dp_ctx.name - cfg_name = model_config_name(datapoint) - - dp_output_dir = os.path.join( - output_dir, UNITY_STREAMING_ASSETS_SUBDIR, datapoint - ) - tree._create_dir_and_log(dp_output_dir) - os.chmod(dp_output_dir, io._CHMOD_URWXGR_XO__) - - # Retrieve stable output flat file to provide vertex number info - x_vertex_file = os.path.join( - os.path.dirname(output_dir), "psp_process", datapoint, "X" - ) - - # Set up to search the different folders for the Unity src files - dp_psp_dir = os.path.join( - os.path.dirname(output_dir), "*", datapoint - ) - - # Iterate through and check unity source files - unity_files = ctx.ctx["processing"]["defaults"]["unity_export"] - for el in unity_files["datasets"]: - dp_psp_search = os.path.join(dp_psp_dir, el["src"]) - unity_dst_file = os.path.join(dp_output_dir, el["dst"]) - log.info("searching bytes and stats for %s", dp_psp_search) - for unity_src_file in glob.glob(dp_psp_search, recursive=True): - unity_stats.write_unity_dataset_with_stats( - x_vertex_file, unity_src_file, unity_dst_file - ) - - # Populate files for the unity output tree - file_data_entry = { - "Name": os.path.join( - UNITY_STREAMING_ASSETS_SUBDIR, - dp_ctx.name, - el["dst"] + ".bytes" - ), - "Size": os.path.getsize( - unity_dst_file + ".bytes" - ), - } - file_stats_entry = { - "Name": os.path.join( - UNITY_STREAMING_ASSETS_SUBDIR, - dp_ctx.name, - el["dst"] + "_stats.txt" - ), - "Size": os.path.getsize( - unity_dst_file + "_stats.txt" - ), - } - - # Fill out the unity output tree - add_dp_entry_to_unity_tree(cfg_name, datapoint, file_data_entry) - add_dp_entry_to_unity_tree(cfg_name, datapoint, file_stats_entry) - - -def _create_wtd_text_files(ctx: context.Pipeline, output_dir): - """Visualization: create wind tunnel data text file from .wtd file""" - - for dp_ctx in ctx.datapoints: - datapoint = dp_ctx.name - cfg_name = model_config_name(datapoint) - - dp_output_dir = os.path.join(output_dir, UNITY_TEXT_ASSETS_SUBDIR, datapoint) - tree._create_dir_and_log(dp_output_dir) - wtd_filename = dp_ctx.inputs["wtd_file"] - wtd_basename = os.path.basename(wtd_filename).split(".")[0] - wtd_text_filename = os.path.join( - dp_output_dir, "%s_wtd.txt" % wtd_basename - ) - wtd_text_str = "\nExperimental Data\n\nRUN NUMBER:\t" - with open(wtd_filename, "r") as fp: - # Parse first line - line1 = fp.readline().split() - if len(line1[2]) < 2: - wtd_text_str += "%s0" % (line1[1]) - wtd_text_str += "%s\n" % (line1[2]) - else: - wtd_text_str += "%s" % (line1[1]) - wtd_text_str += "%s\n" % (line1[2]) - - # Parse second and third lines - line2 = fp.readline().split() - line3 = fp.readline().split() - if len(line3) + 1 != len(line2): - log.error( - '"%s": 2nd and 3rd lines do not have same number of columns', - wtd_filename, - ) - idx = 1 - for element in line3: - wtd_text_str += "%s:\t" % (line2[idx].replace("_", " ")) - wtd_text_str += "%s\n" % (element) - idx += 1 - - # Write wind tunnel output file - with open(wtd_text_filename, "w") as fp: - fp.write(wtd_text_str) - os.chmod(wtd_text_filename, io._CHMOD_URW_GR__O__) - - # Populate files for the unity output tree - file_data_entry = { - "Name": os.path.join( - UNITY_TEXT_ASSETS_SUBDIR, - datapoint, - wtd_basename + "_wtd.txt" - ), - "Size": os.path.getsize( - wtd_text_filename - ) - } - - # Fill out the unity output tree - add_dp_entry_to_unity_tree(cfg_name, datapoint, file_data_entry) - - -def _create_kulite_location_files(ctx: context.Pipeline, output_dir: str): - """Visualization: create kulite labels text file from targets files.""" - - k_output_dir = os.path.join(output_dir, UNITY_TEXT_ASSETS_SUBDIR) - tree._create_dir_and_log(k_output_dir) - - targets_files = set([dp.inputs["targets_file"] for dp in ctx.datapoints]) - for src_filename in targets_files: - cfg_name = os.path.basename(src_filename).split(".")[0] - kulite_labels_file = os.path.join( - k_output_dir, "%s-kulites-left-handed-coordinates.txt" % cfg_name - ) - - # Process the targets information to get kulite information - kul_mat = kulite_utilities.read_targets_matrix(src_filename, kul_strs="all") - - if kul_mat.size == 0: - break - - # Convert the kulite locations into Unity locations - unity_kul_mat = unity_conversions.convert_kulite_locations(kul_mat) - - # Write the kulite label information - test_num = ctx.ctx["__meta__"]["datapoints"]["test_name"] - kulite_processing.write_matrix_txt(kulite_labels_file, unity_kul_mat, test_num) - os.chmod(kulite_labels_file, io._CHMOD_URW_GR__O__) - - # Populate files for the unity output tree - file_entry = { - "Name": os.path.join( - UNITY_TEXT_ASSETS_SUBDIR, - cfg_name + "-kulites-left-handed-coordinates.txt" - ), - "Size": os.path.getsize( - kulite_labels_file - ), - } - - # Fill out the unity output tree - add_config_entry_to_unity_tree(cfg_name, file_entry) - - -def _create_grid_obj_files(ctx: context.Pipeline, output_dir: str): - """Visualization: create mesh + vertex mapping files for Unity consumption: - - obj file converted from the plot3d grid file - - vertex-to-zones mapping binary file from plot3D zones - """ - - obj_output_dir = os.path.join(output_dir, UNITY_GRID_ASSETS_SUBDIR) - tree._create_dir_and_log(obj_output_dir) - zone_output_dir = os.path.join(output_dir, UNITY_STREAMING_ASSETS_SUBDIR) - tree._create_dir_and_log(zone_output_dir) - - # Iterate through the grid p3d files - grid_files = set([dp.inputs["grid_file"] for dp in ctx.datapoints]) - for src_filename in grid_files: - - # Build the obj destination file - cfg_name = os.path.basename(src_filename).split(".")[0] - obj_filename = os.path.join( - obj_output_dir, "%s-left-handed-coordinates.obj" % cfg_name - ) - zones_map_filename = os.path.join( - zone_output_dir, "%s-zones.bytes" % cfg_name - ) - - # Call conversion function to write obj file, and zones and patches - # mapping files - log.info("Writing %s, %s ...", obj_filename, zones_map_filename) - unity_conversions.convert_to_unity_obj( - src_filename, - obj_filename, - zones_map_filename, - ) - - # Populate files for the unity output tree - file_obj_entry = { - "Name": os.path.join( - UNITY_GRID_ASSETS_SUBDIR, - cfg_name + "-left-handed-coordinates.obj" - ), - "Size": os.path.getsize( - obj_filename - ) - } - file_zones_entry = { - "Name": os.path.join( - UNITY_STREAMING_ASSETS_SUBDIR, - cfg_name + "-zones.bytes" - ), - "Size": os.path.getsize( - zones_map_filename - ) - } - - # Fill out the unity output tree - add_config_entry_to_unity_tree(cfg_name, file_obj_entry) - add_config_entry_to_unity_tree(cfg_name, file_zones_entry) - - -def main(): - logging.basicConfig(level=logging.INFO) - ap = argparse.ArgumentParser( - prog="upsp-unity-export", - description="Export resources for Unity3D visualization application", - ) - ap.add_argument("--pipeline_dir", required=True, help="Pipeline directory") - ap.add_argument("--output_dir", required=True, help="Output directory") - args = ap.parse_args() - ctx = context.Pipeline(args.pipeline_dir) - - # Start the build of unity tree - unity_tree["NasLocation"] = args.output_dir, - unity_tree["Configs"] = [] - - _create_data_and_stats_files(ctx, args.output_dir) - _create_wtd_text_files(ctx, args.output_dir) - _create_grid_obj_files(ctx, args.output_dir) - _create_kulite_location_files(ctx, args.output_dir) - - # Write unity tree - json_output = os.path.join(args.output_dir, "unity-tree.json") - io.json_write_or_die(unity_tree, json_output, indent=2) - - -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) - main() diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..e2876d7 --- /dev/null +++ b/setup.py @@ -0,0 +1,9 @@ +from skbuild import setup +from setuptools import find_packages + +setup( + packages=find_packages(where="python"), + package_dir={"": "python"}, + include_package_data=True, + package_data={"upsp.processing": ["templates/*.template"]}, +)