From 364972b45b96ef315b62877f8feb09fc8d458861 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 13 Dec 2024 02:45:23 +0100 Subject: [PATCH] Add a CMake EMBED_GRIDS_DIRECTORY option to embed .tif/.json files into libproj and also generalize the existing EMBED_RESOURCE_FILES option to embed the ITRFxxxx resource files. ``` .. option:: EMBED_GRIDS_DIRECTORY= .. versionadded:: 9.6 Embed files from ending with .tif or .json in the PROJ library itself. The pointed directory can potentially be the full PROJ-data package (uncompressed). In that case, about 6 GB of free disk and 16 GB of RAM are required to build PROJ. When using this parameter, EMBED_RESOURCE_FILES must be set to ON. If the content of the directory changes, you need to run CMake again to update the list of files. ``` --- .github/workflows/fedora_rawhide/start.sh | 28 ++++- CMakeLists.txt | 15 +++ data/CMakeLists.txt | 17 +-- docs/source/install.rst | 24 +++- src/embedded_resources.c | 19 +-- src/embedded_resources.h | 4 +- src/filemanager.cpp | 137 +++++++++++++++++++--- src/lib_proj.cmake | 97 +++++++++++---- 8 files changed, 268 insertions(+), 73 deletions(-) diff --git a/.github/workflows/fedora_rawhide/start.sh b/.github/workflows/fedora_rawhide/start.sh index ff219fb606..2450d6972e 100755 --- a/.github/workflows/fedora_rawhide/start.sh +++ b/.github/workflows/fedora_rawhide/start.sh @@ -2,7 +2,7 @@ set -e -dnf install -y cmake clang ccache ninja-build sqlite-devel libtiff-devel libcurl-devel diffutils +dnf install -y cmake clang ccache ninja-build sqlite-devel libtiff-devel libcurl-devel diffutils wget cd "$WORK_DIR" @@ -18,10 +18,34 @@ ccache -s mkdir build cd build +echo "Build with -DEMBED_RESOURCE_FILES=ON" CC=clang CXX=clang++ cmake .. \ - -DEMBED_RESOURCE_FILES=ON -DUSE_ONLY_EMBEDDED_RESOURCE_FILES=ON -DUSE_CCACHE=ON -DPROJ_DB_CACHE_DIR=$HOME/.ccache .. + -DEMBED_RESOURCE_FILES=ON -DUSE_CCACHE=ON -DPROJ_DB_CACHE_DIR=$HOME/.ccache .. make -j$(nproc) ctest -j$(nproc) + +# Try EMBED_GRIDS_DIRECTORY option +wget https://raw.githubusercontent.com/OSGeo/PROJ-data/refs/heads/master/us_nga/us_nga_egm96_15.tif +mkdir grids +mv us_nga_egm96_15.tif grids +echo "Build with -DEMBED_RESOURCE_FILES=ON -DEMBED_GRIDS_DIRECTORY=$PWD/grids" +CC=clang CXX=clang++ cmake .. -DEMBED_GRIDS_DIRECTORY=$PWD/grids +make -j$(nproc) +rm -rf data +echo 49 2 0 | bin/cs2cs "WGS84 + EGM96 height" EPSG:4979 +echo 49 2 0 | bin/cs2cs "WGS84 + EGM96 height" EPSG:4979 | grep 44.643 >/dev/null || (echo "Expected 49dN 2dE 44.643 as a result" && /bin/false) +echo 0 0 0 | bin/cct +init=ITRF2000:ITRF96 +echo 0 0 0 | bin/cct +init=ITRF2000:ITRF96 | grep 0.0067 >/dev/null || (echo "Expected 0.0067 0.0061 -0.0185 as a result" && /bin/false) + +echo "Build with -DEMBED_RESOURCE_FILES=ON -DEMBED_GRIDS_DIRECTORY=$PWD/grids -DUSE_ONLY_EMBEDDED_RESOURCE_FILES=ON" +CC=clang CXX=clang++ cmake .. -DEMBED_GRIDS_DIRECTORY=$PWD/grids -DUSE_ONLY_EMBEDDED_RESOURCE_FILES=ON +make -j$(nproc) +rm -rf data +echo 49 2 0 | bin/cs2cs "WGS84 + EGM96 height" EPSG:4979 +echo 49 2 0 | bin/cs2cs "WGS84 + EGM96 height" EPSG:4979 | grep 44.643 >/dev/null || (echo "Expected 49dN 2dE 44.643 as a result" && /bin/false) +echo 0 0 0 | bin/cct +init=ITRF2000:ITRF96 +echo 0 0 0 | bin/cct +init=ITRF2000:ITRF96 | grep 0.0067 >/dev/null || (echo "Expected 0.0067 0.0061 -0.0185 as a result" && /bin/false) + cd .. ccache -s diff --git a/CMakeLists.txt b/CMakeLists.txt index e70adf19c5..b6cf51b3ee 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -384,6 +384,21 @@ endif() ################################################################################ # Build configured components ################################################################################ + +set(PROJ_DICTIONARY + world + other.extra + nad27 + GL27 + nad83 + nad.lst + CH + ITRF2000 + ITRF2008 + ITRF2014 + ITRF2020 +) + include_directories(${PROJ_SOURCE_DIR}/src) add_subdirectory(data) diff --git a/data/CMakeLists.txt b/data/CMakeLists.txt index 8641d896f6..a32ce135a0 100644 --- a/data/CMakeLists.txt +++ b/data/CMakeLists.txt @@ -15,20 +15,6 @@ set(PROJ_INI proj.ini ) -set(PROJ_DICTIONARY - world - other.extra - nad27 - GL27 - nad83 - nad.lst - CH - ITRF2000 - ITRF2008 - ITRF2014 - ITRF2020 -) - # # gridshift file # @@ -111,14 +97,13 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/tests/conus "${CMAKE_CURRENT_BINARY_D #install # set(ALL_DATA_FILE - ${PROJ_DICTIONARY} ${GRIDSHIFT_FILES} ${SCHEMA_FILES} ${GEOTIFF_FILES} ) if (NOT USE_ONLY_EMBEDDED_RESOURCE_FILES) - list(APPEND ALL_DATA_FILE ${PROJ_INI} ${PROJ_DB}) + list(APPEND ALL_DATA_FILE ${PROJ_INI} ${PROJ_DB} ${PROJ_DICTIONARY}) endif() install( diff --git a/docs/source/install.rst b/docs/source/install.rst index beffee6f53..8c5a95a4b0 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -450,19 +450,35 @@ All cached entries can be viewed using ``cmake -LAH`` from a build directory. .. versionadded:: 9.6 + When ON, :file:`proj.db`, :file:`proj.ini` and ITRF resource files will be + embedded into the PROJ library. Default is OFF for shared library builds (BUILD_SHARED_LIBS=ON), and ON for static library builds (BUILD_SHARED_LIBS=OFF). - When ON, :file:`proj.db` and :file:`proj.ini` will be embedded into the PROJ library. + +.. option:: EMBED_GRIDS_DIRECTORY= + + .. versionadded:: 9.6 + + Embed files from ending with .tif or .json in the PROJ library itself. + + The pointed directory can potentially be the full PROJ-data package (uncompressed). + In that case, about 6 GB of free disk and 16 GB of RAM are required to build PROJ. + + When using this parameter, EMBED_RESOURCE_FILES must be set to ON. + + If the content of the directory changes, you need to run CMake again to + update the list of files. .. option:: USE_ONLY_EMBEDDED_RESOURCE_FILES=ON/OFF .. versionadded:: 9.6 Even if EMBED_RESOURCE_FILES=ON, by default PROJ will still try to locate - :file:`proj.db` and :file:`proj.ini` on the file system, and fallback to the - embedded version if not found. + :file:`proj.db`, :file:`proj.ini`, ITRF resource files or grid files on the + file system, and fallback to the embedded version if not found. By setting USE_ONLY_EMBEDDED_RESOURCE_FILES=ON, no attempt at locating - those files on the file system is made. Default is OFF. + those files on the file system is made. + Default is OFF. Users will also typically want to set EMBED_PROJ_DATA_PATH=OFF if setting USE_ONLY_EMBEDDED_RESOURCE_FILES=OFF. diff --git a/src/embedded_resources.c b/src/embedded_resources.c index 04b721f11e..8b25e8a0cc 100644 --- a/src/embedded_resources.c +++ b/src/embedded_resources.c @@ -1,3 +1,6 @@ +#include +#include + #include "embedded_resources.h" #if USE_SHARP_EMBED @@ -9,14 +12,6 @@ const unsigned char *pj_get_embedded_proj_db(unsigned int *pnSize) { return proj_db; } -const char *pj_get_embedded_proj_ini(unsigned int *pnSize) { - static const char proj_ini[] = { -#embed PROJ_INI - }; - *pnSize = (unsigned int)sizeof(proj_ini); - return proj_ini; -} - #else #include "file_embed/proj_db.h" @@ -25,10 +20,6 @@ const unsigned char *pj_get_embedded_proj_db(unsigned int *pnSize) { return proj_db_data; } -#include "file_embed/proj_ini.h" -const char *pj_get_embedded_proj_ini(unsigned int *pnSize) { - *pnSize = proj_ini_size; - return (const char *)proj_ini_data; -} - #endif + +#include "file_embed/embedded_resources.c" diff --git a/src/embedded_resources.h b/src/embedded_resources.h index 39a119b0a7..6b66db4cca 100644 --- a/src/embedded_resources.h +++ b/src/embedded_resources.h @@ -6,7 +6,9 @@ extern "C" { #endif const unsigned char *pj_get_embedded_proj_db(unsigned int *pnSize); -const char *pj_get_embedded_proj_ini(unsigned int *pnSize); + +const unsigned char *pj_get_embedded_resource(const char *filename, + unsigned int *pnSize); #ifdef __cplusplus } diff --git a/src/filemanager.cpp b/src/filemanager.cpp index 2e3f3d8a26..98d8aff04f 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -143,6 +143,8 @@ std::string File::read_line(size_t maxLen, bool &maxLenReached, // --------------------------------------------------------------------------- +#if !USE_ONLY_EMBEDDED_RESOURCE_FILES + #ifdef _WIN32 /* The bulk of utf8towc()/utf8fromwc() is derived from the utf.c module from @@ -805,6 +807,8 @@ std::unique_ptr FileStdio::open(PJ_CONTEXT *ctx, const char *filename, #endif // _WIN32 +#endif // !USE_ONLY_EMBEDDED_RESOURCE_FILES + // --------------------------------------------------------------------------- class FileApiAdapter : public File { @@ -893,6 +897,80 @@ std::unique_ptr FileApiAdapter::open(PJ_CONTEXT *ctx, // --------------------------------------------------------------------------- +#if EMBED_RESOURCE_FILES + +class FileMemory : public File { + PJ_CONTEXT *m_ctx; + const unsigned char *const m_data; + const size_t m_size; + size_t m_pos = 0; + + FileMemory(const FileMemory &) = delete; + FileMemory &operator=(const FileMemory &) = delete; + + protected: + FileMemory(const std::string &filename, PJ_CONTEXT *ctx, + const unsigned char *data, size_t size) + : File(filename), m_ctx(ctx), m_data(data), m_size(size) {} + + public: + size_t read(void *buffer, size_t sizeBytes) override; + size_t write(const void *, size_t) override; + bool seek(unsigned long long offset, int whence = SEEK_SET) override; + unsigned long long tell() override { return m_pos; } + void reassign_context(PJ_CONTEXT *ctx) override { m_ctx = ctx; } + + bool hasChanged() const override { return false; } + + static std::unique_ptr open(PJ_CONTEXT *ctx, const char *filename, + FileAccess access, + const unsigned char *data, size_t size) { + if (access != FileAccess::READ_ONLY) + return nullptr; + return std::unique_ptr(new FileMemory(filename, ctx, data, size)); + } +}; + +size_t FileMemory::read(void *buffer, size_t sizeBytes) { + if (m_pos >= m_size) + return 0; + if (sizeBytes >= m_size - m_pos) { + const size_t bytesToCopy = m_size - m_pos; + memcpy(buffer, m_data + m_pos, bytesToCopy); + m_pos = m_size; + return bytesToCopy; + } + memcpy(buffer, m_data + m_pos, sizeBytes); + m_pos += sizeBytes; + return sizeBytes; +} + +size_t FileMemory::write(const void *, size_t) { + // shouldn't happen given we have bailed out in open() in non read-only + // modes + return 0; +} + +bool FileMemory::seek(unsigned long long offset, int whence) { + if (whence == SEEK_SET) { + m_pos = static_cast(offset); + return m_pos == offset; + } else if (whence == SEEK_CUR) { + const unsigned long long newPos = m_pos + offset; + m_pos = static_cast(newPos); + return m_pos == newPos; + } else { + if (offset != 0) + return false; + m_pos = m_size; + return true; + } +} + +#endif + +// --------------------------------------------------------------------------- + std::unique_ptr FileManager::open(PJ_CONTEXT *ctx, const char *filename, FileAccess access) { if (starts_with(filename, "http://") || starts_with(filename, "https://")) { @@ -909,11 +987,31 @@ std::unique_ptr FileManager::open(PJ_CONTEXT *ctx, const char *filename, if (ctx->fileApi.open_cbk != nullptr) { return FileApiAdapter::open(ctx, filename, access); } + + std::unique_ptr ret; +#if !(EMBED_RESOURCE_FILES && USE_ONLY_EMBEDDED_RESOURCE_FILES) #ifdef _WIN32 - return FileWin32::open(ctx, filename, access); + ret = FileWin32::open(ctx, filename, access); #else - return FileStdio::open(ctx, filename, access); + ret = FileStdio::open(ctx, filename, access); #endif +#endif + +#if EMBED_RESOURCE_FILES +#if USE_ONLY_EMBEDDED_RESOURCE_FILES + if (!ret) +#endif + { + unsigned int size = 0; + const unsigned char *in_memory_data = + pj_get_embedded_resource(filename, &size); + if (in_memory_data) { + ret = FileMemory::open(ctx, filename, access, in_memory_data, size); + } + } +#endif + + return ret; } // --------------------------------------------------------------------------- @@ -1574,6 +1672,24 @@ static void *pj_open_lib_internal( errno = 0; } +#if EMBED_RESOURCE_FILES + if (!fid && fname != name && name[0] != '.' && name[0] != '/' && + name[0] != '~' && !starts_with(name, "http://") && + !starts_with(name, "https://")) { + fid = open_file(ctx, name, mode); + if (fid) { + if (out_full_filename != nullptr && + out_full_filename_size > 0) { + // cppcheck-suppress nullPointer + strncpy(out_full_filename, name, out_full_filename_size); + out_full_filename[out_full_filename_size - 1] = '\0'; + } + fname = name; + errno = 0; + } + } +#endif + if (ctx->last_errno == 0 && errno != 0) proj_context_errno_set(ctx, errno); @@ -1870,20 +1986,9 @@ void pj_load_ini(PJ_CONTEXT *ctx) { ctx->iniFileLoaded = true; std::string content; - std::unique_ptr file; -#ifndef USE_ONLY_EMBEDDED_RESOURCE_FILES - file.reset(reinterpret_cast(pj_open_lib_internal( - ctx, "proj.ini", "rb", pj_open_file_with_manager, nullptr, 0))); -#endif - if (!file) { -#ifdef EMBED_RESOURCE_FILES - unsigned int content_size = 0; - const char *c_content = pj_get_embedded_proj_ini(&content_size); - content.assign(c_content, content_size); -#else - return; -#endif - } + auto file = std::unique_ptr( + reinterpret_cast(pj_open_lib_internal( + ctx, "proj.ini", "rb", pj_open_file_with_manager, nullptr, 0))); if (file) { file->seek(0, SEEK_END); const auto filesize = file->tell(); diff --git a/src/lib_proj.cmake b/src/lib_proj.cmake index a36633926c..cd7841350d 100644 --- a/src/lib_proj.cmake +++ b/src/lib_proj.cmake @@ -401,7 +401,14 @@ if("${CMAKE_C_COMPILER_ID}" STREQUAL "Intel") PROPERTIES COMPILE_FLAGS ${FP_PRECISE}) endif() -if (EMBED_RESOURCE_FILES AND NOT IS_SHARP_EMBED_AVAILABLE_RES) +if (EMBED_RESOURCE_FILES) + add_library(proj_resources OBJECT embedded_resources.c) + add_dependencies(proj_resources generate_proj_db) + option(PROJ_OBJECT_LIBRARIES_POSITION_INDEPENDENT_CODE "Set ON to produce -fPIC code" ${BUILD_SHARED_LIBS}) + set_property(TARGET proj_resources PROPERTY POSITION_INDEPENDENT_CODE ${PROJ_OBJECT_LIBRARIES_POSITION_INDEPENDENT_CODE}) + target_sources(proj PRIVATE $) + + if (NOT IS_SHARP_EMBED_AVAILABLE_RES) set(EMBEDDED_PROJ_DB "file_embed/proj_db.c") add_custom_command( OUTPUT "${EMBEDDED_PROJ_DB}" @@ -411,29 +418,79 @@ if (EMBED_RESOURCE_FILES AND NOT IS_SHARP_EMBED_AVAILABLE_RES) -P ${PROJECT_SOURCE_DIR}/cmake/FileEmbed.cmake DEPENDS generate_proj_db "${PROJECT_BINARY_DIR}/data/proj.db" ) - target_sources(proj PRIVATE embedded_resources.c "${EMBEDDED_PROJ_DB}") - - set(EMBEDDED_PROJ_INI "file_embed/proj_ini.c") - add_custom_command( - OUTPUT "${EMBEDDED_PROJ_INI}" - COMMAND ${CMAKE_COMMAND} - -DRUN_FILE_EMBED_GENERATE=1 - "-DFILE_EMBED_GENERATE_PATH=${PROJECT_SOURCE_DIR}/data/proj.ini" - -P ${PROJECT_SOURCE_DIR}/cmake/FileEmbed.cmake - DEPENDS "${PROJECT_SOURCE_DIR}/data/proj.ini" - ) - target_sources(proj PRIVATE embedded_resources.c "${EMBEDDED_PROJ_DB}" "${EMBEDDED_PROJ_INI}") -elseif(EMBED_RESOURCE_FILES AND IS_SHARP_EMBED_AVAILABLE_RES) - add_library(proj_resources OBJECT embedded_resources.c) + target_sources(proj_resources PRIVATE "${EMBEDDED_PROJ_DB}") + else() target_compile_definitions(proj_resources PRIVATE "PROJ_DB=\"${PROJECT_BINARY_DIR}/data/proj.db\"") - target_compile_definitions(proj_resources PRIVATE "PROJ_INI=\"${PROJECT_SOURCE_DIR}/data/proj.ini\"") target_compile_definitions(proj_resources PRIVATE USE_SHARP_EMBED) - add_dependencies(proj_resources generate_proj_db) - option(PROJ_OBJECT_LIBRARIES_POSITION_INDEPENDENT_CODE "Set ON to produce -fPIC code" ${BUILD_SHARED_LIBS}) - set_property(TARGET proj_resources PROPERTY POSITION_INDEPENDENT_CODE ${PROJ_OBJECT_LIBRARIES_POSITION_INDEPENDENT_CODE}) set_target_properties(proj_resources PROPERTIES C_STANDARD 23) - target_sources(proj PRIVATE $) + endif() endif() + +set(EMBED_GRIDS_DIRECTORY "" CACHE PATH "Directory that contains .tif and .json files to embed into libproj") +set(FILES_TO_EMBED) +if (EMBED_GRIDS_DIRECTORY) + if (NOT EMBED_RESOURCE_FILES) + message(FATAL_ERROR "EMBED_RESOURCE_FILES should be set to ON when EMBED_GRIDS_DIRECTORY is set") + endif() + + if (NOT IS_DIRECTORY ${EMBED_GRIDS_DIRECTORY}) + message(FATAL_ERROR "${EMBED_GRIDS_DIRECTORY} is not a valid directory") + endif() + + file(GLOB FILES_TO_EMBED "${EMBED_GRIDS_DIRECTORY}/*.tif" "${EMBED_GRIDS_DIRECTORY}/*.json") + if (NOT FILES_TO_EMBED) + message(FATAL_ERROR "No .tif or .json files found in ${EMBED_GRIDS_DIRECTORY}") + endif() +endif() + +if (EMBED_RESOURCE_FILES) + list(APPEND FILES_TO_EMBED "../data/proj.ini") + foreach(FILE ${PROJ_DICTIONARY}) + list(APPEND FILES_TO_EMBED "../data/${FILE}") + endforeach() +endif() + +if (FILES_TO_EMBED) + set(EMBEDDED_RESOURCES_C_PROLOG_CONTENT "") + set(EMBEDDED_RESOURCES_C_CONTENT "") + string(APPEND EMBEDDED_RESOURCES_C_CONTENT + "const unsigned char *pj_get_embedded_resource(const char* filename, unsigned int *pnSize)\n" + "{\n") + foreach(FILE ${FILES_TO_EMBED}) + get_filename_component(FILENAME ${FILE} NAME) + message(STATUS "Embedding ${FILENAME}") + set(C_IDENTIFIER_RESOURCE_NAME "${FILENAME}") + string(REPLACE "." "_" C_IDENTIFIER_RESOURCE_NAME "${C_IDENTIFIER_RESOURCE_NAME}") + string(REPLACE "-" "_" C_IDENTIFIER_RESOURCE_NAME "${C_IDENTIFIER_RESOURCE_NAME}") + set(C_FILENAME "file_embed/${C_IDENTIFIER_RESOURCE_NAME}.c") + add_custom_command( + OUTPUT "${C_FILENAME}" + COMMAND ${CMAKE_COMMAND} + -DRUN_FILE_EMBED_GENERATE=1 + "-DFILE_EMBED_GENERATE_PATH=${FILE}" + -P ${PROJECT_SOURCE_DIR}/cmake/FileEmbed.cmake + DEPENDS "${FILE}" + ) + + string(APPEND EMBEDDED_RESOURCES_C_CONTENT + " if (strcmp(filename, \"${FILENAME}\") == 0)\n" + " {\n" + " *pnSize = ${C_IDENTIFIER_RESOURCE_NAME}_size;\n" + " return ${C_IDENTIFIER_RESOURCE_NAME}_data;\n" + " }\n") + + target_sources(proj_resources PRIVATE "${C_FILENAME}") + + string(APPEND EMBEDDED_RESOURCES_C_PROLOG_CONTENT "extern const uint8_t ${C_IDENTIFIER_RESOURCE_NAME}_data[];\n") + string(APPEND EMBEDDED_RESOURCES_C_PROLOG_CONTENT "extern const unsigned ${C_IDENTIFIER_RESOURCE_NAME}_size;\n") + endforeach() + string(APPEND EMBEDDED_RESOURCES_C_CONTENT + " *pnSize = 0;\n" + " return NULL;\n" + "}\n") + file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/file_embed/embedded_resources.c" "${EMBEDDED_RESOURCES_C_PROLOG_CONTENT}\n${EMBEDDED_RESOURCES_C_CONTENT}") +endif() + if (EMBED_RESOURCE_FILES) target_sources(proj PRIVATE memvfs.c) target_compile_definitions(proj PRIVATE EMBED_RESOURCE_FILES)