Merge pull request #79 from ndsev/release/2024.4.1

Release 2024.4.1

.github/workflows/cmake.yaml

@@ -10,9 +10,9 @@ jobs:
   build-manylinux:
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
     runs-on: ubuntu-latest
-    container: ghcr.io/klebert-engineering/manylinux-cpp17-py${{ matrix.python-version }}-x86_64:2024.1
+    container: ghcr.io/klebert-engineering/manylinux-cpp17-py${{ matrix.python-version }}-x86_64:2024.2
     env:
       ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
       SCCACHE_GHA_ENABLED: "true"
@@ -22,16 +22,13 @@ jobs:
           submodules: recursive
       - name: Run sccache-cache
         uses: mozilla-actions/[email protected]
-      - name: Install Conan and newer Perl
+      - name: Install Conan
         run: |
           pip install conan
           conan profile detect
-          # OpenSSL Requires a newer perl: see https://github.com/openssl/openssl/issues/25366
-          yum update -y && yum install -y perl-IPC-Cmd rh-perl530
-          ldconfig /opt/rh/rh-perl530/root/lib64/
+          conan remote update --url https://center2.conan.io conancenter
       - name: Configure
         run: |
-          export PATH="/opt/rh/rh-perl530/root/bin:$PATH"
           python3 -m venv venv && . ./venv/bin/activate
           pip install -U setuptools wheel pip
           conan install . -s compiler.cppstd=20 -b missing -b editable \
@@ -64,7 +61,7 @@ jobs:
     strategy:
       matrix:
         os: [macos-13, windows-2019]  # Currently, macos-latest is macos 12
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
     env:
       SCCACHE_GHA_ENABLED: "true"
     steps:
@@ -81,6 +78,7 @@ jobs:
         run: |
           pip install conan
           conan profile detect
+          conan remote update --url https://center2.conan.io conancenter
       - run: python -m pip install setuptools wheel ninja
       - name: Build (macOS)
         if: matrix.os == 'macos-13'

.github/workflows/coverage.yml

@@ -27,6 +27,7 @@ jobs:
       run: |
         pip install conan
         conan profile detect
+        conan remote update --url https://center2.conan.io conancenter
     - name: Setup
       run: |
         conan install . -s compiler.cppstd=20 -s build_type=Debug -b missing \

.github/workflows/deploy.yaml

@@ -7,7 +7,7 @@ jobs:
   deploy:
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
         os: [macos-13, ubuntu-latest, windows-2019]
     runs-on: ubuntu-latest
     steps:

CMakeLists.txt

@@ -4,7 +4,7 @@ cmake_policy(SET CMP0117 NEW)
 
 project(mapget CXX)
 
-set(MAPGET_VERSION 2024.4.0)
+set(MAPGET_VERSION 2024.4.1)
 
 set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)

README.md

@@ -308,11 +308,12 @@ which is implemented in `mapget::Service`. Detailed endpoint descriptions:
 | Endpoint   | Method | Description                                                                                                       | Input                                                                                                                                               | Output                                                                                                                                                                                                                                                            |
 |------------|--------|-------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | `/sources` | GET    | Describe the connected Data Sources                                                                               | None                                                                                                                                                | `application/json`: List of DataSourceInfo objects.                                                                                                                                                                                                               |
-| `/tiles`   | POST   | Get streamed features, according to hard constraints. Accepts encoding types `text/jsonl` or `application/binary` | List of objects containing `mapId`, `layerId`, `tileIds`, and optional `stringPoolOffsets`.                                                         | `text/jsonl` or `application/binary`                                                                                                                                                                                                                              |
+| `/tiles`   | POST   | Get streamed features, according to hard constraints. Accepts encoding types `text/jsonl` or `application/binary` | List of objects containing `mapId`, `layerId`, `tileIds`, and optional `stringPoolOffsets` and `clientId`.                                          | `text/jsonl` or `application/binary`                                                                                                                                                                                                                              |
+| `/abort`   | POST   | Abort a currently running `/tiles` request by its `clientId`.                                                     | `clientId`                                                                                                                                          | `text/plain`                                                                                                                                                                                                                                                      |
 | `/status`  | GET    | Server status page                                                                                                | None                                                                                                                                                | `text/html`                                                                                                                                                                                                                                                       |
 | `/locate`  | POST   | Obtain a list of tile-layer combinations providing a feature that satisfies given ID field constraints.           | `application/json`: List of external references, where each is a Request object with `mapId`, `typeId` and `featureId` (list of external ID parts). | `application/json`: List of lists of Resolution objects, where each corresponds to the Request object index. Each Resolution object includes `tileId`, `typeId`, and `featureId`.                                                                                 |
 | `/config`  | GET    | Access the config yaml-file content.                                                                              | None                                                                                                                                                | `application/json`: Contains the `sources` and `http-settings` from the config-yaml as a JSON representation. The returned JSON object has a `model`, `schema` and `readOnly` key. The schema is controlled through the `--config-schema` command line parameter. |
-| `/config`  | POST   | Write the config yaml-file content. Enabled iff `--allow-post-config` is passed to mapget.                      | `application/json`                                                                                                                                  | `text/plain` (if an error occurs)                                                                                                                                                                                                                                 |
+| `/config`  | POST   | Write the config yaml-file content. Enabled iff `--allow-post-config` is passed to mapget.                        | `application/json`                                                                                                                                  | `text/plain` (if an error occurs)                                                                                                                                                                                                                                 |
 
 ### Curl Call Example
 

conanfile.py

@@ -48,7 +48,7 @@ def validate(self):
 
     def requirements(self):
         self.requires("fmt/10.2.1", override=True)
-        self.requires("simfil/0.3.2", transitive_headers=True)
+        self.requires("simfil/0.3.3", transitive_headers=True)
         self.requires("spdlog/[~1]", transitive_headers=True)
         self.requires("bitsery/[~5]")
         # The override=True for is needed, until simfil 0.3.3 is released.

deps.cmake

@@ -34,7 +34,7 @@ else()
 
   FetchContent_Declare(fmt
     GIT_REPOSITORY "https://github.com/fmtlib/fmt.git"
-    GIT_TAG        "10.0.0"
+    GIT_TAG        "11.0.2"
     GIT_SHALLOW    ON)
   FetchContent_MakeAvailable(fmt)
 
@@ -111,7 +111,7 @@ else()
     set(SIMFIL_SHARED          NO  CACHE BOOL "Simfil as static library")
     FetchContent_Declare(simfil
       GIT_REPOSITORY "https://github.com/Klebert-Engineering/simfil.git"
-      GIT_TAG        "v0.3.2"
+      GIT_TAG        "v0.3.3"
       GIT_SHALLOW    ON)
     FetchContent_MakeAvailable(simfil)
   endif()

libs/http-service/src/cli.cpp

@@ -340,9 +340,9 @@ struct FetchCommand
         request->onSourceDataLayer(fn);
         cli.request(request)->wait();
 
-        if (request->getStatus() == NoDataSource)
+        if (request->getStatus() == RequestStatus::NoDataSource)
             raise("Failed to fetch sources: no matching data source.");
-        if (request->getStatus() == Aborted)
+        if (request->getStatus() == RequestStatus::Aborted)
             raise("Failed to fetch sources: request aborted.");
     }
 };

libs/http-service/src/http-service.cpp

@@ -162,6 +162,7 @@ struct HttpService::Impl
         std::mutex mutex_;
         std::condition_variable resultEvent_;
 
+        uint64_t requestId_;
         std::stringstream buffer_;
         std::string responseType_;
         std::unique_ptr<TileLayerStream::Writer> writer_;
@@ -170,9 +171,11 @@ struct HttpService::Impl
 
         HttpTilesRequestState()
         {
+            static std::atomic_uint64_t nextRequestId;
             writer_ = std::make_unique<TileLayerStream::Writer>(
                 [&, this](auto&& msg, auto&& msgType) { buffer_ << msg; },
                 stringOffsets_);
+            requestId_ = nextRequestId++;
         }
 
         void parseRequestFromJson(nlohmann::json const& requestJson)
@@ -217,6 +220,32 @@ struct HttpService::Impl
         }
     };
 
+    mutable std::mutex clientRequestMapMutex_;
+    mutable std::map<std::string, std::shared_ptr<HttpTilesRequestState>> requestStatePerClientId_;
+
+    void abortRequestsForClientId(std::string clientId, std::shared_ptr<HttpTilesRequestState> newState = nullptr) const
+    {
+        std::unique_lock clientRequestMapAccess(clientRequestMapMutex_);
+        auto clientRequestIt = requestStatePerClientId_.find(clientId);
+        if (clientRequestIt != requestStatePerClientId_.end()) {
+            // Ensure that any previous requests from the same clientId
+            // are finished post-haste!
+            bool anySoftAbort = false;
+            for (auto const& req : clientRequestIt->second->requests_) {
+                if (!req->isDone()) {
+                    self_.abort(req);
+                    anySoftAbort = true;
+                }
+            }
+            if (anySoftAbort)
+                log().warn("Soft-aborting tiles request {}", clientRequestIt->second->requestId_);
+            requestStatePerClientId_.erase(clientRequestIt);
+        }
+        if (newState) {
+            requestStatePerClientId_.emplace(clientId, newState);
+        }
+    }
+
     /**
      * Wraps around the generic mapget service's request() function
      * to include httplib request decoding and response encoding.
@@ -231,6 +260,7 @@ struct HttpService::Impl
         // Within one HTTP request, all requested tiles from the same map+layer
         // combination should be in a single LayerTilesRequest.
         auto state = std::make_shared<HttpTilesRequestState>();
+        log().info("Processing tiles request {}", state->requestId_);
         for (auto& requestJson : requestsJson) {
             state->parseRequestFromJson(requestJson);
         }
@@ -260,16 +290,22 @@ struct HttpService::Impl
             // Send a status report detailing for each request
             // whether its data source is unavailable or it was aborted.
             res.status = 400;
-            std::vector<int> requestStatuses{};
+            std::vector<std::underlying_type_t<RequestStatus>> requestStatuses{};
             for (const auto& r : state->requests_) {
-                requestStatuses.push_back(r->getStatus());
+                requestStatuses.push_back(static_cast<std::underlying_type_t<RequestStatus>>(r->getStatus()));
             }
             res.set_content(
                 nlohmann::json::object({{"requestStatuses", requestStatuses}}).dump(),
                 "application/json");
             return;
         }
 
+        // Parse/Process clientId.
+        if (j.contains("clientId")) {
+            auto clientId = j["clientId"].get<std::string>();
+            abortRequestsForClientId(clientId, state);
+        }
+
         // For efficiency, set up httplib to stream tile layer responses to client:
         // (1) Lambda continuously supplies response data to httplib's DataSink,
         //     picking up data from state->buffer_ until all tile requests are done.
@@ -319,16 +355,33 @@ struct HttpService::Impl
             // cleanup callback to abort the requests.
             [state, this](bool success)
             {
-                log().debug("Request finished, success: {}", success);
                 if (!success) {
+                    log().warn("Aborting tiles request {}", state->requestId_);
                     for (auto& request : state->requests_) {
                         self_.abort(request);
                     }
                 }
+                else {
+                    log().info("Tiles request {} was successful.", state->requestId_);
+                }
             });
     }
 
-    void handleSourcesRequest(const httplib::Request&, httplib::Response& res)
+    void handleAbortRequest(const httplib::Request& req, httplib::Response& res) const
+    {
+        // Parse the JSON request.
+        nlohmann::json j = nlohmann::json::parse(req.body);
+        if (j.contains("clientId")) {
+            auto const clientId = j["clientId"].get<std::string>();
+            abortRequestsForClientId(clientId);
+        }
+        else {
+            res.status = 400;
+            res.set_content("Missing clientId", "text/plain");
+        }
+    }
+
+    void handleSourcesRequest(const httplib::Request&, httplib::Response& res) const
     {
         auto sourcesInfo = nlohmann::json::array();
         for (auto& source : self_.info()) {
@@ -337,7 +390,7 @@ struct HttpService::Impl
         res.set_content(sourcesInfo.dump(), "application/json");
     }
 
-    void handleStatusRequest(const httplib::Request&, httplib::Response& res)
+    void handleStatusRequest(const httplib::Request&, httplib::Response& res) const
     {
         auto serviceStats = self_.getStatistics();
         auto cacheStats = self_.cache()->getStatistics();
@@ -358,7 +411,7 @@ struct HttpService::Impl
         res.set_content(oss.str(), "text/html");
     }
 
-    void handleLocateRequest(const httplib::Request& req, httplib::Response& res)
+    void handleLocateRequest(const httplib::Request& req, httplib::Response& res) const
     {
         // Parse the JSON request.
         nlohmann::json j = nlohmann::json::parse(req.body);
@@ -578,6 +631,11 @@ void HttpService::setup(httplib::Server& server)
         [&](const httplib::Request& req, httplib::Response& res)
         { impl_->handleTilesRequest(req, res); });
 
+    server.Post(
+        "/abort",
+        [&](const httplib::Request& req, httplib::Response& res)
+        { impl_->handleAbortRequest(req, res); });
+
     server.Get(
         "/sources",
         [this](const httplib::Request& req, httplib::Response& res)

libs/model/src/featurelayer.cpp

@@ -808,27 +808,29 @@ std::vector<IdPart> const& TileFeatureLayer::getPrimaryIdComposition(const std::
 
 void TileFeatureLayer::setStrings(std::shared_ptr<simfil::StringPool> const& newDict)
 {
+    auto oldDict = strings();
+    // Reset simfil environment and clear expression cache
+    impl_->expressionCache_.reset(makeEnvironment(newDict));
+    ModelPool::setStrings(newDict);
+    if (!oldDict || *newDict == *oldDict)
+        return;
+
     // Re-map old string IDs to new string IDs
     for (auto& attr : impl_->attributes_) {
-        if (auto resolvedName = strings()->resolve(attr.name_)) {
+        if (auto resolvedName = oldDict->resolve(attr.name_)) {
             attr.name_ = newDict->emplace(*resolvedName);
         }
     }
     for (auto& fid : impl_->featureIds_) {
-        if (auto resolvedName = strings()->resolve(fid.typeId_)) {
+        if (auto resolvedName = oldDict->resolve(fid.typeId_)) {
             fid.typeId_ = newDict->emplace(*resolvedName);
         }
     }
     for (auto& rel : impl_->relations_) {
-        if (auto resolvedName = strings()->resolve(rel.name_)) {
+        if (auto resolvedName = oldDict->resolve(rel.name_)) {
             rel.name_ = newDict->emplace(*resolvedName);
         }
     }
-
-    // Reset simfil environment and clear expression cache
-    impl_->expressionCache_.reset(makeEnvironment(newDict));
-
-    ModelPool::setStrings(newDict);
 }
 
 simfil::ModelNode::Ptr TileFeatureLayer::clone(

libs/service/include/mapget/service/service.h

@@ -13,7 +13,7 @@
 namespace mapget
 {
 
-enum RequestStatus {
+enum class RequestStatus {
     Open = 0x0,
     Success = 0x1, /** The request has been fully satisfied. */
     NoDataSource = 0x2, /** No data source could provide the requested map + layer. */
@@ -97,7 +97,7 @@ class LayerTilesRequest
     // Mutex/condition variable for reading/setting request status.
     std::mutex statusMutex_;
     std::condition_variable statusConditionVariable_;
-    RequestStatus status_ = Open;
+    RequestStatus status_ = RequestStatus::Open;
 };
 
 /**