Add entity groups

changelog/unreleased/pr-19547.toml

@@ -0,0 +1,5 @@
+type = "a"
+message = "Added entity groups feature."
+
+issues = ["graylog-plugin-enterprise#7385"]
+pulls = ["19547"]

graylog2-server/src/main/java/org/graylog2/audit/AuditEventTypes.java

@@ -167,6 +167,10 @@ public class AuditEventTypes implements PluginAuditEventTypes {
     public static final String TELEMETRY_USER_SETTINGS_UPDATE = PREFIX + "telemetry_user_settings:update";
     public static final String CONTENT_STREAM_USER_SETTINGS_UPDATE = PREFIX + "content_stream_user_settings:update";
 
+    public static final String ENTITY_GROUP_CREATE = PREFIX + "category:create";
+    public static final String ENTITY_GROUP_UPDATE = PREFIX + "category:update";
+    public static final String ENTITY_GROUP_DELETE = PREFIX + "category:delete";
+
     private static final ImmutableSet<String> EVENT_TYPES = ImmutableSet.<String>builder()
             .add(ALARM_CALLBACK_CREATE)
             .add(ALARM_CALLBACK_DELETE)
@@ -309,6 +313,9 @@ public class AuditEventTypes implements PluginAuditEventTypes {
             .add(TELEMETRY_USER_SETTINGS_UPDATE)
             .add(CONTENT_STREAM_USER_SETTINGS_UPDATE)
             .add(CERTIFICATE_RENEWAL_MANUALLY_INITIATED)
+            .add(ENTITY_GROUP_CREATE)
+            .add(ENTITY_GROUP_UPDATE)
+            .add(ENTITY_GROUP_DELETE)
             .build();
 
     @Override

graylog2-server/src/main/java/org/graylog2/entitygroups/EntityGroupService.java

@@ -0,0 +1,80 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog2.entitygroups;
+
+import org.graylog2.entitygroups.model.EntityGroup;
+import org.graylog2.entitygroups.model.DBEntityGroupService;
+import org.graylog2.database.PaginatedList;
+import org.graylog2.entitygroups.model.EntityType;
+import org.graylog2.rest.models.SortOrder;
+
+import jakarta.inject.Inject;
+
+import jakarta.ws.rs.NotFoundException;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.function.Predicate;
+
+public class EntityGroupService {
+    private final DBEntityGroupService dbEntityGroupService;
+
+    @Inject
+    public EntityGroupService(DBEntityGroupService dbEntityGroupService) {
+        this.dbEntityGroupService = dbEntityGroupService;
+    }
+
+    public PaginatedList<EntityGroup> findPaginated(String query, int page, int perPage, SortOrder order,
+                                                    String sortByField, Predicate<EntityGroup> filter) {
+
+        return dbEntityGroupService.findPaginated(query, page, perPage, order.toBsonSort(sortByField), filter);
+    }
+
+    public Optional<EntityGroup> getByName(String groupName) {
+        return dbEntityGroupService.getByName(groupName);
+    }
+
+    public List<EntityGroup> getAllForEntity(EntityType type, String entityId) {
+        return dbEntityGroupService.getAllForEntity(type, entityId);
+    }
+
+    public EntityGroup create(EntityGroup group) {
+        return dbEntityGroupService.save(group);
+    }
+
+    public EntityGroup update(String id, EntityGroup group) {
+        if (dbEntityGroupService.get(id).isEmpty()) {
+            throw new NotFoundException("Unable to find entity group to update");
+        }
+        return dbEntityGroupService.save(group);
+
+    }
+
+    public EntityGroup addEntityToGroup(String groupId, EntityType type, String entityId) {
+        final EntityGroup group = requireEntityGroup(groupId);
+        return dbEntityGroupService.save(group.addEntity(type, entityId));
+    }
+
+    public long delete(String id) {
+        return dbEntityGroupService.delete(id);
+    }
+
+    public EntityGroup requireEntityGroup(String id) {
+        return dbEntityGroupService.get(id)
+                .orElseThrow(() -> new IllegalArgumentException("Unable to find entity group to update"));
+    }
+}

graylog2-server/src/main/java/org/graylog2/entitygroups/contentpacks/EntityGroupFacade.java

@@ -0,0 +1,68 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog2.entitygroups.contentpacks;
+
+import org.graylog2.contentpacks.EntityDescriptorIds;
+import org.graylog2.contentpacks.facades.EntityFacade;
+import org.graylog2.contentpacks.model.ModelType;
+import org.graylog2.contentpacks.model.entities.Entity;
+import org.graylog2.contentpacks.model.entities.EntityDescriptor;
+import org.graylog2.contentpacks.model.entities.EntityExcerpt;
+import org.graylog2.contentpacks.model.entities.NativeEntity;
+import org.graylog2.contentpacks.model.entities.NativeEntityDescriptor;
+import org.graylog2.contentpacks.model.entities.references.ValueReference;
+import org.graylog2.entitygroups.model.EntityGroup;
+import org.graylog2.plugin.indexer.searches.timeranges.InvalidRangeParametersException;
+
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+
+// TODO: Implement methods
+public class EntityGroupFacade implements EntityFacade<EntityGroup> {
+    public static final ModelType TYPE_V1 = ModelType.of("entity_group", "1");
+
+    @Override
+    public Optional<Entity> exportEntity(EntityDescriptor entityDescriptor, EntityDescriptorIds entityDescriptorIds) {
+        return Optional.empty();
+    }
+
+    @Override
+    public NativeEntity<EntityGroup> createNativeEntity(Entity entity, Map<String, ValueReference> parameters, Map<EntityDescriptor, Object> nativeEntities, String username) throws InvalidRangeParametersException {
+        return null;
+    }
+
+    @Override
+    public Optional<NativeEntity<EntityGroup>> loadNativeEntity(NativeEntityDescriptor nativeEntityDescriptor) {
+        return Optional.empty();
+    }
+
+    @Override
+    public void delete(EntityGroup nativeEntity) {
+
+    }
+
+    @Override
+    public EntityExcerpt createExcerpt(EntityGroup nativeEntity) {
+        return null;
+    }
+
+    @Override
+    public Set<EntityExcerpt> listEntityExcerpts() {
+        return Set.of();
+    }
+}

graylog2-server/src/main/java/org/graylog2/entitygroups/contentpacks/entities/EntityGroupEntity.java

@@ -0,0 +1,89 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog2.entitygroups.contentpacks.entities;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.google.auto.value.AutoValue;
+import com.google.common.graph.MutableGraph;
+import org.graylog2.contentpacks.NativeEntityConverter;
+import org.graylog2.contentpacks.model.entities.Entity;
+import org.graylog2.contentpacks.model.entities.EntityDescriptor;
+import org.graylog2.contentpacks.model.entities.EntityV1;
+import org.graylog2.contentpacks.model.entities.ScopedContentPackEntity;
+import org.graylog2.contentpacks.model.entities.references.ValueReference;
+import org.graylog2.entitygroups.model.EntityGroup;
+import org.graylog2.entitygroups.model.EntityType;
+
+import java.util.List;
+import java.util.Map;
+
+import static org.graylog2.entitygroups.model.EntityGroup.FIELD_ENTITIES;
+import static org.graylog2.entitygroups.model.EntityGroup.FIELD_NAME;
+
+@JsonAutoDetect
+@AutoValue
+@JsonDeserialize(builder = EntityGroupEntity.Builder.class)
+public abstract class EntityGroupEntity extends ScopedContentPackEntity implements NativeEntityConverter<EntityGroup> {
+    @JsonProperty(FIELD_NAME)
+    public abstract String name();
+
+    @JsonProperty(FIELD_ENTITIES)
+    public abstract Map<EntityType, List<String>> entities();
+
+    public static Builder builder() {
+        return Builder.create();
+    }
+
+    public abstract Builder toBuilder();
+
+    @AutoValue.Builder
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    public abstract static class Builder extends ScopedContentPackEntity.AbstractBuilder<Builder> {
+        @JsonProperty(FIELD_NAME)
+        public abstract Builder name(String name);
+
+        @JsonProperty(FIELD_ENTITIES)
+        public abstract Builder entities(Map<EntityType, List<String>> entities);
+
+        public abstract EntityGroupEntity build();
+
+        @JsonCreator
+        public static Builder create() {
+            return new AutoValue_EntityGroupEntity.Builder();
+        }
+    }
+
+    @Override
+    public EntityGroup toNativeEntity(Map<String, ValueReference> parameters,
+                                               Map<EntityDescriptor, Object> nativeEntities) {
+        // TODO: Need to convert content pack IDs to DB object IDs for the entities map here.
+        return null;
+    }
+
+    @Override
+    public void resolveForInstallation(EntityV1 entity,
+                                       Map<String, ValueReference> parameters,
+                                       Map<EntityDescriptor, Entity> entities,
+                                       MutableGraph<Entity> graph) {
+        // TODO: Need to flag all entity dependencies in the entities map so that they get installed first.
+        // TODO: They will need to exist in the DB with IDs we can reference once this `toNativeEntity` is called.
+    }
+}

graylog2-server/src/main/java/org/graylog2/entitygroups/model/DBEntityGroupService.java

@@ -0,0 +1,115 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog2.entitygroups.model;
+
+import com.google.common.collect.ImmutableMap;
+import com.mongodb.BasicDBObject;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.model.Collation;
+import com.mongodb.client.model.CollationStrength;
+import com.mongodb.client.model.IndexOptions;
+import org.bson.conversions.Bson;
+import org.graylog2.database.MongoCollections;
+import org.graylog2.database.PaginatedList;
+import org.graylog2.database.entities.EntityScopeService;
+import org.graylog2.database.pagination.MongoPaginationHelper;
+import org.graylog2.database.utils.MongoUtils;
+import org.graylog2.database.utils.ScopedEntityMongoUtils;
+import org.graylog2.search.SearchQuery;
+import org.graylog2.search.SearchQueryField;
+import org.graylog2.search.SearchQueryParser;
+
+import jakarta.inject.Inject;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.function.Predicate;
+
+import static com.mongodb.client.model.Filters.and;
+import static com.mongodb.client.model.Filters.eq;
+import static com.mongodb.client.model.Filters.exists;
+import static com.mongodb.client.model.Filters.in;
+
+public class DBEntityGroupService {
+    public static final String COLLECTION_NAME = "entity_groups";
+
+    private static final ImmutableMap<String, SearchQueryField> ALLOWED_FIELDS = ImmutableMap.<String, SearchQueryField>builder()
+            .put(EntityGroup.FIELD_NAME, SearchQueryField.create(EntityGroup.FIELD_NAME))
+            .build();
+
+    private final MongoCollection<EntityGroup> collection;
+    private final MongoUtils<EntityGroup> mongoUtils;
+    private final ScopedEntityMongoUtils<EntityGroup> scopedEntityMongoUtils;
+    private final MongoPaginationHelper<EntityGroup> paginationHelper;
+    private final SearchQueryParser searchQueryParser;
+
+    @Inject
+    public DBEntityGroupService(MongoCollections mongoCollections,
+                                EntityScopeService entityScopeService) {
+        this.collection = mongoCollections.collection(COLLECTION_NAME, EntityGroup.class);
+        this.mongoUtils = mongoCollections.utils(collection);
+        this.scopedEntityMongoUtils = mongoCollections.scopedEntityUtils(collection, entityScopeService);
+        this.paginationHelper = mongoCollections.paginationHelper(collection);
+        this.searchQueryParser = new SearchQueryParser(EntityGroup.FIELD_ENTITIES, ALLOWED_FIELDS);
+
+        final IndexOptions caseInsensitiveOptions = new IndexOptions()
+                .collation(Collation.builder().locale("en").collationStrength(CollationStrength.SECONDARY).build())
+                .unique(true);
+        collection.createIndex(new BasicDBObject(EntityGroup.FIELD_NAME, 1), caseInsensitiveOptions);
+    }
+
+    public Optional<EntityGroup> get(String id) {
+        return mongoUtils.getById(id);
+    }
+
+    public PaginatedList<EntityGroup> findPaginated(String query, int page, int perPage, Bson sort, Predicate<EntityGroup> filter) {
+        final SearchQuery searchQuery = searchQueryParser.parse(query);
+        return filter == null ?
+                paginationHelper.filter(searchQuery.toBson()).sort(sort).perPage(perPage).page(page) :
+                paginationHelper.filter(searchQuery.toBson()).sort(sort).perPage(perPage).page(page, filter);
+    }
+
+    public EntityGroup save(EntityGroup EntityGroup) {
+        if (EntityGroup.id() != null) {
+            return scopedEntityMongoUtils.update(EntityGroup);
+        }
+        String newId = scopedEntityMongoUtils.create(EntityGroup);
+        return EntityGroup.toBuilder().id(newId).build();
+    }
+
+    public Optional<EntityGroup> getByName(String name) {
+        final Bson query = eq(EntityGroup.FIELD_NAME, name);
+
+        return Optional.ofNullable(collection.find(query).first());
+    }
+
+    public List<EntityGroup> getAllForEntity(EntityType type, String entityId) {
+        final Bson query = and(
+                exists(typeField(type)),
+                in(typeField(type), entityId)
+        );
+        return MongoUtils.stream(collection.find(query)).toList();
+    }
+
+    public long delete(String id) {
+        return collection.deleteOne(MongoUtils.idEq(id)).getDeletedCount();
+    }
+
+    private String typeField(EntityType type) {
+        return EntityGroup.FIELD_ENTITIES + "." + type.getName();
+    }
+}