Add entity groups

changelog/unreleased/pr-19547.toml

@@ -0,0 +1,5 @@
+type = "a"
+message = "Added entity groups feature."
+
+issues = ["graylog-plugin-enterprise#7385"]
+pulls = ["19547", "graylog-plugin-enterprise#7585"]

graylog2-server/src/main/java/org/graylog2/commands/Server.java

@@ -84,6 +84,7 @@
 import org.graylog2.database.entities.ScopedEntitiesModule;
 import org.graylog2.datatiering.DataTieringModule;
 import org.graylog2.decorators.DecoratorBindings;
+import org.graylog2.entitygroups.EntityGroupModule;
 import org.graylog2.featureflag.FeatureFlags;
 import org.graylog2.indexer.FieldTypeManagementModule;
 import org.graylog2.indexer.IndexerBindings;
@@ -215,7 +216,8 @@ protected List<Module> getCommandBindings(FeatureFlags featureFlags) {
                 new DataTieringModule(),
                 new DatanodeMigrationBindings(),
                 new CaModule(),
-                new TelemetryModule()
+                new TelemetryModule(),
+                new EntityGroupModule(featureFlags)
         );
 
         modules.add(new FieldTypeManagementModule());

graylog2-server/src/main/java/org/graylog2/entitygroups/EntityGroupAuditEventTypes.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog2.entitygroups;
+
+import com.google.common.collect.ImmutableSet;
+import org.graylog2.audit.PluginAuditEventTypes;
+
+import java.util.Set;
+
+public class EntityGroupAuditEventTypes implements PluginAuditEventTypes {
+    private static final String NAMESPACE = "entity_groups";
+    private static final String PREFIX = NAMESPACE + ":entity_group";
+
+    public static final String ENTITY_GROUP_CREATE = PREFIX + ":create";
+    public static final String ENTITY_GROUP_UPDATE = PREFIX + ":update";
+    public static final String ENTITY_GROUP_ADD_ENTITY = PREFIX + ":addEntity";
+    public static final String ENTITY_GROUP_DELETE = PREFIX + ":delete";
+
+    private static final Set<String> EVENT_TYPES = ImmutableSet.<String>builder()
+            .add(ENTITY_GROUP_CREATE)
+            .add(ENTITY_GROUP_UPDATE)
+            .add(ENTITY_GROUP_ADD_ENTITY)
+            .add(ENTITY_GROUP_DELETE)
+            .build();
+
+    @Override
+    public Set<String> auditEventTypes() {
+        return EVENT_TYPES;
+    }
+}

graylog2-server/src/main/java/org/graylog2/entitygroups/EntityGroupModule.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog2.entitygroups;
+
+import org.graylog2.entitygroups.contentpacks.EntityGroupFacade;
+import org.graylog2.entitygroups.rest.EntityGroupResource;
+import org.graylog2.featureflag.FeatureFlags;
+import org.graylog2.plugin.PluginModule;
+import org.graylog2.shared.utilities.StringUtils;
+
+public class EntityGroupModule extends PluginModule {
+    private static final String FEATURE_FLAG = "entity_groups";
+
+    private final FeatureFlags featureFlags;
+
+    public EntityGroupModule(FeatureFlags featureFlags) {
+        this.featureFlags = featureFlags;
+    }
+
+    @Override
+    protected void configure() {
+        if (featureFlags.getAll().keySet().stream()
+                .map(StringUtils::toLowerCase)
+                .anyMatch(s -> s.equals(FEATURE_FLAG)) && featureFlags.isOn(FEATURE_FLAG)) {
+            addSystemRestResource(EntityGroupResource.class);
+            addAuditEventTypes(EntityGroupAuditEventTypes.class);
+            addEntityFacade(EntityGroupFacade.TYPE_V1, EntityGroupFacade.class);
+        }
+    }
+}

graylog2-server/src/main/java/org/graylog2/entitygroups/EntityGroupService.java

@@ -0,0 +1,89 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog2.entitygroups;
+
+import org.graylog2.entitygroups.model.EntityGroup;
+import org.graylog2.entitygroups.model.DBEntityGroupService;
+import org.graylog2.database.PaginatedList;
+import org.graylog2.rest.models.SortOrder;
+
+import jakarta.inject.Inject;
+
+import jakarta.ws.rs.NotFoundException;
+
+import java.util.Collection;
+import java.util.Map;
+import java.util.Optional;
+import java.util.function.Predicate;
+import java.util.stream.Stream;
+
+public class EntityGroupService {
+    private final DBEntityGroupService dbEntityGroupService;
+
+    @Inject
+    public EntityGroupService(DBEntityGroupService dbEntityGroupService) {
+        this.dbEntityGroupService = dbEntityGroupService;
+    }
+
+    public PaginatedList<EntityGroup> findPaginated(String query, int page, int perPage, SortOrder order,
+                                                    String sortByField, Predicate<EntityGroup> filter) {
+
+        return dbEntityGroupService.findPaginated(query, page, perPage, order.toBsonSort(sortByField), filter);
+    }
+
+    public PaginatedList<EntityGroup> findPaginatedForEntity(String type, String entityId, int page, int perPage, SortOrder order,
+                                                             String sortByField, Predicate<EntityGroup> filter) {
+        return dbEntityGroupService.findPaginatedForEntity(type, entityId, page, perPage, order.toBsonSort(sortByField), filter);
+    }
+
+    public Optional<EntityGroup> getByName(String groupName) {
+        return dbEntityGroupService.getByName(groupName);
+    }
+
+    public Stream<EntityGroup> streamAllForEntity(String type, String entityId) {
+        return dbEntityGroupService.streamAllForEntity(type, entityId);
+    }
+
+    public Map<String, Collection<EntityGroup>> getAllForEntities(String type, Collection<String> entities) {
+        return dbEntityGroupService.getAllForEntities(type, entities).asMap();
+    }
+
+    public EntityGroup create(EntityGroup group) {
+        return dbEntityGroupService.save(group);
+    }
+
+    public EntityGroup update(String id, EntityGroup group) {
+        final EntityGroup saved = dbEntityGroupService.update(group.toBuilder().id(id).build());
+        if (saved == null) {
+            throw new NotFoundException("Unable to find mutable entity group to update");
+        }
+        return saved;
+    }
+
+    public EntityGroup addEntityToGroup(String groupId, String type, String entityId) {
+        return dbEntityGroupService.addEntityToGroup(groupId, type, entityId);
+    }
+
+    public long delete(String id) {
+        return dbEntityGroupService.delete(id);
+    }
+
+    public EntityGroup requireEntityGroup(String id) {
+        return dbEntityGroupService.get(id)
+                .orElseThrow(() -> new IllegalArgumentException("Unable to find entity group to update"));
+    }
+}

graylog2-server/src/main/java/org/graylog2/entitygroups/contentpacks/EntityGroupFacade.java

@@ -0,0 +1,68 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog2.entitygroups.contentpacks;
+
+import org.graylog2.contentpacks.EntityDescriptorIds;
+import org.graylog2.contentpacks.facades.EntityFacade;
+import org.graylog2.contentpacks.model.ModelType;
+import org.graylog2.contentpacks.model.entities.Entity;
+import org.graylog2.contentpacks.model.entities.EntityDescriptor;
+import org.graylog2.contentpacks.model.entities.EntityExcerpt;
+import org.graylog2.contentpacks.model.entities.NativeEntity;
+import org.graylog2.contentpacks.model.entities.NativeEntityDescriptor;
+import org.graylog2.contentpacks.model.entities.references.ValueReference;
+import org.graylog2.entitygroups.model.EntityGroup;
+import org.graylog2.plugin.indexer.searches.timeranges.InvalidRangeParametersException;
+
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+
+// TODO: Implement methods
+public class EntityGroupFacade implements EntityFacade<EntityGroup> {
+    public static final ModelType TYPE_V1 = ModelType.of("entity_group", "1");
+
+    @Override
+    public Optional<Entity> exportEntity(EntityDescriptor entityDescriptor, EntityDescriptorIds entityDescriptorIds) {
+        return Optional.empty();
+    }
+
+    @Override
+    public NativeEntity<EntityGroup> createNativeEntity(Entity entity, Map<String, ValueReference> parameters, Map<EntityDescriptor, Object> nativeEntities, String username) throws InvalidRangeParametersException {
+        return null;
+    }
+
+    @Override
+    public Optional<NativeEntity<EntityGroup>> loadNativeEntity(NativeEntityDescriptor nativeEntityDescriptor) {
+        return Optional.empty();
+    }
+
+    @Override
+    public void delete(EntityGroup nativeEntity) {
+
+    }
+
+    @Override
+    public EntityExcerpt createExcerpt(EntityGroup nativeEntity) {
+        return null;
+    }
+
+    @Override
+    public Set<EntityExcerpt> listEntityExcerpts() {
+        return Set.of();
+    }
+}

graylog2-server/src/main/java/org/graylog2/entitygroups/contentpacks/entities/EntityGroupEntity.java

@@ -0,0 +1,88 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog2.entitygroups.contentpacks.entities;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.google.auto.value.AutoValue;
+import com.google.common.graph.MutableGraph;
+import org.graylog2.contentpacks.NativeEntityConverter;
+import org.graylog2.contentpacks.model.entities.Entity;
+import org.graylog2.contentpacks.model.entities.EntityDescriptor;
+import org.graylog2.contentpacks.model.entities.EntityV1;
+import org.graylog2.contentpacks.model.entities.ScopedContentPackEntity;
+import org.graylog2.contentpacks.model.entities.references.ValueReference;
+import org.graylog2.entitygroups.model.EntityGroup;
+
+import java.util.List;
+import java.util.Map;
+
+import static org.graylog2.entitygroups.model.EntityGroup.FIELD_ENTITIES;
+import static org.graylog2.entitygroups.model.EntityGroup.FIELD_NAME;
+
+@JsonAutoDetect
+@AutoValue
+@JsonDeserialize(builder = EntityGroupEntity.Builder.class)
+public abstract class EntityGroupEntity extends ScopedContentPackEntity implements NativeEntityConverter<EntityGroup> {
+    @JsonProperty(FIELD_NAME)
+    public abstract String name();
+
+    @JsonProperty(FIELD_ENTITIES)
+    public abstract Map<String, List<String>> entities();
+
+    public static Builder builder() {
+        return Builder.create();
+    }
+
+    public abstract Builder toBuilder();
+
+    @AutoValue.Builder
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    public abstract static class Builder extends ScopedContentPackEntity.AbstractBuilder<Builder> {
+        @JsonProperty(FIELD_NAME)
+        public abstract Builder name(String name);
+
+        @JsonProperty(FIELD_ENTITIES)
+        public abstract Builder entities(Map<String, List<String>> entities);
+
+        public abstract EntityGroupEntity build();
+
+        @JsonCreator
+        public static Builder create() {
+            return new AutoValue_EntityGroupEntity.Builder();
+        }
+    }
+
+    @Override
+    public EntityGroup toNativeEntity(Map<String, ValueReference> parameters,
+                                               Map<EntityDescriptor, Object> nativeEntities) {
+        // TODO: Need to convert content pack IDs to DB object IDs for the entities map here.
+        return null;
+    }
+
+    @Override
+    public void resolveForInstallation(EntityV1 entity,
+                                       Map<String, ValueReference> parameters,
+                                       Map<EntityDescriptor, Entity> entities,
+                                       MutableGraph<Entity> graph) {
+        // TODO: Need to flag all entity dependencies in the entities map so that they get installed first.
+        // TODO: They will need to exist in the DB with IDs we can reference once this `toNativeEntity` is called.
+    }
+}