feat(structuredProperties): add hide property and show as badge validators

entity-registry/src/main/java/com/linkedin/metadata/entity/GenericScrollIterator.java

@@ -0,0 +1,35 @@
+package com.linkedin.metadata.entity;
+
+import com.linkedin.metadata.query.filter.Filter;
+import com.linkedin.metadata.search.ScrollResult;
+import java.util.Iterator;
+import java.util.List;
+import javax.annotation.Nonnull;
+import lombok.Builder;
+
+/**
+ * Fetches pages of structured properties which have been applied to an entity urn with a specified
+ * filter
+ */
+@Builder
+public class GenericScrollIterator implements Iterator<ScrollResult> {
+  @Nonnull private final Filter filter;
+  @Nonnull private final List<String> entities;
+  @Nonnull private final SearchRetriever searchRetriever;
+  private int count;
+  @Builder.Default private String scrollId = null;
+  @Builder.Default private boolean started = false;
+
+  @Override
+  public boolean hasNext() {
+    return !started || scrollId != null;
+  }
+
+  @Override
+  public ScrollResult next() {
+    started = true;
+    ScrollResult result = searchRetriever.scroll(entities, filter, scrollId, count);
+    scrollId = result.getScrollId();
+    return result;
+  }
+}

metadata-io/src/main/java/com/linkedin/metadata/structuredproperties/hooks/PropertyDefinitionDeleteSideEffect.java

@@ -3,8 +3,6 @@
 import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTIES_ASPECT_NAME;
 import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME;
 import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_KEY_ASPECT_NAME;
-import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_MAPPING_FIELD_PREFIX;
-import static com.linkedin.metadata.utils.CriterionUtils.buildExistsCriterion;
 
 import com.linkedin.common.AuditStamp;
 import com.linkedin.common.urn.Urn;
@@ -17,30 +15,19 @@
 import com.linkedin.metadata.aspect.patch.PatchOperationType;
 import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig;
 import com.linkedin.metadata.aspect.plugins.hooks.MCPSideEffect;
-import com.linkedin.metadata.entity.SearchRetriever;
 import com.linkedin.metadata.entity.ebean.batch.PatchItemImpl;
 import com.linkedin.metadata.models.EntitySpec;
-import com.linkedin.metadata.models.StructuredPropertyUtils;
-import com.linkedin.metadata.query.filter.ConjunctiveCriterion;
-import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray;
-import com.linkedin.metadata.query.filter.Criterion;
-import com.linkedin.metadata.query.filter.CriterionArray;
-import com.linkedin.metadata.query.filter.Filter;
-import com.linkedin.metadata.search.ScrollResult;
+import com.linkedin.metadata.structuredproperties.util.EntityWithPropertyIterator;
 import com.linkedin.structured.StructuredPropertyDefinition;
 import java.util.Collection;
-import java.util.Collections;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Spliterator;
 import java.util.Spliterators;
-import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import java.util.stream.StreamSupport;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
-import lombok.Builder;
 import lombok.Getter;
 import lombok.Setter;
 import lombok.experimental.Accessors;
@@ -141,60 +128,4 @@ private static Stream<MCPItem> generatePatchMCPs(
                               .build(retrieverContext.getAspectRetriever().getEntityRegistry());
                         }));
   }
-
-  /**
-   * Fetches pages of entity urns which have a value for the given structured property definition
-   */
-  @Builder
-  public static class EntityWithPropertyIterator implements Iterator<ScrollResult> {
-    @Nonnull private final Urn propertyUrn;
-    @Nullable private final StructuredPropertyDefinition definition;
-    @Nonnull private final SearchRetriever searchRetriever;
-    private int count;
-    @Builder.Default private String scrollId = null;
-    @Builder.Default private boolean started = false;
-
-    private List<String> getEntities() {
-      if (definition != null && definition.getEntityTypes() != null) {
-        return definition.getEntityTypes().stream()
-            .map(StructuredPropertyUtils::getValueTypeId)
-            .collect(Collectors.toList());
-      } else {
-        return Collections.emptyList();
-      }
-    }
-
-    private Filter getFilter() {
-      Filter propertyFilter = new Filter();
-      final ConjunctiveCriterionArray disjunction = new ConjunctiveCriterionArray();
-      final ConjunctiveCriterion conjunction = new ConjunctiveCriterion();
-      final CriterionArray andCriterion = new CriterionArray();
-
-      // Cannot rely on automatic field name since the definition is deleted
-      final Criterion propertyExistsCriterion =
-          buildExistsCriterion(
-              STRUCTURED_PROPERTY_MAPPING_FIELD_PREFIX
-                  + StructuredPropertyUtils.toElasticsearchFieldName(propertyUrn, definition));
-
-      andCriterion.add(propertyExistsCriterion);
-      conjunction.setAnd(andCriterion);
-      disjunction.add(conjunction);
-      propertyFilter.setOr(disjunction);
-
-      return propertyFilter;
-    }
-
-    @Override
-    public boolean hasNext() {
-      return !started || scrollId != null;
-    }
-
-    @Override
-    public ScrollResult next() {
-      started = true;
-      ScrollResult result = searchRetriever.scroll(getEntities(), getFilter(), scrollId, count);
-      scrollId = result.getScrollId();
-      return result;
-    }
-  }
 }

metadata-io/src/main/java/com/linkedin/metadata/structuredproperties/util/EntityWithPropertyIterator.java

@@ -0,0 +1,76 @@
+package com.linkedin.metadata.structuredproperties.util;
+
+import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_MAPPING_FIELD_PREFIX;
+import static com.linkedin.metadata.utils.CriterionUtils.buildExistsCriterion;
+
+import com.linkedin.common.urn.Urn;
+import com.linkedin.metadata.entity.SearchRetriever;
+import com.linkedin.metadata.models.StructuredPropertyUtils;
+import com.linkedin.metadata.query.filter.ConjunctiveCriterion;
+import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray;
+import com.linkedin.metadata.query.filter.Criterion;
+import com.linkedin.metadata.query.filter.CriterionArray;
+import com.linkedin.metadata.query.filter.Filter;
+import com.linkedin.metadata.search.ScrollResult;
+import com.linkedin.structured.StructuredPropertyDefinition;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.stream.Collectors;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import lombok.Builder;
+
+/** Fetches pages of entity urns which have a value for the given structured property definition */
+@Builder
+public class EntityWithPropertyIterator implements Iterator<ScrollResult> {
+  @Nonnull private final Urn propertyUrn;
+  @Nullable private final StructuredPropertyDefinition definition;
+  @Nonnull private final SearchRetriever searchRetriever;
+  private int count;
+  @Builder.Default private String scrollId = null;
+  @Builder.Default private boolean started = false;
+
+  private List<String> getEntities() {
+    if (definition != null && definition.getEntityTypes() != null) {
+      return definition.getEntityTypes().stream()
+          .map(StructuredPropertyUtils::getValueTypeId)
+          .collect(Collectors.toList());
+    } else {
+      return Collections.emptyList();
+    }
+  }
+
+  private Filter getFilter() {
+    Filter propertyFilter = new Filter();
+    final ConjunctiveCriterionArray disjunction = new ConjunctiveCriterionArray();
+    final ConjunctiveCriterion conjunction = new ConjunctiveCriterion();
+    final CriterionArray andCriterion = new CriterionArray();
+
+    // Cannot rely on automatic field name since the definition is deleted
+    final Criterion propertyExistsCriterion =
+        buildExistsCriterion(
+            STRUCTURED_PROPERTY_MAPPING_FIELD_PREFIX
+                + StructuredPropertyUtils.toElasticsearchFieldName(propertyUrn, definition));
+
+    andCriterion.add(propertyExistsCriterion);
+    conjunction.setAnd(andCriterion);
+    disjunction.add(conjunction);
+    propertyFilter.setOr(disjunction);
+
+    return propertyFilter;
+  }
+
+  @Override
+  public boolean hasNext() {
+    return !started || scrollId != null;
+  }
+
+  @Override
+  public ScrollResult next() {
+    started = true;
+    ScrollResult result = searchRetriever.scroll(getEntities(), getFilter(), scrollId, count);
+    scrollId = result.getScrollId();
+    return result;
+  }
+}

metadata-io/src/main/java/com/linkedin/metadata/structuredproperties/validation/HidePropertyValidator.java

@@ -0,0 +1,64 @@
+package com.linkedin.metadata.structuredproperties.validation;
+
+import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_SETTINGS_ASPECT_NAME;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.linkedin.metadata.aspect.RetrieverContext;
+import com.linkedin.metadata.aspect.batch.BatchItem;
+import com.linkedin.metadata.aspect.batch.ChangeMCP;
+import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig;
+import com.linkedin.metadata.aspect.plugins.validation.AspectPayloadValidator;
+import com.linkedin.metadata.aspect.plugins.validation.AspectValidationException;
+import com.linkedin.metadata.aspect.plugins.validation.ValidationExceptionCollection;
+import com.linkedin.metadata.models.StructuredPropertyUtils;
+import com.linkedin.structured.StructuredPropertySettings;
+import java.util.Collection;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import javax.annotation.Nonnull;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.experimental.Accessors;
+import lombok.extern.slf4j.Slf4j;
+import org.jetbrains.annotations.NotNull;
+
+@Setter
+@Getter
+@Slf4j
+@Accessors(chain = true)
+public class HidePropertyValidator extends AspectPayloadValidator {
+
+  @Nonnull private AspectPluginConfig config;
+
+  @Override
+  protected Stream<AspectValidationException> validateProposedAspects(
+      @NotNull Collection<? extends BatchItem> mcpItems,
+      @NotNull RetrieverContext retrieverContext) {
+    return validateSettingsUpserts(
+        mcpItems.stream()
+            .filter(i -> STRUCTURED_PROPERTY_SETTINGS_ASPECT_NAME.equals(i.getAspectName()))
+            .collect(Collectors.toList()));
+  }
+
+  @Override
+  protected Stream<AspectValidationException> validatePreCommitAspects(
+      @NotNull Collection<ChangeMCP> changeMCPs, @NotNull RetrieverContext retrieverContext) {
+    return Stream.empty();
+  }
+
+  @VisibleForTesting
+  public static Stream<AspectValidationException> validateSettingsUpserts(
+      @NotNull Collection<? extends BatchItem> mcpItems) {
+    ValidationExceptionCollection exceptions = ValidationExceptionCollection.newCollection();
+    for (BatchItem mcpItem : mcpItems) {
+      StructuredPropertySettings structuredPropertySettings =
+          mcpItem.getAspect(StructuredPropertySettings.class);
+      boolean isValid =
+          StructuredPropertyUtils.validatePropertySettings(structuredPropertySettings, false);
+      if (!isValid) {
+        exceptions.addException(mcpItem, StructuredPropertyUtils.INVALID_SETTINGS_MESSAGE);
+      }
+    }
+    return exceptions.streamAllExceptions();
+  }
+}