SOLR-17192: Add "field-limiting" URP to catch ill-designed schemas

solr/core/src/java/org/apache/solr/update/processor/NumFieldLimitingUpdateRequestProcessor.java

@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.update.processor;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.Locale;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.update.AddUpdateCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class NumFieldLimitingUpdateRequestProcessor extends UpdateRequestProcessor {
+  private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+  private SolrQueryRequest req;
+  private int fieldThreshold;
+  private int currentNumFields;
+  private boolean warnOnly;
+
+  public NumFieldLimitingUpdateRequestProcessor(
+      SolrQueryRequest req,
+      UpdateRequestProcessor next,
+      int fieldThreshold,
+      int currentNumFields,
+      boolean warnOnly) {
+    super(next);
+    this.req = req;
+    this.fieldThreshold = fieldThreshold;
+    this.currentNumFields = currentNumFields;
+    this.warnOnly = warnOnly;
+  }
+
+  public void processAdd(AddUpdateCommand cmd) throws IOException {
+    if (coreExceedsFieldLimit()) {
+      throwExceptionOrLog(cmd.getPrintableId());
+    } else {
+      if (log.isDebugEnabled()) {
+        log.debug(
+            "Allowing document {}, since current core is under the 'maxFields' limit (numFields={}, maxFields={})",
+            cmd.getPrintableId(),
+            currentNumFields,
+            fieldThreshold);
+      }
+    }
+    super.processAdd(cmd);
+  }
+
+  protected boolean coreExceedsFieldLimit() {
+    return currentNumFields > fieldThreshold;
+  }
+
+  protected void throwExceptionOrLog(String id) {
+    final String messageSuffix = warnOnly ? "Blocking update of document " + id : "";
+    final String message =
+        String.format(
+            Locale.ROOT,
+            "Current core has %d fields, exceeding the max-fields limit of %d.  %s",
+            currentNumFields,
+            fieldThreshold,
+            messageSuffix);
+    if (warnOnly) {
+      log.warn(message);
+    } else {
+      throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, message);
+    }
+  }
+}

solr/core/src/java/org/apache/solr/update/processor/NumFieldLimitingUpdateRequestProcessorFactory.java

@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.update.processor;
+
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.util.plugin.SolrCoreAware;
+
+/**
+ * This factory generates an UpdateRequestProcessor which fails update requests once a core has
+ * exceeded a configurable maximum number of fields. Meant as a safeguard to help users notice
+ * potentially-dangerous schema design before performance and stability problems start to occur.
+ *
+ * <p>The URP uses the core's {@link SolrIndexSearcher} to judge the current number of fields.
+ * Accordingly, it undercounts the number of fields in the core - missing all fields added since the
+ * previous searcher was opened. As such, the URP's request-blocking is "best effort" - it cannot be
+ * relied on as a precise limit on the number of fields.
+ *
+ * <p>Additionally, the field-counting includes all documents present in the index, including any
+ * deleted docs that haven't yet been purged via segment merging. Note that this can differ
+ * significantly from the number of fields defined in managed-schema.xml - especially when dynamic
+ * fields are enabled. The only way to reduce this field count is to delete documents and wait until
+ * the deleted documents have been removed by segment merges. Users may of course speed up this
+ * process by tweaking Solr's segment-merging, triggering an "optimize" operation, etc.
+ *
+ * <p>{@link NumFieldLimitingUpdateRequestProcessorFactory} accepts two configuration parameters:
+ *
+ * <ul>
+ *   <li><code>maxFields</code> - (required) The maximum number of fields before update requests
+ *       should be aborted. Once this limit has been exceeded, additional update requests will fail
+ *       until fields have been removed or the "maxFields" is increased.
+ *   <li><code>warnOnly</code> - (optional) If <code>true</code> then the URP logs verbose warnings
+ *       about the limit being exceeded but doesn't abort update requests. Defaults to <code>false
+ *       </code> if not specified
+ * </ul>
+ *
+ * @since 9.6.0
+ */
+public class NumFieldLimitingUpdateRequestProcessorFactory extends UpdateRequestProcessorFactory
+    implements SolrCoreAware {
+
+  private static final String MAXIMUM_FIELDS_PARAM = "maxFields";
+  private static final String WARN_ONLY_PARAM = "warnOnly";
+
+  private NumFieldsMonitor numFieldsMonitor;
+  private int maximumFields;
+  private boolean warnOnly;
+
+  @Override
+  public void inform(final SolrCore core) {
+    // register a commit callback for monitoring the number of fields in the schema
+    numFieldsMonitor = new NumFieldsMonitor(core);
+    core.getUpdateHandler().registerCommitCallback(numFieldsMonitor);
+    core.registerNewSearcherListener(numFieldsMonitor);
+  }
+
+  @Override
+  public void init(NamedList<?> args) {
+    warnOnly = args.indexOf(WARN_ONLY_PARAM, 0) > 0 ? args.getBooleanArg(WARN_ONLY_PARAM) : false;
+
+    if (args.indexOf(MAXIMUM_FIELDS_PARAM, 0) < 0) {
+      throw new IllegalArgumentException(
+          "The "
+              + MAXIMUM_FIELDS_PARAM
+              + " parameter is required for "
+              + getClass().getName()
+              + ", but no value was provided.");
+    }
+    final Object rawMaxFields = args.get(MAXIMUM_FIELDS_PARAM);
+    if (rawMaxFields == null || !(rawMaxFields instanceof Integer)) {
+      throw new IllegalArgumentException(
+          MAXIMUM_FIELDS_PARAM + " must be configured as a non-null <int>");
+    }
+    maximumFields = (Integer) rawMaxFields;
+    if (maximumFields <= 0) {
+      throw new IllegalArgumentException(MAXIMUM_FIELDS_PARAM + " must be a positive integer");
+    }
+  }
+
+  @Override
+  public UpdateRequestProcessor getInstance(
+      SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor next) {
+
+    // TODO Should we skip to the next URP if running in SolrCloud and *not* a leader?
+    return new NumFieldLimitingUpdateRequestProcessor(
+        req, next, maximumFields, numFieldsMonitor.getCurrentNumFields(), warnOnly);
+  }
+
+  public int getFieldThreshold() {
+    return maximumFields;
+  }
+
+  public boolean getWarnOnly() {
+    return warnOnly;
+  }
+}

solr/core/src/java/org/apache/solr/update/processor/NumFieldsMonitor.java

@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.update.processor;
+
+import org.apache.solr.core.AbstractSolrEventListener;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.util.RefCounted;
+
+public class NumFieldsMonitor extends AbstractSolrEventListener {
+
+  private int numFields;
+
+  public NumFieldsMonitor(SolrCore core) {
+    super(core);
+    this.numFields = -1;
+  }
+
+  @Override
+  public void postCommit() {
+    RefCounted<SolrIndexSearcher> indexSearcher = null;
+    try {
+      indexSearcher = getCore().getSearcher();
+      // Get the number of fields directly from the IndexReader instead of the Schema object to also
+      // include the fields that are missing from the Schema file (dynamic/deleted etc.)
+      numFields = indexSearcher.get().getFieldInfos().size();
+    } finally {
+      if (indexSearcher != null) {
+        indexSearcher.decref();
+      }
+    }
+  }
+
+  @Override
+  public void newSearcher(SolrIndexSearcher newSearcher, SolrIndexSearcher currentSearcher) {
+    numFields = newSearcher.getFieldInfos().size();
+  }
+
+  public int getCurrentNumFields() {
+    return numFields;
+  }
+}

solr/core/src/test-files/solr/configsets/cloud-minimal-field-limiting/conf/schema.xml

@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<schema name="minimal" version="1.1">
+  <fieldType name="string" class="solr.StrField"/>
+  <fieldType name="int" class="${solr.tests.IntegerFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+  <fieldType name="long" class="${solr.tests.LongFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+  <fieldType name="text_general" class="solr.TextField" positionIncrementGap="100" multiValued="true" />
+  <dynamicField name="*" type="string" indexed="true" stored="true"/>
+  <!-- for versioning -->
+  <field name="_version_" type="long" indexed="true" stored="true"/>
+  <field name="_root_" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
+  <field name="id" type="string" indexed="true" stored="true"/>
+  <dynamicField name="*_s"  type="string"  indexed="true"  stored="true" />
+  <uniqueKey>id</uniqueKey>
+</schema>

solr/core/src/test-files/solr/configsets/cloud-minimal-field-limiting/conf/solrconfig.xml

@@ -0,0 +1,73 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Minimal solrconfig.xml with /select, /admin and /update only -->
+
+<config>
+
+  <dataDir>${solr.data.dir:}</dataDir>
+
+  <directoryFactory name="DirectoryFactory"
+                    class="${solr.directoryFactory:solr.NRTCachingDirectoryFactory}"/>
+  <schemaFactory class="ClassicIndexSchemaFactory"/>
+
+  <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
+
+  <updateHandler class="solr.DirectUpdateHandler2">
+    <commitWithin>
+      <softCommit>${solr.commitwithin.softcommit:true}</softCommit>
+    </commitWithin>
+    <updateLog class="${solr.ulog:solr.UpdateLog}"></updateLog>
+  </updateHandler>
+
+  <requestHandler name="/select" class="solr.SearchHandler">
+    <lst name="defaults">
+      <str name="echoParams">explicit</str>
+      <str name="indent">true</str>
+      <str name="df">text</str>
+    </lst>
+  </requestHandler>
+
+  <updateProcessor class="solr.NumFieldLimitingUpdateRequestProcessorFactory" name="max-fields">
+    <int name="maxFields">${solr.test.maxFields:1234}</int>
+  </updateProcessor>
+  <updateProcessor class="solr.AddSchemaFieldsUpdateProcessorFactory" name="add-schema-fields">
+    <lst name="typeMapping">
+      <str name="valueClass">java.lang.String</str>
+      <str name="fieldType">text_general</str>
+      <lst name="copyField">
+        <str name="dest">*_str</str>
+        <int name="maxChars">256</int>
+      </lst>
+      <!-- Use as default mapping instead of defaultFieldType -->
+      <bool name="default">true</bool>
+    </lst>
+  </updateProcessor>
+  <updateRequestProcessorChain name="add-unknown-fields-to-the-schema" default="true"
+           processor="max-fields">
+    <processor class="solr.LogUpdateProcessorFactory"/>
+    <processor class="solr.DistributedUpdateProcessorFactory"/>
+    <processor class="solr.RunUpdateProcessorFactory"/>
+  </updateRequestProcessorChain>
+
+  <indexConfig>
+    <mergeScheduler class="${solr.mscheduler:org.apache.lucene.index.ConcurrentMergeScheduler}"/>
+  </indexConfig>
+</config>
+