Merge branch 'moqui:master' into master

build.gradle

@@ -155,24 +155,33 @@ task startElasticSearch(type:Exec) {
 }
 */
 task startElasticSearch { doLast {
-    if (file(moquiRuntime + '/elasticsearch/bin').exists() && !file(moquiRuntime + '/elasticsearch/pid').exists()) {
+    def pidFile = file(moquiRuntime + '/elasticsearch/pid')
+    def esBinFile = file(moquiRuntime + '/elasticsearch/bin')
+    if (esBinFile.exists() && !pidFile.exists()) {
         logger.lifecycle("Starting ElasticSearch installed in runtime/elasticsearch")
         ProcessBuilder pb = new ProcessBuilder('./bin/elasticsearch', '-d', '-p', 'pid')
         pb.directory(file(moquiRuntime + '/elasticsearch'))
         pb.redirectOutput()
         pb.redirectError()
         pb.start()
+    } else {
+        if (pidFile.exists()) logger.lifecycle("Not Starting ElasticSearch installed in runtime/elasticsearch, pid file already exists")
+        if (!esBinFile.exists()) logger.lifecycle("Not Starting ElasticSearch, no runtime/elasticsearch/bin directory found")
     }
 } }
 task stopElasticSearch { doLast {
     String esDir = moquiRuntime + '/elasticsearch'
     def pidFile = file(moquiRuntime + '/elasticsearch/pid')
-    if (pidFile.exists() && file(moquiRuntime + '/elasticsearch/bin').exists()) {
+    def esBinFile = file(moquiRuntime + '/elasticsearch/bin')
+    if (pidFile.exists() && esBinFile.exists()) {
         String pid = pidFile.getText()
         logger.lifecycle("Stopping ElasticSearch installed in runtime/elasticsearch with pid ${pid}")
         exec { workingDir esDir; commandLine 'kill', pid }
         // don't bother waiting in this case: exec { workingDir esDir; commandLine 'tail', "--pid=${pid}", '-f', '/dev/null' }
         if (pidFile.exists()) delete pidFile
+    } else {
+        if (!pidFile.exists()) logger.lifecycle("Not Stopping ElasticSearch installed in runtime/elasticsearch, no pid file found")
+        if (!esBinFile.exists()) logger.lifecycle("Not Stopping ElasticSearch, no runtime/elasticsearch/bin directory found")
     }
 } }
 

framework/build.gradle

@@ -72,6 +72,7 @@ dependencies {
     //     and updated Spock required (tried 2.0-M2-groovy-3.0, not final release and tests didn't work)
     compile 'org.codehaus.groovy:groovy:2.5.15' // Apache 2.0
     compile 'org.codehaus.groovy:groovy-dateutil:2.5.15' // Apache 2.0
+    compile 'org.codehaus.groovy:groovy-groovysh:2.5.15' // Apache 2.0
     compile 'org.codehaus.groovy:groovy-json:2.5.15' // Apache 2.0
     compile 'org.codehaus.groovy:groovy-templates:2.5.15' // Apache 2.0
     compile 'org.codehaus.groovy:groovy-xml:2.5.15' // Apache 2.0
@@ -99,6 +100,7 @@ dependencies {
     compile 'commons-digester:commons-digester:2.1' // Apache 2.0
     compile 'commons-fileupload:commons-fileupload:1.4' // Apache 2.0
     compile 'commons-io:commons-io:2.11.0' // Apache 2.0
+    compile 'commons-net:commons-net:3.8.0' // Apache 2.0
     compile 'commons-logging:commons-logging:1.2' // Apache 2.0
     compile 'commons-validator:commons-validator:1.7' // Apache 2.0
 
@@ -132,20 +134,13 @@ dependencies {
     // Java TOTP
     compile 'dev.samstevens.totp:totp:1.7.1' // MIT
     // dev.samstevens.totp:totp depends on com.google.zxing:javase which depends on com.beust:jcommander, but an older version with a CVE, so specify latest to fix
-    compile 'com.beust:jcommander:1.81'
+    compile 'com.beust:jcommander:1.82'
 
     // H2 Database
-    // CVE pending: https://github.com/advisories/GHSA-7rpj-hg47-cx62 https://nvd.nist.gov/vuln/detail/CVE-2021-23463
-    // NOTE: Moqui does not use org.h2.jdbc.JdbcResultSet.getSQLXML() or any H2 specific API except to init server in H2ServerToolFactory
-    // NOTE: update to H2 2.0.202 works well, except VALUE is now a reserved word, problem for:
-    //    DbFormFieldAttribute, DbFormFieldEntOptsCond, PartyDimension, ProductDimension, ProductUomDimension
-    //    this can be addressed with something like column-name="DIM_VALUE" which fixes H2
-    //    but then all existing databases need data migration (move data from VALUE column to DIM_VALUE)
-    //    so, we need a different solution before updating H2
-    compile 'com.h2database:h2:1.4.200' // MPL 2.0, EPL 1.0
+    compile 'com.h2database:h2:2.1.210' // MPL 2.0, EPL 1.0
 
     // Jackson Databind (JSON, etc)
-    compile 'com.fasterxml.jackson.core:jackson-databind:2.12.5'
+    compile 'com.fasterxml.jackson.core:jackson-databind:2.13.2'
 
     // Jetty HTTP Client and Proxy Servlet
     compile 'org.eclipse.jetty:jetty-client:9.4.44.v20210927' // Apache 2.0
@@ -167,7 +162,7 @@ dependencies {
     compile module('org.apache.shiro:shiro-web:1.8.0') // Apache 2.0
 
     // SLF4J, Log4j 2 (note Log4j 2 is used by various libraries, best not to replace it even if mostly possible with SLF4J)
-    compile 'org.slf4j:slf4j-api:1.7.32'
+    compile 'org.slf4j:slf4j-api:1.7.35'
     compile 'org.apache.logging.log4j:log4j-core:2.17.1'
     compile 'org.apache.logging.log4j:log4j-api:2.17.1'
     runtime 'org.apache.logging.log4j:log4j-slf4j-impl:2.17.1'
@@ -177,7 +172,7 @@ dependencies {
     compile module('org.subethamail:subethasmtp:3.1.7')
 
     // Snake YAML
-    compile 'org.yaml:snakeyaml:1.29' // Apache 2.0
+    compile 'org.yaml:snakeyaml:1.30' // Apache 2.0
 
     // Apache Jackrabbit - uncomment here or include elsewhere when Jackrabbit repository configurations are used
     // compile 'org.apache.jackrabbit:jackrabbit-jcr-rmi:2.12.1' // Apache 2.0

framework/data/SecurityTypeData.xml

@@ -16,8 +16,9 @@ along with this software (see the LICENSE.md file). If not, see
     <moqui.basic.Enumeration description="Moqui Administrators" enumId="UgtMoquiAdmin" enumTypeId="UserGroupType"/>
     <moqui.basic.Enumeration description="Remote Systems" enumId="UgtRemoteSystems" enumTypeId="UserGroupType"/>
 
-    <!-- An group for full-access Administrators -->
+    <!-- User Group for full-access Administrators -->
     <moqui.security.UserGroup userGroupId="ADMIN" description="Administrators (full access)" groupTypeEnumId="UgtMoquiAdmin"/>
+    <moqui.security.UserGroup userGroupId="ADMIN_ADV" description="Administrators - Advanced" groupTypeEnumId="UgtMoquiAdmin"/>
 
     <!-- A default/automatic group for all users -->
     <moqui.security.UserGroup userGroupId="ALL_USERS" description="All Users (all users members by default)"/>
@@ -46,5 +47,11 @@ along with this software (see the LICENSE.md file). If not, see
     <!-- Special Permissions -->
     <moqui.security.UserPermission userPermissionId="ADMIN_PASSWORD" description="Admin Password Update"/>
     <moqui.security.UserGroupPermission userGroupId="ADMIN" userPermissionId="ADMIN_PASSWORD" fromDate="0"/>
+
     <moqui.security.UserPermission userPermissionId="ADMIN_LOGIN_AS" description="Admin Login As User"/>
+    <moqui.security.UserGroupPermission userGroupId="ADMIN_ADV" userPermissionId="ADMIN_LOGIN_AS" fromDate="0"/>
+    <moqui.security.UserPermission userPermissionId="SQL_RUNNER_WEB" description="Tools: SQL Runner"/>
+    <moqui.security.UserGroupPermission userGroupId="ADMIN_ADV" userPermissionId="SQL_RUNNER_WEB" fromDate="0"/>
+    <moqui.security.UserPermission userPermissionId="GROOVY_SHELL_WEB" description="Tools: Groovy Shell"/>
+    <moqui.security.UserGroupPermission userGroupId="ADMIN_ADV" userPermissionId="GROOVY_SHELL_WEB" fromDate="0"/>
 </entity-facade-xml>

framework/service/org/moqui/impl/EntityServices.xml

@@ -251,6 +251,7 @@ along with this software (see the LICENSE.md file). If not, see
         <in-parameters>
             <parameter name="fromDate" type="Timestamp"/>
             <parameter name="thruDate" type="Timestamp"/>
+            <parameter name="entitiesToInclude" type="List"><parameter name="entityName"/></parameter>
             <parameter name="entitiesToSkip" type="List"><parameter name="entityName"/></parameter>
             <parameter name="baseFilename"/>
             <parameter name="fileType" default-value="XML"/>
@@ -264,10 +265,15 @@ along with this software (see the LICENSE.md file). If not, see
                 import org.moqui.entity.EntityDataWriter
                 import org.moqui.context.ExecutionContext
                 ExecutionContext ec = context.ec
-                EntityDataWriter edw = ec.entity.makeDataWriter().allEntities()
+                EntityDataWriter edw = ec.entity.makeDataWriter()
 
                 if (fromDate) edw.fromDate(ec.l10n.parseTimestamp(fromDate, null))
                 if (thruDate) edw.thruDate(ec.l10n.parseTimestamp(thruDate, null))
+                if (entitiesToInclude) {
+                    edw.entityNames(entitiesToInclude)
+                } else {
+                    edw.allEntities()
+                }
                 if (entitiesToSkip) edw.skipEntityNames(entitiesToSkip)
 
                 edw.fileType((String) fileType)

framework/src/main/groovy/org/moqui/impl/context/ExecutionContextFactoryImpl.groovy

@@ -106,7 +106,7 @@ class ExecutionContextFactoryImpl implements ExecutionContextFactory {
 
     protected LinkedHashMap<String, ComponentInfo> componentInfoMap = new LinkedHashMap<>()
     public final ThreadLocal<ExecutionContextImpl> activeContext = new ThreadLocal<>()
-    protected final Map<Long, ExecutionContextImpl> activeContextMap = new HashMap<>()
+    public final Map<Long, ExecutionContextImpl> activeContextMap = new HashMap<>()
     protected final LinkedHashMap<String, ToolFactory> toolFactoryMap = new LinkedHashMap<>()
 
     protected final Map<String, WebappInfo> webappInfoMap = new HashMap<>()

framework/src/main/groovy/org/moqui/impl/context/ExecutionContextImpl.java

@@ -67,8 +67,8 @@ public class ExecutionContextImpl implements ExecutionContext {
     private Cache<String, String> l10nMessageCache;
     private Cache<String, ArrayList> tarpitHitCache;
 
-    public final String forThreadName;
-    public final long forThreadId;
+    public String forThreadName;
+    public long forThreadId;
     // public final Exception createLoc;
 
     public ExecutionContextImpl(ExecutionContextFactoryImpl ecfi, Thread forThread) {

framework/src/main/groovy/org/moqui/impl/context/WebFacadeImpl.groovy

@@ -466,7 +466,12 @@ class WebFacadeImpl implements WebFacade {
         String servletPath = request.getServletPath()
         // subtract 1 to exclude empty string before leading '/' that will always be there
         int servletPathSize = servletPath.isEmpty() ? 0 : (servletPath.split("/").length - 1)
-        ArrayList<String> pathList = StringUtilities.pathStringToList(reqURI, servletPathSize)
+
+        // exclude context path segments
+        String contextPath = request.getContextPath()
+        int contextPathSize = contextPath.isEmpty() ? 0 : (contextPath.split("/").length - 1)
+
+        ArrayList<String> pathList = StringUtilities.pathStringToList(reqURI, servletPathSize + contextPathSize)
         // logger.warn("pathInfo ${request.getPathInfo()} servletPath ${servletPath} reqURI ${request.getRequestURI()} pathList ${pathList}")
         return pathList
     }

framework/src/main/groovy/org/moqui/impl/entity/EntityDbMeta.groovy

@@ -698,7 +698,9 @@ class EntityDbMeta {
         }
 
         // do fk auto indexes
-        if (databaseNode.attribute("use-foreign-key-indexes") == "false") return
+        // nothing after fk indexes to return now if disabled
+        if (databaseNode.attribute("use-foreign-key-indexes") == "false") return created
+
         for (RelationshipInfo relInfo in ed.getRelationshipsInfo(false)) {
             if (relInfo.type != "one") continue
 

framework/src/main/groovy/org/moqui/impl/entity/EntityDefinition.groovy

@@ -872,12 +872,12 @@ class EntityDefinition {
             for (Map.Entry<String, EntityDependents> entry in dependentEntities) {
                 RelationshipInfo relInfo = relationshipInfos.get(entry.getKey())
                 builder.append(indent).append(relInfo.relationshipName).append(" ").append(relInfo.keyMap).append("\n")
-                if (level < 8 && !entitiesVisited.contains(entry.getValue().entityName)) {
+                if (level < 4 && !entitiesVisited.contains(entry.getValue().entityName)) {
                     entry.getValue().buildString(builder, level + 1I, entitiesVisited)
                     entitiesVisited.add(entry.getValue().entityName)
                 } else if (entitiesVisited.contains(entry.getValue().entityName)) {
                     builder.append(indent).append(indentBase).append("Dependants already displayed\n")
-                } else if (level == 8) {
+                } else if (level == 4) {
                     builder.append(indent).append(indentBase).append("Reached level limit\n")
                 }
             }
@@ -1041,6 +1041,7 @@ class EntityDefinition {
             if (validDate == (Timestamp) null) validDate = efi.ecfi.getEci().userFacade.getNowTimestamp()
 
             String entityAliasAttr = dateFilter.attribute("entity-alias")
+            // if no entity-alias specified, use entity-alias from join member-entity node (if field exists on join entity)
             if (joinEntityDef != null && (entityAliasAttr == null || entityAliasAttr.isEmpty()) && joinEntityDef.isField(fromFieldName))
                 entityAliasAttr = joinMemberEntityNode.attribute("entity-alias")
 
@@ -1068,8 +1069,19 @@ class EntityDefinition {
             EntityDefinition condEd
 
             String entityAliasAttr = econdition.attribute("entity-alias")
-            if (joinEntityDef != null && (entityAliasAttr == null || entityAliasAttr.isEmpty()) && joinEntityDef.isField(fieldNameAttr))
-                entityAliasAttr = joinMemberEntityNode.attribute("entity-alias")
+            // if no entity-alias specified, use entity-alias from join member-entity node (if field exists on join entity)
+            if (joinEntityDef != null && (entityAliasAttr == null || entityAliasAttr.isEmpty()) && joinEntityDef.isField(fieldNameAttr)) {
+                String joinMemberAlias = joinMemberEntityNode.attribute("entity-alias")
+                if (memberEntityAliasMap.containsKey(joinMemberAlias)) {
+                    entityAliasAttr = joinMemberAlias
+                } else {
+                    // special case for entity-condition.econdition under view-entity.member-entity with sub-select=true
+                    //     and when doing lateral joins, because WHERE clause is inside sub-select so should default to member-entity's internal alias
+                    // is the field an alias on this entity? use that entity-alias
+                    MNode aliasNode = this.getFieldNode(fieldNameAttr)
+                    if (aliasNode != null) entityAliasAttr = aliasNode.attribute("entity-alias")
+                }
+            }
 
             if (entityAliasAttr != null && !entityAliasAttr.isEmpty()) {
                 MNode memberEntity = (MNode) memberEntityAliasMap.get(entityAliasAttr)

framework/src/main/groovy/org/moqui/impl/entity/EntityFacadeImpl.groovy

@@ -1139,7 +1139,7 @@ class EntityFacadeImpl implements EntityFacade {
             }
 
             eil.add([entityName:ed.entityInfo.internalEntityName, "package":ed.entityNode.attribute("package"),
-                    isView:(ed.isViewEntity ? "true" : "false"), fullEntityName:ed.fullEntityName] as Map<String, Object>)
+                    isView:(ed.isViewEntity ? "true" : "false"), fullEntityName:ed.fullEntityName, tableName:ed.tableName] as Map<String, Object>)
         }
 
         if (orderByField != null && !orderByField.isEmpty()) CollectionUtilities.orderMapList(eil, [orderByField])

framework/src/main/groovy/org/moqui/impl/entity/EntityFindBuilder.java

@@ -393,9 +393,10 @@ public void appendJoinConditions(MNode relatedMemberEntityNode, String entityAli
         if (entityConditionList != null && entityConditionList.size() > 0) {
             // add any additional manual conditions for the member-entity view link here
             MNode entityCondition = entityConditionList.get(0);
+            // logger.warn("======== appendJoinConditions() localEntityDefinition " + localEntityDefinition.fullEntityName + " linkEntityDefinition " + linkEntityDefinition.fullEntityName + " relatedLinkEntityDefinition " + relatedLinkEntityDefinition.fullEntityName);
             EntityConditionImplBase linkEcib = localEntityDefinition.makeViewListCondition(entityCondition, relatedMemberEntityNode);
             if (keyMapsSize > 0) localBuilder.append(" AND ");
-            // TODO: is this correct? what does it append to? not localBuilder?
+            // TODO: does this need to use localBuilder? seems to be working so far...
             linkEcib.makeSqlWhere(this, null);
         }
     }
@@ -635,7 +636,7 @@ public void makeSqlMemberSubSelect(String entityAlias, MNode memberEntity, Entit
             if (condition != null) localBuilder.append(" AND ");
         }
         if (condition != null) {
-            // TODO: does this need to use localBuilder?
+            // TODO: does this need to use localBuilder? seems to be working so far...
             condition.makeSqlWhere(this, localEntityDefinition);
         }
 

framework/src/main/groovy/org/moqui/impl/entity/FieldInfo.java

@@ -65,8 +65,24 @@ public FieldInfo(EntityDefinition ed, MNode fieldNode, int index) {
         // NOTE: intern a must here for use with LiteStringMap, without this all sorts of bad behavior, not finding any fields sort of thing
         name = nameAttr.intern();
         conditionField = new ConditionField(this);
+        // column name from attribute or underscored name, may have override per DB
         String columnNameAttr = fnAttrs.get("column-name");
-        columnName = columnNameAttr != null && columnNameAttr.length() > 0 ? columnNameAttr : EntityJavaUtil.camelCaseToUnderscored(name);
+        String colNameToUse = columnNameAttr != null && columnNameAttr.length() > 0 ? columnNameAttr :
+                EntityJavaUtil.camelCaseToUnderscored(name);
+        // column name: see if there is a name-replace
+        String groupName = ed.getEntityGroupName();
+        MNode databaseNode = ed.efi.getDatabaseNode(groupName);
+        ArrayList<MNode> nameReplaceNodes = databaseNode.children("name-replace");
+        for (int i = 0; i < nameReplaceNodes.size(); i++) {
+            MNode nameReplaceNode = nameReplaceNodes.get(i);
+            if (colNameToUse.equalsIgnoreCase(nameReplaceNode.attribute("original"))) {
+                String replaceName = nameReplaceNode.attribute("replace");
+                logger.info("Replacing column name " + colNameToUse + " with replace name " + replaceName + " for entity " + entityName);
+                colNameToUse = replaceName;
+            }
+        }
+        columnName = colNameToUse;
+
         defaultStr = fnAttrs.get("default");
 
         String typeAttr = fnAttrs.get("type");
@@ -142,6 +158,7 @@ public FieldInfo(EntityDefinition ed, MNode fieldNode, int index) {
         }
     }
 
+    /** Full column name for complex finds on view entities; plain entity column names are never expanded */
     public String getFullColumnName() {
         if (fullColumnNameInternal != null) return fullColumnNameInternal;
         return ed.efi.ecfi.resourceFacade.expand(expandColumnName, "", null, false);
@@ -561,10 +578,20 @@ private void setPreparedStatementValue(PreparedStatement ps, int index, Object v
                         ps.setBytes(index, valueBb.array());
                     } else if (value instanceof Blob) {
                         Blob valueBlob = (Blob) value;
-                        // calling setBytes instead of setBlob
+                        // calling setBytes instead of setBlob - old github.com/moqui/moqui repo issue #28 with Postgres JDBC driver
                         // ps.setBlob(index, (Blob) value)
                         // Blob blb = value
-                        ps.setBytes(index, valueBlob.getBytes(1, (int) valueBlob.length()));
+                        try {
+                            ps.setBytes(index, valueBlob.getBytes(1, (int) valueBlob.length()));
+                        } catch (Exception bytesExc) {
+                            // try ps.setBlob for larger byte arrays that H2 throws an exception for
+                            try {
+                                ps.setBlob(index, valueBlob);
+                            } catch (Exception blobExc) {
+                                // throw the original exception from setBytes()
+                                throw bytesExc;
+                            }
+                        }
                     } else {
                         if (value != null) {
                             throw new EntityException("Type not supported for BLOB field: " + value.getClass().getName() + ", for field " + entityName + "." + name);