PDI-191 : As a developer/contributor, I want pluggable types

PDI-7103 : Requirement for supporting Oracle Timestamp(6) datatype (Microseconds)
PDI-9383 : As an ETL developer I want to have an overview of all the plugins that are loaded in Kettle
+ Additional cleanups


git-svn-id: svn://source.pentaho.org/svnkettleroot/Kettle/trunk@18117 5fb7f6ec-07c1-534a-b4ca-9155e429e800

README.txt

@@ -171,15 +171,15 @@ I ran Spoon from the project's dist folder.  Why can't I see my changes I just c
 If I needed to change something in DB, like the default port for PostgreSQL, do I need to check out all
 of Kettle and build it?
 
-     No you don't. In this example you can check out only the "db" module.  Run the module's 
+     No you don't. In this example you can check out only the "core" module.  Run the module's 
      default Ant target. If you are using Eclipse, run the "create-dot-classpath" Ant target and refresh
      the project in Eclipse. Then make your code change in PostgreSQLDatabaseMeta and run the "dist" ant target.  
-     A kettle-db JAR will be built and placed in the project's dist/ folder.
+     A kettle-core JAR will be built and placed in the project's dist/ folder.
 
      To test out your changes you can grab a Kettle build from CI:
         http://ci.pentaho.com/view/Data Integration/job/Kettle/
 
-     Replace the kettle-db jar in the CI build's lib/ folder and run Spoon.  Create
+     Replace the kettle-core jar in the CI build's lib/ folder and run Spoon.  Create
      a new DB connection with PostgreSQL as the connection type.  You should see
      your new default port number.
 
@@ -243,4 +243,4 @@ What is "package-res" in assembly?
      once was under the root of the Kettle project.  These folders are packaged up 
      into the distributable product.
 
-     Changes to shell scripts, launcher, images, and docs are made here.     
+     Changes to shell scripts, launcher, images, and docs are made here.     

assembly/ivysettings.xml

@@ -22,4 +22,4 @@
     </chain>
   </resolvers>
   <caches lockStrategy="artifact-lock" resolutionCacheDir="${ivy.default.ivy.user.dir}/resol-cache${env.EXECUTOR_NUMBER}" />
-</ivysettings>
+</ivysettings>

assembly/package-res/ui/laf.properties

@@ -113,6 +113,7 @@ UnconditionalHop_image = ui/images/unconditional-hop.png
 Busy_image = ui/images/busy.png
 Inject_image = ui/images/inject.png
 Gantt_image = ui/images/gantt.png
+HelpWeb_image = ui/images/help_web.png
 
 Database_image = ui/images/CNC.png
 
@@ -125,4 +126,4 @@ defaultRepositoryName=Kettle
 documentationDirBase=docs/
 
 #should trans/job file name be equal to true
-#sync_trans_name_to_file_name=true
+#sync_trans_name_to_file_name=true

assembly/package-res/ui/main_perspective_overlay.xul

@@ -40,8 +40,7 @@
         <menuitem id="help-tip" visible="true"/>
         <menuitem id="help-welcome" visible="true"/>
         <menuseparator id="help-separator-1" visible="true"/>
-        <menuitem id="help-step-plugins" visible="true"/>
-        <menuitem id="help-jobentry-plugins" visible="true"/>
+        <menuitem id="help-plugins" visible="true"/>
         <menuitem id="help-about" visible="true"/>
 
     <!--   

assembly/package-res/ui/menubar.xul

@@ -179,8 +179,7 @@
         <menuitem id="help-tip" label="${Spoon.Menu.Help.Tip}" command="spoon.showTips()" />
         <menuitem id="help-welcome" label="${Spoon.Menu.Help.Welcome}" command="spoon.showWelcomePage()" />
         <menuseparator id="help-separator-1"/>
-        <menuitem id="help-step-plugins" label="${Spoon.Menu.StepPlugins}" command="spoon.helpShowStepPlugins()" />
-        <menuitem id="help-jobentry-plugins" label="${Spoon.Menu.JobEntryPlugins}" command="spoon.helpShowJobEntryPlugins()" />
+        <menuitem id="help-plugins" label="${Spoon.Menu.ShowPluginInfo}" command="spoon.showPluginInfo()" />
         <menuitem id="help-about" label="${Spoon.Menu.About}" command="spoon.helpAbout()" />
       </menupopup>
     </menu>

build-res/ivysettings.xml

@@ -22,4 +22,4 @@
     </chain>
   </resolvers>
   <caches lockStrategy="artifact-lock" resolutionCacheDir="${ivy.default.ivy.user.dir}/resol-cache${env.EXECUTOR_NUMBER}" />
-</ivysettings>
+</ivysettings>

build-res/publish.xml

@@ -29,7 +29,7 @@
 
 
   <property name="kettle.module.list"
-            value="kettle-core, kettle-db, kettle-engine, kettle-dbdialog, kettle-ui-swt, kettle-test"
+            value="kettle-core, kettle-db, kettle-engine, kettle-dbdialog, kettle-ui-swt, kettle-test, kettle-gui"
             description="The list of kettle modules" />
 
   <!-- Import the common_build.xml file which contains all the default tasks -->  

build.properties

@@ -19,15 +19,13 @@ src=engine/src
 src-ui=ui/src
 src-dbdialog=dbdialog/src
 src-core=core/src
-src-db=db/src
 src-jdbc=jdbc/src
 test=test
 classes=classes
 classes.dir=${classes}
 classes-ui=${classes}/ui
 classes-dbdialog=${classes}/dbdialog
 classes-core=${classes}/core
-classes-db=${classes}/db
 classes-jdbc=${classes}/jdbc
 testClasses=testClasses
 testsrc.dir=test
@@ -91,8 +89,8 @@ coverage.html.dir=${coverage.dir}/html
 
 includeantruntime=false
 
-plugin-ids=kettle-gpload-plugin,kettle-hl7-plugin,kettle-palo-plugin,kettle-openerp-plugin,star-modeler,market
-dev-project.list=core,db,engine,db,dbdialog,jdbc,ui
+plugin-ids=kettle-gpload-plugin,kettle-hl7-plugin,kettle-palo-plugin,kettle-openerp-plugin,star-modeler,market,kettle5-log4j-plugin
+dev-project.list=core,engine,dbdialog,jdbc,ui
 assembly.dir=assembly
 assembly.stage.dir=${assembly.dir}/bin/stage
 assembly.stage.pdi-client.dir=${assembly.stage.dir}/data-integration
@@ -112,4 +110,4 @@ dependency.pentaho-xul.revision=TRUNK-SNAPSHOT
 dependency.pentaho-vfs-browser.revision=TRUNK-SNAPSHOT
 dependency.pentaho-reporting-core.revision=TRUNK-SNAPSHOT
 dependency.pdi-palo-plugin.revision=TRUNK-SNAPSHOT
-launcher.revision=1.0.0
+launcher.revision=1.0.0

build.xml

@@ -114,7 +114,6 @@
 		<junit printsummary="no" fork="yes" forkmode="perTest" jvm="java" maxmemory="512M" showoutput="yes">
 	    	<classpath>
 				<fileset dir="${lib}" includes="kettle-core.jar"/>
-				<fileset dir="${lib}" includes="kettle-db.jar"/>
 				<fileset dir="${lib}" includes="kettle-engine.jar"/>
 				<fileset dir="test/libext/" includes="kettle.jar"/>
 	    		<pathelement path="${testClasses}"/>
@@ -193,7 +192,6 @@
 		<junit printsummary="no" fork="yes" forkmode="perTest" jvm="java" maxmemory="512M" showoutput="yes">
 	    	<classpath>
 				<fileset dir="${lib}" includes="kettle-core.jar"/>
-				<fileset dir="${lib}" includes="kettle-db.jar"/>
 				<fileset dir="${lib}" includes="kettle-engine.jar"/>
 				<fileset dir="test/libext/" includes="kettle.jar"/>
 	    		<pathelement path="${testClasses}"/>
@@ -270,7 +268,6 @@
 		<junit printsummary="no" fork="yes" forkmode="perTest" jvm="java" maxmemory="512M" showoutput="yes">
 	    	<classpath>
 				<fileset dir="${lib}" includes="kettle-core.jar"/>
-				<fileset dir="${lib}" includes="kettle-db.jar"/>
 				<fileset dir="${lib}" includes="kettle-engine.jar"/>
 				<!-- fileset dir="test/libext/" includes="junit.jar"/ -->
 	    		<pathelement path="${testClasses}"/>
@@ -298,7 +295,6 @@
 		<junit printsummary="no" fork="yes" forkmode="perTest" jvm="java" maxmemory="512M" showoutput="yes">
 	    	<classpath>
 				<fileset dir="${lib}" includes="kettle-core.jar"/>
-				<fileset dir="${lib}" includes="kettle-db.jar"/>
 				<fileset dir="${lib}" includes="kettle-engine.jar"/>
 	    		<fileset dir="${lib}" includes="kettle-dbdialog.jar"/>
 	    		<fileset dir="${lib}" includes="kettle-ui-swt.jar"/>

classpath.template

@@ -2,8 +2,6 @@
 <classpath>
 	<classpathentry kind="src" path="core/src"/>
 	<classpathentry kind="src" path="core/test-src"/>
-	<classpathentry kind="src" path="db/src"/>
-	<classpathentry kind="src" path="db/test-src"/>
 	<classpathentry kind="src" path="dbdialog/src"/>
 	<classpathentry kind="src" path="dbdialog/test-src"/>
 	<classpathentry kind="src" path="jdbc/src"/>

core/ivy.xml

@@ -25,11 +25,15 @@
     <dependency org="commons-lang"        name="commons-lang"        rev="2.6"              transitive="false"/>
     <dependency org="commons-httpclient"  name="commons-httpclient"  rev="3.1"              transitive="false"/>
     <dependency org="commons-vfs"         name="commons-vfs"         rev="20091118-pentaho" transitive="false"/>
+    <dependency org="commons-io"          name="commons-io"          rev="1.4"              transitive="false"/>
+    <dependency org="commons-dbcp"        name="commons-dbcp"        rev="1.2.1"            transitive="false"/>
+    <dependency org="commons-pool"        name="commons-pool"        rev="1.5.7"            transitive="false"/>
+
+    <dependency org="org.apache.commons"  name="commons-compress"    rev="1.4"              transitive="false"/>
     <dependency org="dom4j"               name="dom4j"               rev="1.6.1"            transitive="false"/>
     <dependency org="org.mortbay.jetty"   name="jetty"               rev="6.1.21"           transitive="false"/>
     <dependency org="jug-lgpl"            name="jug-lgpl"            rev="2.0.0"            transitive="false"/>
     <dependency org="com.jcraft"          name="jsch"                rev="0.1.46"           transitive="false"/>
-    <dependency org="log4j"               name="log4j"               rev="1.2.16"           transitive="false"/>
     <dependency org="ognl"                name="ognl"                rev="2.6.9"            transitive="false"/>
     <dependency org="net.sf.scannotation" name="scannotation"        rev="1.0.2"            transitive="false"/>
     <dependency org="secondstring"        name="secondstring"        rev="20060615"         transitive="false"/>

core/ivysettings.xml

@@ -21,4 +21,4 @@
     </chain>
   </resolvers>
   <caches lockStrategy="artifact-lock" resolutionCacheDir="${ivy.default.ivy.user.dir}/resol-cache${env.EXECUTOR_NUMBER}" />
-</ivysettings>
+</ivysettings>

engine/src/kettle-valuemeta-plugins.xml → core/src/kettle-valuemeta-plugins.xml

@@ -45,4 +45,9 @@
     <classname>org.pentaho.di.core.row.value.ValueMetaBinary</classname> 
   </valuemeta-plugin>
 
+  <valuemeta-plugin id="9"> 
+    <description>Timestamp</description> 
+    <classname>org.pentaho.di.core.row.value.ValueMetaTimestamp</classname> 
+  </valuemeta-plugin>
+
 </valuemeta-plugins>

core/src/org/pentaho/di/core/Const.java

@@ -741,7 +741,17 @@ public enum ReleaseType {RELEASE_CANDIDATE, MILESTONE, PREVIEW, GA}
   * The name of the environment variable that will contain the alternative location of the kettle-valuemeta-plugins.xml file
   */
 	public static final String   KETTLE_VALUEMETA_PLUGINS_FILE = "KETTLE_VALUEMETA_PLUGINS_FILE";
-
+
+	 /**
+   * The XML file that contains the list of native Kettle logging plugins
+   */
+  public static final String  XML_FILE_KETTLE_LOGGING_PLUGINS = "kettle-logging-plugins.xml";
+
+   /**
+  * The name of the environment variable that will contain the alternative location of the kettle-logging-plugins.xml file
+  */
+  public static final String   KETTLE_LOGGING_PLUGINS_FILE = "KETTLE_LOGGING_PLUGINS_FILE";
+
   /**
    * The name of the environment variable that will contain the alternative location of the kettle-servlets.xml file
    */

core/src/org/pentaho/di/core/KettleClientEnvironment.java

@@ -5,6 +5,8 @@
 import java.io.IOException;
 
 import org.pentaho.di.core.exception.KettleException;
+import org.pentaho.di.core.logging.LoggingPluginType;
+import org.pentaho.di.core.plugins.DatabasePluginType;
 import org.pentaho.di.core.plugins.PluginRegistry;
 import org.pentaho.di.core.row.value.ValueMetaPluginType;
 import org.pentaho.di.core.util.EnvUtil;
@@ -46,7 +48,9 @@ public static synchronized void init() throws KettleException {
 
 		// Load value meta data plugins
 		//
+    PluginRegistry.addPluginType(LoggingPluginType.getInstance());
 		PluginRegistry.addPluginType(ValueMetaPluginType.getInstance());
+    PluginRegistry.addPluginType(DatabasePluginType.getInstance());
 		PluginRegistry.init(true);
 
 		initialized=new Boolean(true);

db/src/org/pentaho/di/core/database/BaseDatabaseMeta.java → core/src/org/pentaho/di/core/database/BaseDatabaseMeta.java

@@ -22,7 +22,6 @@
 
 package org.pentaho.di.core.database;
 
-import java.sql.Blob;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
@@ -55,7 +54,7 @@
  * @author Matt
  * @since  11-mrt-2005
  */
-public abstract class BaseDatabaseMeta implements Cloneable
+public abstract class BaseDatabaseMeta implements Cloneable, DatabaseInterface
 {
     /**
      * The port number of the database as string: allows for parameterization.
@@ -161,7 +160,13 @@ public abstract class BaseDatabaseMeta implements Cloneable
      * Defaults to "false" for backward compatibility! 
      */
     public static final String ATTRIBUTE_SUPPORTS_BOOLEAN_DATA_TYPE = "SUPPORTS_BOOLEAN_DATA_TYPE";
-
+
+    /**
+     * Checkbox to allow you to configure if the database supports the Timestamp data type or not.
+     * Defaults to "false" for backward compatibility! 
+     */
+    public static final String ATTRIBUTE_SUPPORTS_TIMESTAMP_DATA_TYPE = "SUPPORTS_TIMESTAMP_DATA_TYPE";
+
     /**
      * Checkbox to allow you to configure if the reserved words will have their case changed during the handleCase call 
      */
@@ -987,7 +992,22 @@ public void setSupportsBooleanDataType(boolean b)
 		attributes.setProperty(ATTRIBUTE_SUPPORTS_BOOLEAN_DATA_TYPE, b?"Y":"N");
 	}
 
+  /**
+   * @return true if the database supports the Timestamp data type (nanosecond precision and all)
+   */
+  public boolean supportsTimestampDataType() {
+    String supportsTimestamp = attributes.getProperty(ATTRIBUTE_SUPPORTS_TIMESTAMP_DATA_TYPE, "N");
+    return "Y".equalsIgnoreCase(supportsTimestamp);
+  }
 
+  /**
+   * 
+   * @param b Set to true if the database supports the Timestamp data type (nanosecond precision and all)
+   */
+  public void setSupportsTimestampDataType(boolean b) {
+    attributes.setProperty(ATTRIBUTE_SUPPORTS_TIMESTAMP_DATA_TYPE, b?"Y":"N");
+  }
+
 
   /**
    * @return true if reserved words' case should be preserved
@@ -1887,60 +1907,9 @@ public String getTablespaceDDL(VariableSpace variables, DatabaseMeta databaseMet
    * @throws KettleDatabaseException
    */
   public Object getValueFromResultSet(ResultSet rs, ValueMetaInterface val, int i) throws KettleDatabaseException {
-    Object data = null;
 
-    try {
-      switch (val.getType()) {
-      case ValueMetaInterface.TYPE_BOOLEAN:
-        data = Boolean.valueOf(rs.getBoolean(i + 1));
-        break;
-      case ValueMetaInterface.TYPE_NUMBER:
-        data = new Double(rs.getDouble(i + 1));
-        break;
-      case ValueMetaInterface.TYPE_BIGNUMBER:
-        data = rs.getBigDecimal(i + 1);
-        break;
-      case ValueMetaInterface.TYPE_INTEGER:
-        data = Long.valueOf(rs.getLong(i + 1));
-        break;
-      case ValueMetaInterface.TYPE_STRING: 
-        if (val.isStorageBinaryString()) {
-          data = rs.getBytes(i + 1);
-        } else {
-          data = rs.getString(i + 1);
-        }
-        break;
-      case ValueMetaInterface.TYPE_BINARY:
-        if (supportsGetBlob()) {
-          Blob blob = rs.getBlob(i + 1);
-          if (blob != null) {
-            data = blob.getBytes(1L, (int) blob.length());
-          } else {
-            data = null;
-          }
-        } else {
-          data = rs.getBytes(i + 1);
-        }
-        break;
-      case ValueMetaInterface.TYPE_DATE:
-        if (val.getPrecision() != 1 && supportsTimeStampToDateConversion()) {
-          data = rs.getTimestamp(i + 1);
-          break; // Timestamp extends java.util.Date
-        } else {
-          data = rs.getDate(i + 1);
-          break;
-        }
-      default:
-        break;
-      }
-      if (rs.wasNull()) {
-        data = null; 
-      }
-    } catch(SQLException e) {
-      throw new KettleDatabaseException("Unable to get value '"+val.toStringMeta()+"' from database resultset, index "+i, e);
-    }
-
-    return data;
+    return val.getValueFromResultSet(this, rs, i);
+
   }
 
   /**