Skip to content

Commit

Permalink
Merge pull request pentaho#1786 from TatsianaKasiankova/PDI-13271
Browse files Browse the repository at this point in the history
[PDI-13271]: External Ids are not handled properly when incoming key is null
  • Loading branch information
Bryan Rosander committed Dec 1, 2015
2 parents 9859b8f + 0067514 commit 4316ce3
Show file tree
Hide file tree
Showing 9 changed files with 801 additions and 25 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,9 @@
import org.pentaho.di.trans.step.StepMeta;
import org.pentaho.di.trans.step.StepMetaInterface;
import org.pentaho.di.trans.steps.salesforceinput.SalesforceConnection;
import org.pentaho.di.trans.steps.salesforceutils.SalesforceUtils;

import com.google.common.annotations.VisibleForTesting;
import com.sforce.soap.partner.sobject.SObject;

/**
Expand Down Expand Up @@ -115,7 +117,8 @@ public boolean processRow( StepMetaInterface smi, StepDataInterface sdi ) throws
return true;
}

private void writeToSalesForce( Object[] rowData ) throws KettleException {
@VisibleForTesting
void writeToSalesForce( Object[] rowData ) throws KettleException {
try {

if ( log.isDetailed() ) {
Expand All @@ -137,7 +140,8 @@ private void writeToSalesForce( Object[] rowData ) throws KettleException {
if ( valueMeta.isNull( value ) ) {
// The value is null
// We need to keep track of this field
fieldsToNull.add( meta.getUpdateLookup()[i] );
fieldsToNull.add( SalesforceUtils.getFieldToNullName( log, meta.getUpdateLookup()[i], meta
.getUseExternalId()[i] ) );
} else {
if ( valueMeta.isDate() ) {
// Pass date field converted to UTC, see PDI-10836
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,9 @@
import org.pentaho.di.trans.step.StepMeta;
import org.pentaho.di.trans.step.StepMetaInterface;
import org.pentaho.di.trans.steps.salesforceinput.SalesforceConnection;
import org.pentaho.di.trans.steps.salesforceutils.SalesforceUtils;

import com.google.common.annotations.VisibleForTesting;
import com.sforce.soap.partner.sobject.SObject;

/**
Expand All @@ -53,8 +55,8 @@ public class SalesforceUpdate extends BaseStep implements StepInterface {
private SalesforceUpdateMeta meta;
private SalesforceUpdateData data;

public SalesforceUpdate( StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr,
TransMeta transMeta, Trans trans ) {
public SalesforceUpdate( StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr, TransMeta transMeta,
Trans trans ) {
super( stepMeta, stepDataInterface, copyNr, transMeta, trans );
}

Expand Down Expand Up @@ -83,8 +85,8 @@ public boolean processRow( StepMetaInterface smi, StepDataInterface sdi ) throws

// Check if field list is filled
if ( data.nrfields == 0 ) {
throw new KettleException( BaseMessages.getString(
PKG, "SalesforceUpdateDialog.FieldsMissing.DialogMessage" ) );
throw new KettleException( BaseMessages.getString( PKG,
"SalesforceUpdateDialog.FieldsMissing.DialogMessage" ) );
}

// Create the output row meta-data
Expand All @@ -97,8 +99,8 @@ public boolean processRow( StepMetaInterface smi, StepDataInterface sdi ) throws
for ( int i = 0; i < meta.getUpdateStream().length; i++ ) {
data.fieldnrs[i] = getInputRowMeta().indexOfValue( meta.getUpdateStream()[i] );
if ( data.fieldnrs[i] < 0 ) {
throw new KettleException( "Field ["
+ meta.getUpdateStream()[i] + "] couldn't be found in the input stream!" );
throw new KettleException( "Field [" + meta.getUpdateStream()[i]
+ "] couldn't be found in the input stream!" );
}
}
}
Expand All @@ -112,7 +114,8 @@ public boolean processRow( StepMetaInterface smi, StepDataInterface sdi ) throws
return true;
}

private void writeToSalesForce( Object[] rowData ) throws KettleException {
@VisibleForTesting
void writeToSalesForce( Object[] rowData ) throws KettleException {
try {

if ( log.isDetailed() ) {
Expand All @@ -131,10 +134,11 @@ private void writeToSalesForce( Object[] rowData ) throws KettleException {
if ( valueIsNull ) {
// The value is null
// We need to keep track of this field
fieldsToNull.add( meta.getUpdateLookup()[i] );
fieldsToNull.add( SalesforceUtils.getFieldToNullName( log, meta.getUpdateLookup()[i], meta
.getUseExternalId()[i] ) );
} else {
updatefields.add( SalesforceConnection.createMessageElement(
meta.getUpdateLookup()[i], rowData[data.fieldnrs[i]], meta.getUseExternalId()[i] ) );
updatefields.add( SalesforceConnection.createMessageElement( meta.getUpdateLookup()[i],
rowData[data.fieldnrs[i]], meta.getUseExternalId()[i] ) );
}
}

Expand Down Expand Up @@ -209,18 +213,17 @@ private void flushBuffers() throws KettleException {
// Only send the first error
//
com.sforce.soap.partner.Error err = data.saveResult[j].getErrors()[0];
throw new KettleException( BaseMessages
.getString( PKG, "SalesforceUpdate.Error.FlushBuffer", new Integer( j ), err.getStatusCode(), err
.getMessage() ) );
throw new KettleException( BaseMessages.getString( PKG, "SalesforceUpdate.Error.FlushBuffer", new Integer(
j ), err.getStatusCode(), err.getMessage() ) );
}

String errorMessage = "";
for ( int i = 0; i < data.saveResult[j].getErrors().length; i++ ) {
// get the next error
com.sforce.soap.partner.Error err = data.saveResult[j].getErrors()[i];
errorMessage +=
BaseMessages.getString( PKG, "SalesforceUpdate.Error.FlushBuffer", new Integer( j ), err
.getStatusCode(), err.getMessage() );
BaseMessages.getString( PKG, "SalesforceUpdate.Error.FlushBuffer", new Integer( j ), err
.getStatusCode(), err.getMessage() );
}

// Simply add this row to the error row
Expand Down Expand Up @@ -286,7 +289,7 @@ public boolean init( StepMetaInterface smi, StepDataInterface sdi ) {
data.realURL = environmentSubstitute( meta.getTargetURL() );
// create a Salesforce connection
data.connection =
new SalesforceConnection( log, data.realURL, realUser, environmentSubstitute( meta.getPassword() ) );
new SalesforceConnection( log, data.realURL, realUser, environmentSubstitute( meta.getPassword() ) );
// set timeout
data.connection.setTimeOut( Const.toInt( environmentSubstitute( meta.getTimeOut() ), 0 ) );
// Do we use compression?
Expand All @@ -299,8 +302,8 @@ public boolean init( StepMetaInterface smi, StepDataInterface sdi ) {

return true;
} catch ( KettleException ke ) {
logError( BaseMessages.getString( PKG, "SalesforceUpdate.Log.ErrorOccurredDuringStepInitialize" )
+ ke.getMessage() );
logError( BaseMessages.getString( PKG, "SalesforceUpdate.Log.ErrorOccurredDuringStepInitialize" ) + ke
.getMessage() );
}
return true;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
package org.pentaho.di.trans.steps.salesforceupsert;

import java.util.ArrayList;

import org.apache.axis.message.MessageElement;
import org.pentaho.di.core.Const;
import org.pentaho.di.core.exception.KettleException;
Expand All @@ -39,7 +38,9 @@
import org.pentaho.di.trans.step.StepMeta;
import org.pentaho.di.trans.step.StepMetaInterface;
import org.pentaho.di.trans.steps.salesforceinput.SalesforceConnection;
import org.pentaho.di.trans.steps.salesforceutils.SalesforceUtils;

import com.google.common.annotations.VisibleForTesting;
import com.sforce.soap.partner.sobject.SObject;

/**
Expand Down Expand Up @@ -112,7 +113,8 @@ public boolean processRow( StepMetaInterface smi, StepDataInterface sdi ) throws
return true;
}

private void writeToSalesForce( Object[] rowData ) throws KettleException {
@VisibleForTesting
void writeToSalesForce( Object[] rowData ) throws KettleException {
try {

if ( log.isDetailed() ) {
Expand All @@ -133,11 +135,12 @@ private void writeToSalesForce( Object[] rowData ) throws KettleException {
if ( valueMeta.isNull( object ) ) {
// The value is null
// We need to keep track of this field
fieldsToNull.add( meta.getUpdateLookup()[i] );
fieldsToNull.add( SalesforceUtils.getFieldToNullName( log, meta.getUpdateLookup()[i], meta
.getUseExternalId()[i] ) );
} else {
Object normalObject = valueMeta.convertToNormalStorageType( object );
upsertfields.add( SalesforceConnection.createMessageElement(
meta.getUpdateLookup()[i], normalObject, meta.getUseExternalId()[i] ) );
upsertfields.add( SalesforceConnection.createMessageElement( meta.getUpdateLookup()[i], normalObject, meta
.getUseExternalId()[i] ) );
}
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
/*! ******************************************************************************
*
* Pentaho Data Integration
*
* Copyright (C) 2002-2015 by Pentaho : http://www.pentaho.com
*
*******************************************************************************
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
******************************************************************************/
package org.pentaho.di.trans.steps.salesforceutils;

import static java.util.regex.Pattern.compile;

import java.util.regex.Pattern;

import org.pentaho.di.core.logging.LogChannelInterface;
import org.pentaho.di.i18n.BaseMessages;

/**
* Utility class to process salesforce specific features.
*
* @author Tatsiana_Kasiankova
*
*/
public class SalesforceUtils {
private static Class<?> PKG = SalesforceUtils.class; // for i18n purposes, needed by Translator2!!

private static final String EXTID_SEPARATOR = "/";

private static final String CUSTOM_OBJECT_RELATIONSHIP_FIELD_SUFFIX = "_r";

private static final String CUSTOM_OBJECT_SUFFIX = "_c";

private static final Pattern FIELD_NAME_WITH_EXTID_PATTERN = compile( "^\\w+\\:\\w+\\/\\w+$" );

/**
* Extract and return the correct name for the field that should be processed as NULL
*
* @param log
* the logging object
* @param field
* the field that should be processed as NULL
* @param isUseExtId
* the flag that indicates if the field is external id or not
* @return return the correct name for the field that should be processed as NULL
*/
public static String getFieldToNullName( LogChannelInterface log, String field, boolean isUseExtId ) {
String fieldToNullName = field;
if ( isUseExtId ) {
// verify if the field has correct syntax
if ( !FIELD_NAME_WITH_EXTID_PATTERN.matcher( field ).matches() ) {
if ( log.isDebug() ) {
log.logDebug( BaseMessages.getString( PKG, "SalesforceUtils.Warn.IncorrectExternalKeySyntax", field,
fieldToNullName ) );
}
return fieldToNullName;
}

String lookupField = field.substring( field.indexOf( EXTID_SEPARATOR ) + 1 );
// working with custom objects and relationship
// cut off _r and then add _c in the end of the name
if ( lookupField.endsWith( CUSTOM_OBJECT_RELATIONSHIP_FIELD_SUFFIX ) ) {
fieldToNullName =
lookupField.substring( 0, lookupField.length() - CUSTOM_OBJECT_RELATIONSHIP_FIELD_SUFFIX.length() )
+ CUSTOM_OBJECT_SUFFIX;
if ( log.isDebug() ) {
log.logDebug( BaseMessages.getString( PKG, "SalesforceUtils.Debug.NullFieldName", fieldToNullName ) );
}
return fieldToNullName;
}

fieldToNullName = lookupField + "Id";
}

if ( log.isDebug() ) {
log.logDebug( BaseMessages.getString( PKG, "SalesforceUtils.Debug.NullFieldName", fieldToNullName ) );
}

return fieldToNullName;
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
SalesforceUtils.Warn.IncorrectExternalKeySyntax=The field has incorrect external key syntax: {0}. Syntax for external key should be : object:externalId/lookupField. Trying to use fieldToNullName={1}.
SalesforceUtils.Debug.NullFieldName=fieldToNullName={0}
Loading

0 comments on commit 4316ce3

Please sign in to comment.