Skip to content

Commit

Permalink
SALTO-2450 log and retry on invalid suiteapp results (salto-io#3153)
Browse files Browse the repository at this point in the history
  • Loading branch information
@dantal4
dantal4 authored Jul 3, 2022
1 parent ac29df0 commit 52b98cc
Show file tree
Hide file tree
Showing 6 changed files with 182 additions and 41 deletions.
30 changes: 30 additions & 0 deletions packages/netsuite-adapter/src/client/suiteapp_client/errors.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,38 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { logger } from '@salto-io/logging'

const log = logger(module)

const REQUEST_MAX_RETRIES = 5

export class ReadFileError extends Error {}

export class ReadFileEncodingError extends ReadFileError {}

export class ReadFileInsufficientPermissionError extends ReadFileError {}

export class RetryableError extends Error {
constructor(readonly originalError: Error) {
super(originalError.message)
}
}

export const retryOnRetryableError = async <T>(
call: () => Promise<T>,
retriesLeft = REQUEST_MAX_RETRIES
): Promise<T> => {
try {
return await call()
} catch (e) {
if (e instanceof RetryableError) {
if (retriesLeft === 0) {
log.error('Retryable request exceed max retries with error: %s', e.message)
throw e.originalError
}
return retryOnRetryableError(call, retriesLeft - 1)
}
throw e
}
}
22 changes: 11 additions & 11 deletions packages/netsuite-adapter/src/client/suiteapp_client/soap_client/soap_client.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ const REQUEST_RETRY_DELAY = 5000
const NETSUITE_VERSION = '2020_2'
const SEARCH_PAGE_SIZE = 100

const RETRYABLE_MESSAGES = ['ECONN', 'UNEXPECTED_ERROR', 'INSUFFICIENT_PERMISSION']
const RETRYABLE_MESSAGES = ['ECONN', 'UNEXPECTED_ERROR', 'INSUFFICIENT_PERMISSION', 'VALIDATION_ERROR']
const SOAP_RETRYABLE_MESSAGES = ['CONCURRENT']

type SoapSearchType = {
Expand All @@ -67,8 +67,8 @@ const retryOnBadResponseWithDelay = (
// eslint-disable-next-line @typescript-eslint/return-await
return await call.call()
} catch (e) {
if (retryableMessages.some(message => e.message.toUpperCase().includes(message)
|| e.code?.toUpperCase?.()?.includes(message)) && retriesLeft > 0) {
if (retryableMessages.some(message => e?.message?.toUpperCase?.()?.includes?.(message)
|| e?.code?.toUpperCase?.()?.includes?.(message)) && retriesLeft > 0) {
log.warn('Retrying soap request with error: %s. Retries left: %d', e.message, retriesLeft)
if (retryDelay) {
await new Promise(f => setTimeout(f, retryDelay))
Expand Down Expand Up @@ -135,7 +135,7 @@ export default class SoapClient {
response
)) {
log.error(`Got invalid response from get request with id ${id} in SOAP api. Errors: ${this.ajv.errorsText()}. Response: ${JSON.stringify(response, undefined, 2)}`)
throw new Error(`Got invalid response from get request with id ${id} in SOAP api. Errors: ${this.ajv.errorsText()}. Response: ${JSON.stringify(response, undefined, 2)}`)
throw new Error(`VALIDATION_ERROR - Got invalid response from get request with id ${id} in SOAP api. Errors: ${this.ajv.errorsText()}. Response: ${JSON.stringify(response, undefined, 2)}`)
}

if (!isGetSuccess(response)) {
Expand Down Expand Up @@ -234,7 +234,7 @@ export default class SoapClient {
response
)) {
log.error(`Got invalid response from addList request with in SOAP api. Errors: ${this.ajv.errorsText()}. Response: ${JSON.stringify(response, undefined, 2)}`)
throw new Error(`Got invalid response from addList request. Errors: ${this.ajv.errorsText()}. Response: ${JSON.stringify(response, undefined, 2)}`)
throw new Error(`VALIDATION_ERROR - Got invalid response from addList request. Errors: ${this.ajv.errorsText()}. Response: ${JSON.stringify(response, undefined, 2)}`)
}

if (!isDeployListSuccess(response)) {
Expand Down Expand Up @@ -267,7 +267,7 @@ export default class SoapClient {
response
)) {
log.error(`Got invalid response from deleteList request with in SOAP api. Errors: ${this.ajv.errorsText()}. Response: ${JSON.stringify(response, undefined, 2)}`)
throw new Error(`Got invalid response from deleteList request. Errors: ${this.ajv.errorsText()}. Response: ${JSON.stringify(response, undefined, 2)}`)
throw new Error(`VALIDATION_ERROR - Got invalid response from deleteList request. Errors: ${this.ajv.errorsText()}. Response: ${JSON.stringify(response, undefined, 2)}`)
}

if (!isDeployListSuccess(response)) {
Expand Down Expand Up @@ -300,7 +300,7 @@ export default class SoapClient {
response
)) {
log.error(`Got invalid response from updateList request with in SOAP api. Errors: ${this.ajv.errorsText()}. Response: ${JSON.stringify(response, undefined, 2)}`)
throw new Error(`Got invalid response from updateList request. Errors: ${this.ajv.errorsText()}. Response: ${JSON.stringify(response, undefined, 2)}`)
throw new Error(`VALIDATION_ERROR - Got invalid response from updateList request. Errors: ${this.ajv.errorsText()}. Response: ${JSON.stringify(response, undefined, 2)}`)
}

if (!isDeployListSuccess(response)) {
Expand Down Expand Up @@ -442,7 +442,7 @@ export default class SoapClient {
response
)) {
log.error(`Got invalid response from ${action} request with in SOAP api. Errors: ${this.ajv.errorsText()}. Response: ${JSON.stringify(response, undefined, 2)}`)
throw new Error(`Got invalid response from ${action} request. Errors: ${this.ajv.errorsText()}. Response: ${JSON.stringify(response, undefined, 2)}`)
throw new Error(`VALIDATION_ERROR - Got invalid response from ${action} request. Errors: ${this.ajv.errorsText()}. Response: ${JSON.stringify(response, undefined, 2)}`)
}

if (!isDeployListSuccess(response)) {
Expand Down Expand Up @@ -544,7 +544,7 @@ export default class SoapClient {
response
)) {
log.error(`Got invalid response from get all request with in SOAP api. Errors: ${this.ajv.errorsText()}. Response: ${JSON.stringify(response, undefined, 2)}`)
throw new Error(`Got invalid response from get all request. Errors: ${this.ajv.errorsText()}. Response: ${JSON.stringify(response, undefined, 2)}`)
throw new Error(`VALIDATION_ERROR - Got invalid response from get all request. Errors: ${this.ajv.errorsText()}. Response: ${JSON.stringify(response, undefined, 2)}`)
}

return response.getAllResult.recordList.record
Expand Down Expand Up @@ -591,7 +591,7 @@ export default class SoapClient {
response
)) {
log.error(`Got invalid response from search request with SOAP api of type ${type}. Errors: ${this.ajv.errorsText()}. Response: ${JSON.stringify(response, undefined, 2)}`)
throw new Error(`Got invalid response from search request of type ${type}. Errors: ${this.ajv.errorsText()}. Response: ${JSON.stringify(response, undefined, 2)}`)
throw new Error(`VALIDATION_ERROR - Got invalid response from search request of type ${type}. Errors: ${this.ajv.errorsText()}. Response: ${JSON.stringify(response, undefined, 2)}`)
}
log.debug(`Finished sending search request for page 1/${Math.max(response.searchResult.totalPages, 1)} of type ${type}`)
return response
Expand All @@ -611,7 +611,7 @@ export default class SoapClient {
response
)) {
log.error(`Got invalid response from search with id request with in SOAP api. Id: ${args.searchId}, index: ${args.pageIndex}, errors: ${this.ajv.errorsText()}. Response: ${JSON.stringify(response, undefined, 2)}`)
throw new Error(`Got invalid response from search with id request. Errors: ${this.ajv.errorsText()}. Response: ${JSON.stringify(response, undefined, 2)}`)
throw new Error(`VALIDATION_ERROR - Got invalid response from search with id request. Errors: ${this.ajv.errorsText()}. Response: ${JSON.stringify(response, undefined, 2)}`)
}

return response
Expand Down
77 changes: 62 additions & 15 deletions packages/netsuite-adapter/src/client/suiteapp_client/suiteapp_client.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ import compareVersions from 'compare-versions'
import os from 'os'
import { logger } from '@salto-io/logging'
import _ from 'lodash'
import { values } from '@salto-io/lowerdash'
import { values, decorators } from '@salto-io/lowerdash'
import { safeJsonStringify } from '@salto-io/adapter-utils'
import { client as clientUtils } from '@salto-io/adapter-components'
import { WSDL } from 'soap'
Expand All @@ -40,7 +40,7 @@ import { SUITEAPP_CONFIG_RECORD_TYPES } from '../../types'
import { DEFAULT_CONCURRENCY } from '../../config'
import { CONSUMER_KEY, CONSUMER_SECRET } from './constants'
import SoapClient from './soap_client/soap_client'
import { ReadFileEncodingError, ReadFileError, ReadFileInsufficientPermissionError } from './errors'
import { ReadFileEncodingError, ReadFileError, ReadFileInsufficientPermissionError, RetryableError, retryOnRetryableError } from './errors'
import { InvalidSuiteAppCredentialsError } from '../types'

const { isDefined } = values
Expand Down Expand Up @@ -89,6 +89,12 @@ const getAxiosErrorDetailedMessage = (error: AxiosError): string | undefined =>
return detailedMessages.length > 0 ? detailedMessages.join(os.EOL) : undefined
}

export const retryable = decorators.wrapMethodWith(
async (
call: decorators.OriginalCall,
): Promise<unknown> => retryOnRetryableError(async () => call.call())
)

export default class SuiteAppClient {
private credentials: SuiteAppCredentials
private callsLimiter: CallsLimiter
Expand Down Expand Up @@ -191,7 +197,11 @@ export default class SuiteAppClient {
SYSTEM_INFORMATION_SCHEME,
results
)) {
log.error(`getSystemInformation failed. Got invalid results: ${this.ajv.errorsText()}`)
log.error(
'getSystemInformation failed. Got invalid results - %s: %o',
this.ajv.errorsText(),
results
)
return undefined
}

Expand All @@ -213,11 +223,12 @@ export default class SuiteAppClient {
try {
const results = await this.sendRestletRequest('readFile', { ids })

if (!this.ajv.validate<ReadResults>(
FILES_READ_SCHEMA,
results
)) {
log.error(`readFiles failed. Got invalid results: ${this.ajv.errorsText()}`)
if (!this.ajv.validate<ReadResults>(FILES_READ_SCHEMA, results)) {
log.error(
'readFiles failed. Got invalid results - %s: %o',
this.ajv.errorsText(),
results
)
return undefined
}

Expand Down Expand Up @@ -252,7 +263,11 @@ export default class SuiteAppClient {
})

if (!this.ajv.validate<GetConfigResult>(GET_CONFIG_RESULT_SCHEMA, result)) {
log.error('getConfigRecords failed. Got invalid results: %s', this.ajv.errorsText())
log.error(
'getConfigRecords failed. Got invalid results - %s: %o',
this.ajv.errorsText(),
result
)
return []
}

Expand All @@ -271,13 +286,23 @@ export default class SuiteAppClient {
return results.map(configRecord => {
const { configType, fieldsDef, data } = configRecord
if (!this.ajv.validate<ConfigRecordData>(CONFIG_RECORD_DATA_SCHEMA, data)) {
log.error('failed parsing ConfigRecordData of type \'%s\': %s', configType, this.ajv.errorsText())
log.error(
'failed parsing ConfigRecordData of type \'%s\' - %s: %o',
configType,
this.ajv.errorsText(),
data
)
return undefined
}

const validatedFields = fieldsDef.filter(fieldDef => {
if (!this.ajv.validate<ConfigFieldDefinition>(CONFIG_FIELD_DEFINITION_SCHEMA, fieldDef)) {
log.error('failed parsing ConfigFieldDefinition of type \'%s\': %s', configType, this.ajv.errorsText())
log.error(
'failed parsing ConfigFieldDefinition of type \'%s\' - %s: %o',
configType,
this.ajv.errorsText(),
fieldDef
)
return false
}
return true
Expand All @@ -302,7 +327,11 @@ export default class SuiteAppClient {
const result = await this.sendRestletRequest('config', { action: 'set', types })

if (!this.ajv.validate<SetConfigResult>(SET_CONFIG_RESULT_SCHEMA, result)) {
log.error('setConfigRecordsValues failed. Got invalid results: %s', this.ajv.errorsText())
log.error(
'setConfigRecordsValues failed. Got invalid results - %s: %o',
this.ajv.errorsText(),
result
)
return { errorMessage: this.ajv.errorsText() }
}
return result
Expand Down Expand Up @@ -348,6 +377,7 @@ export default class SuiteAppClient {
}
}

@retryable
private async sendSuiteQLRequest(query: string, offset: number, limit: number):
Promise<SuiteQLResults> {
const url = new URL(this.suiteQLUrl.href)
Expand All @@ -360,7 +390,12 @@ export default class SuiteAppClient {
}
const response = await this.safeAxiosPost(url.href, { q: query }, headers)
if (!this.ajv.validate<SuiteQLResults>(SUITE_QL_RESULTS_SCHEMA, response.data)) {
throw new Error(`Got invalid results from the SuiteQL query: ${this.ajv.errorsText()}`)
log.error(
'Got invalid results from the SuiteQL query - %s: %o',
this.ajv.errorsText(),
response.data
)
throw new RetryableError(new Error('Invalid SuiteQL query result'))
}

return response.data
Expand Down Expand Up @@ -399,6 +434,7 @@ export default class SuiteAppClient {
})
}

@retryable
private async innerSendRestletRequest(
operation: RestletOperation,
args: Record<string, unknown> = {}
Expand All @@ -418,7 +454,12 @@ export default class SuiteAppClient {
)

if (!this.ajv.validate<RestletResults>(RESTLET_RESULTS_SCHEMA, response.data)) {
throw new Error(`Got invalid results from a Restlet request: ${this.ajv.errorsText()}`)
log.error(
'Got invalid results from a Restlet request - %s: %o',
this.ajv.errorsText(),
response.data
)
throw new RetryableError(new Error('Invalid Restlet query result'))
}

if (isError(response.data)) {
Expand All @@ -436,6 +477,7 @@ export default class SuiteAppClient {
return this.innerSendRestletRequest(operation, args)
}

@retryable
private async sendSavedSearchRequest(query: SavedSearchQuery, offset: number, limit: number):
Promise<SavedSearchResults> {
const results = await this.sendRestletRequest('search', {
Expand All @@ -445,7 +487,12 @@ export default class SuiteAppClient {
})

if (!this.ajv.validate<SavedSearchResults>(SAVED_SEARCH_RESULTS_SCHEMA, results)) {
throw new Error(`Got invalid results from the saved search query: ${this.ajv.errorsText()}`)
log.error(
'Got invalid results from the saved search query - %s: %o',
this.ajv.errorsText(),
results
)
throw new RetryableError(new Error('Invalid Saved Search query error'))
}

return results
Expand Down
Loading

0 comments on commit 52b98cc

Please sign in to comment.