SALTO-1687 add support for standalone fields in referenced idFields (s…

…alto-io#2951)

* add support for standalone fields in referenced idFields and apply changes in Workato

packages/adapter-components/src/config/transformation.ts

@@ -56,7 +56,8 @@ export type TransformationConfig = {
   fieldsToHide?: FieldToHideType[]
   // fields to convert into their instances (and reference from the parent)
   standaloneFields?: StandaloneFieldConfigType[]
-
+  // set to true if the instance id depends on its parent
+  extendsParentId?: boolean
   // set '.' to indicate that the full object should be returned
   dataField?: string
   // set to true if the defined instance element has only one instance

packages/adapter-components/src/filters/referenced_instance_names.ts

@@ -36,6 +36,19 @@ const log = logger(module)
 const { isDefined } = lowerDashValues
 const { getUpdatedReference, createReferencesTransformFunc } = references
 
+type TransformationIdConfig = {
+  idFields: string[]
+  extendsParentId?: boolean
+}
+
+const getFirstParentElemId = (instance: InstanceElement): ElemID | undefined => {
+  const parentsElemIds = getParents(instance)
+    .filter(parent => isReferenceExpression(parent) && isInstanceElement(parent.value))
+    .map(parent => parent.elemID)
+  // we are only using the first parent ElemId
+  return parentsElemIds.length > 0 ? parentsElemIds[0] : undefined
+}
+
 const createInstanceReferencedNameParts = (
   instance: InstanceElement,
   idFields: string[],
@@ -62,8 +75,9 @@ const createInstanceReferencedNameParts = (
 
 /* Finds all elemIDs that the current instance relies on based on the idFields */
 const getInstanceNameDependencies = (
-  idFields: string[],
   instance: InstanceElement,
+  idFields: string[],
+  extendsParentId?: boolean,
 ): string[] => {
   const referencedInstances = idFields
     .filter(isReferencedIdField)
@@ -75,23 +89,24 @@ const getInstanceNameDependencies = (
       return undefined
     })
     .filter(isDefined)
+  const parentFullName = getFirstParentElemId(instance)?.getFullName()
+  if (extendsParentId && parentFullName !== undefined) {
+    referencedInstances.push(parentFullName)
+  }
   return referencedInstances
 }
 
 /* Calculates the new instance name and file path */
 const createInstanceNameAndFilePath = (
   instance: InstanceElement,
-  idFields: string[],
+  idConfig: TransformationIdConfig,
   configByType: Record<string, TransformationConfig>,
   getElemIdFunc?: ElemIdGetter,
 ): { newNaclName: string; filePath: string[] } => {
+  const { idFields } = idConfig
   const newNameParts = createInstanceReferencedNameParts(instance, idFields)
   const newName = joinInstanceNameParts(newNameParts) ?? instance.elemID.name
-  const parentIds = getParents(instance)
-    .filter(parent => isReferenceExpression(parent) && isInstanceElement(parent.value))
-    .map(parent => parent.value.elemID.name)
-  const parentName = parentIds.length > 0 ? parentIds[0] : undefined
-
+  const parentName = getFirstParentElemId(instance)?.name
   const { typeName, adapter } = instance.elemID
   const { fileNameFields, serviceIdField } = configByType[typeName]
 
@@ -161,6 +176,15 @@ const getReferencesToElemIds = (
   return refs
 }
 
+/* Creates the same nested path under the updated instance */
+const createUpdatedPath = (
+  oldPath: ElemID,
+  updatedInstance: InstanceElement
+): ElemID => {
+  const { path } = oldPath.createBaseID()
+  return updatedInstance.elemID.createNestedID(...path)
+}
+
 /* Update all references for all the renamed instances with the new elemIds */
 const updateAllReferences = ({
   referenceIndex,
@@ -192,7 +216,9 @@ const updateAllReferences = ({
         ref.path.createTopLevelParentID().parent.getFullName()
       ]
       if (topLevelInstance !== undefined) {
-        setPath(topLevelInstance, ref.path, updatedReference)
+        // update the path so it would match the new instance
+        const updatedPath = createUpdatedPath(ref.path, topLevelInstance as InstanceElement)
+        setPath(topLevelInstance, updatedPath, updatedReference)
       }
     })
   // We changed all the references to the original instance, so this entry can be removed
@@ -202,7 +228,7 @@ const updateAllReferences = ({
 /* Create a graph with instance names as nodes and instance name dependencies as edges */
 const createGraph = (
   instances: InstanceElement[],
-  instancesToIdFields: {instance: InstanceElement; idFields: string[]}[]
+  instanceToIdConfig: {instance: InstanceElement; idConfig: TransformationIdConfig}[]
 ): DAG<InstanceElement> => {
   const hasReferencedIdFields = (idFields: string[]): boolean => (
     idFields.some(field => isReferencedIdField(field))
@@ -218,12 +244,13 @@ const createGraph = (
   }
 
   const graph = new DAG<InstanceElement>()
-  instancesToIdFields.forEach(({ instance, idFields }) => {
-    if (hasReferencedIdFields(idFields)) {
+  instanceToIdConfig.forEach(({ instance, idConfig }) => {
+    const { idFields, extendsParentId } = idConfig
+    if (hasReferencedIdFields(idFields) || extendsParentId) {
       // removing duplicate elemIDs to create a graph
       // we can traverse based on references to unique elemIDs
       if (!isDuplicateInstance(instance.elemID.getFullName())) {
-        const nameDependencies = getInstanceNameDependencies(idFields, instance)
+        const nameDependencies = getInstanceNameDependencies(instance, idFields, extendsParentId)
           .filter(instanceName => !isDuplicateInstance(instanceName))
         graph.addNode(
           instance.elemID.getFullName(),
@@ -275,15 +302,18 @@ export const addReferencesToInstanceNames = async (
       ])
   )
 
-  const instancesToIdFields = instances.map(instance => ({
+  const instancesToIdConfig = instances.map(instance => ({
     instance,
-    idFields: configByType[instance.elemID.typeName].idFields,
+    idConfig: {
+      idFields: configByType[instance.elemID.typeName].idFields,
+      extendsParentId: configByType[instance.elemID.typeName].extendsParentId,
+    },
   }))
 
-  const graph = createGraph(instances, instancesToIdFields)
+  const graph = createGraph(instances, instancesToIdConfig)
 
-  const nameToInstanceIdFields = _.keyBy(
-    instancesToIdFields,
+  const nameToInstanceIdConfig = _.keyBy(
+    instancesToIdConfig,
     obj => obj.instance.elemID.getFullName()
   )
   const nameToInstance = _.keyBy(instances, i => i.elemID.getFullName())
@@ -294,13 +324,13 @@ export const addReferencesToInstanceNames = async (
 
   await awu(graph.evaluationOrder()).forEach(
     async graphNode => {
-      const instanceIdFields = nameToInstanceIdFields[graphNode.toString()]
-      if (instanceIdFields !== undefined) {
-        const { instance, idFields } = instanceIdFields
+      const instanceIdConfig = nameToInstanceIdConfig[graphNode.toString()]
+      if (instanceIdConfig !== undefined) {
+        const { instance, idConfig } = instanceIdConfig
         const originalFullName = instance.elemID.getFullName()
         const { newNaclName, filePath } = createInstanceNameAndFilePath(
           instance,
-          idFields,
+          idConfig,
           configByType,
           getElemIdFunc,
         )
@@ -313,6 +343,10 @@ export const addReferencesToInstanceNames = async (
           newElemId: newInstance.elemID,
         })
 
+        if (nameToInstance[originalFullName] !== undefined) {
+          nameToInstance[originalFullName] = newInstance
+        }
+
         const originalInstanceIdx = elements
           .findIndex(e => (e.elemID.getFullName()) === originalFullName)
         elements.splice(originalInstanceIdx, 1, newInstance)

packages/adapter-components/test/filters/referenced_instance_name.test.ts

@@ -103,6 +103,10 @@ describe('referenced instances', () => {
         name: 'lastRecipe',
         book_id: new ReferenceExpression(anotherBook.elemID, anotherBook),
       },
+      undefined,
+      {
+        _parent: [new ReferenceExpression(recipes[0].elemID, recipes[0])],
+      }
     )
     const groups = [
       new InstanceElement(
@@ -122,8 +126,37 @@ describe('referenced instances', () => {
         },
       ),
     ]
+    const folderType = new ObjectType({
+      elemID: new ElemID(ADAPTER_NAME, 'folder'),
+      fields: {
+        name: { refType: BuiltinTypes.STRING },
+      },
+    })
+    const folderOne = new InstanceElement(
+      'folderOne',
+      folderType,
+      {
+        name: 'Desktop',
+      },
+      undefined,
+      {
+        _parent: [new ReferenceExpression(lastRecipe.elemID, lastRecipe)],
+      }
+    )
+    const folderTwo = new InstanceElement(
+      'folderTwo',
+      folderType,
+      {
+        name: 'Documents',
+      },
+      undefined,
+      {
+        _parent: [new ReferenceExpression(lastRecipe.elemID, lastRecipe)],
+      }
+    )
     return [recipeType, bookType, ...recipes, anotherBook, rootBook,
-      sameRecipeOne, sameRecipeTwo, lastRecipe, groupType, ...groups]
+      sameRecipeOne, sameRecipeTwo, lastRecipe, groupType, ...groups,
+      folderType, folderOne, folderTwo]
   }
   const config = {
     apiDefinitions: {
@@ -143,6 +176,12 @@ describe('referenced instances', () => {
             idFields: ['name'],
           },
         },
+        folder: {
+          transformation: {
+            idFields: ['name'],
+            extendsParentId: true,
+          },
+        },
       },
       typeDefaults: {
         transformation: {
@@ -175,10 +214,12 @@ describe('referenced instances', () => {
         .map(e => e.elemID.getFullName()).sort())
         .toEqual(['myAdapter.book.instance.123_ROOT',
           'myAdapter.book.instance.456_123_ROOT',
+          'myAdapter.folder.instance.recipe123_123_ROOT__lastRecipe_456_123_ROOT__Desktop',
+          'myAdapter.folder.instance.recipe123_123_ROOT__lastRecipe_456_123_ROOT__Documents',
           'myAdapter.group.instance.group1',
           'myAdapter.group.instance.group2',
-          'myAdapter.recipe.instance.lastRecipe_456_123_ROOT',
           'myAdapter.recipe.instance.recipe123_123_ROOT',
+          'myAdapter.recipe.instance.recipe123_123_ROOT__lastRecipe_456_123_ROOT',
           'myAdapter.recipe.instance.recipe456_456_123_ROOT',
           'myAdapter.recipe.instance.sameRecipe',
           'myAdapter.recipe.instance.sameRecipe',
@@ -192,6 +233,7 @@ describe('referenced instances', () => {
       recipe: config.apiDefinitions.types.recipe.transformation,
       book: config.apiDefinitions.types.book.transformation,
       group: config.apiDefinitions.types.group.transformation,
+      folder: config.apiDefinitions.types.folder.transformation,
     }
 
     it('should change name and references correctly', async () => {
@@ -204,32 +246,39 @@ describe('referenced instances', () => {
       const sortedResult = result
         .filter(isInstanceElement)
         .map(i => i.elemID.getFullName()).sort()
-      expect(result.length).toEqual(10)
+      expect(result.length).toEqual(13)
       expect(sortedResult)
         .toEqual(['myAdapter.book.instance.123_ROOT',
           'myAdapter.book.instance.456_123_ROOT',
+          'myAdapter.folder.instance.recipe123_123_ROOT__lastRecipe_456_123_ROOT__Desktop',
+          'myAdapter.folder.instance.recipe123_123_ROOT__lastRecipe_456_123_ROOT__Documents',
           'myAdapter.group.instance.group1',
           'myAdapter.group.instance.group2',
-          'myAdapter.recipe.instance.lastRecipe_456_123_ROOT',
           'myAdapter.recipe.instance.recipe123_123_ROOT',
+          'myAdapter.recipe.instance.recipe123_123_ROOT__lastRecipe_456_123_ROOT',
           'myAdapter.recipe.instance.recipe456_456_123_ROOT',
         ])
     })
     it('should not change name for duplicate elemIDs', async () => {
       elements = generateElements()
       const result = await addReferencesToInstanceNames(
-        elements.slice(0, 2).concat(elements.slice(4, 9)),
+        elements.slice(0, 9).concat(elements.slice(12)),
         transformationConfigByType,
         transformationDefaultConfig
       )
-      expect(result.length).toEqual(7)
+      expect(result.length).toEqual(12)
       expect(result
         .map(e => e.elemID.getFullName()).sort())
         .toEqual(['myAdapter.book',
           'myAdapter.book.instance.123_ROOT',
           'myAdapter.book.instance.456_123_ROOT',
+          'myAdapter.folder',
+          'myAdapter.folder.instance.recipe123_123_ROOT__lastRecipe_456_123_ROOT__Desktop',
+          'myAdapter.folder.instance.recipe123_123_ROOT__lastRecipe_456_123_ROOT__Documents',
           'myAdapter.recipe',
-          'myAdapter.recipe.instance.lastRecipe_456_123_ROOT',
+          'myAdapter.recipe.instance.recipe123_123_ROOT',
+          'myAdapter.recipe.instance.recipe123_123_ROOT__lastRecipe_456_123_ROOT',
+          'myAdapter.recipe.instance.recipe456_456_123_ROOT',
           'myAdapter.recipe.instance.sameRecipe',
           'myAdapter.recipe.instance.sameRecipe',
         ])
@@ -245,9 +294,10 @@ describe('referenced instances', () => {
         'myAdapter.book.instance.rootBook',
         'myAdapter.book.instance.book',
         'myAdapter.recipe.instance.recipe123',
+        'myAdapter.recipe.instance.last',
       ])
       expect(Object.values(res).map(n => n.length))
-        .toEqual([4, 2, 2])
+        .toEqual([4, 2, 3, 2])
     })
   })
 })

packages/workato-adapter/src/adapter.ts

@@ -25,7 +25,7 @@ import { FilterCreator, Filter, filtersRunner } from './filter'
 import { FETCH_CONFIG, WorkatoConfig } from './config'
 import addRootFolderFilter from './filters/add_root_folder'
 import fieldReferencesFilter from './filters/field_references'
-import fixMultienvIDs from './filters/fix_multienv_ids'
+import referencedIdFieldsFilter from './filters/referenced_id_fields'
 import recipeCrossServiceReferencesFilter from './filters/cross_service/recipe_references'
 import serviceUrlFilter from './filters/service_url'
 import ducktypeCommonFilters from './filters/ducktype_common'
@@ -40,11 +40,11 @@ const { getAllElements } = elementUtils.ducktype
 
 export const DEFAULT_FILTERS = [
   addRootFolderFilter,
-  // fixMultienvIDs should run after addRootFolderFilter
-  fixMultienvIDs,
   // fieldReferencesFilter should run after all element manipulations are done
   fieldReferencesFilter,
   recipeCrossServiceReferencesFilter,
+  // referencedIdFieldsFilter should run after element references are resolved
+  referencedIdFieldsFilter,
   serviceUrlFilter,
   ...ducktypeCommonFilters,
 ]
@@ -88,6 +88,7 @@ export default class WorkatoAdapter implements AdapterOperations {
           fetch: config.fetch,
           apiDefinitions: config.apiDefinitions,
         },
+        getElemIdFunc,
         fetchQuery: this.fetchQuery,
       },
       filterCreators,

packages/workato-adapter/src/config.ts

@@ -88,7 +88,7 @@ export const DEFAULT_TYPES: Record<string, configUtils.TypeDuckTypeConfig> = {
       paginationField: 'since_id',
     },
     transformation: {
-      idFields: ['name', 'id'], // not multienv-friendly - see SALTO-1241
+      idFields: ['name', '&folder_id'],
       fieldsToHide: [
         ...FIELDS_TO_HIDE,
         { fieldName: 'id' },
@@ -110,6 +110,7 @@ export const DEFAULT_TYPES: Record<string, configUtils.TypeDuckTypeConfig> = {
   [RECIPE_CODE_TYPE]: {
     transformation: {
       idFields: [], // there is one code per recipe, so no need for additional details
+      extendsParentId: true,
     },
   },
   [FOLDER_TYPE]: {
@@ -122,7 +123,7 @@ export const DEFAULT_TYPES: Record<string, configUtils.TypeDuckTypeConfig> = {
       paginationField: 'page',
     },
     transformation: {
-      idFields: ['name', 'parent_id'], // not multienv-friendly - see SALTO-1241
+      idFields: ['name', '&parent_id'],
       fieldsToHide: [
         ...FIELDS_TO_HIDE,
         { fieldName: 'id' },