feat: Support Relative Path Imports (#891)

* Handle relative paths for stack imports

* Tests for relative paths

* Refactor handling of relative paths in ProcessImportSection

* Handle relative paths in StackImport.Path and string imports

* Add new test2/us-west-1.yaml file and update test counts

* Update import paths in us-west-1.yaml file

* Refactor handling of import paths in ProcessImportSection

* Resolve relative paths in ProcessImportSection function

* Update internal/exec/stack_processor_utils.go

Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>

* Refactor resolveRelativePath for improved path handling

* Refactor filepath handling for better readability

* Remove unnecessary path normalization code and simplify logic

* Improve path resolution handling in stack_processor_utils

* relative path handling improvements for windows; added security checks for import paths

* Validate import paths for security and base path inclusion

* Update path.Join calls to use filepath.Join for consistency

* Update stack_processor_utils.go to use filepath package

* validate stack paths to check against the global base path

* Refactor path resolution and validation logic

* Refactor resolving relative path logic for clarity

* Revert changes

* Refactor path resolution for consistency

* Refactor resolveRelativePath function for path consistency

* Convert relative path to base path if starts with "." or ".."

* Simplify resolving relative paths in stack_processor_utils

* added tests for atmos stacks with relative paths

* PR feedback

* merged main, incorporate new test fixture pattern

* Delete nonprod and prod cache weather files

* Delete outdated Atmos manifest JSON schema

* consistent indentation

* Add ignore rule for cache text files

* Update stack processor test cases and relative paths

* revert cahnges

* use relative paths in advanced usage example

* use relative paths in advanced usage example

* reset advanced usage

* added relative paths to demo-context

---------

Co-authored-by: Erik Osterman (CEO @ Cloud Posse) <[email protected]>
Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>

.gitignore

@@ -35,3 +35,4 @@
 .direnv/
 
 .atmos/cache.yaml
+**/cache.*.txt

examples/demo-context/stacks/deploy/dev/demo.yaml

@@ -1,7 +1,7 @@
 # yaml-language-server: $schema=https://atmos.tools/schemas/atmos/atmos-manifest/1.0/atmos-manifest.json
 
 import:
-- deploy/_defaults
+- ../_defaults
 - catalog/demo
 - mixins/west-coast
 

examples/demo-context/stacks/deploy/prod/demo.yaml

@@ -1,7 +1,7 @@
 # yaml-language-server: $schema=https://atmos.tools/schemas/atmos/atmos-manifest/1.0/atmos-manifest.json
 
 import:
-- deploy/_defaults
+- ../_defaults
 - catalog/demo
 - mixins/east-coast
 

examples/demo-context/stacks/deploy/staging/demo.yaml

@@ -1,7 +1,7 @@
 # yaml-language-server: $schema=https://atmos.tools/schemas/atmos/atmos-manifest/1.0/atmos-manifest.json
 
 import:
-- deploy/_defaults
+- ../_defaults
 - catalog/demo
 - mixins/east-coast
 

internal/exec/stack_processor_utils.go

@@ -44,7 +44,6 @@ func ProcessYAMLConfigFiles(
 	map[string]map[string]any,
 	error,
 ) {
-
 	count := len(filePaths)
 	listResult := make([]string, count)
 	mapResult := map[string]any{}
@@ -170,7 +169,6 @@ func ProcessYAMLConfigFile(
 	map[string]any,
 	error,
 ) {
-
 	var stackConfigs []map[string]any
 	relativeFilePath := u.TrimBasePathFromPath(basePath+"/", filePath)
 
@@ -468,6 +466,7 @@ func ProcessYAMLConfigFile(
 			if err != nil {
 				return nil, nil, nil, nil, nil, err
 			}
+
 			importRelativePathWithExt := strings.Replace(filepath.ToSlash(importFile), filepath.ToSlash(basePath)+"/", "", 1)
 			ext2 := filepath.Ext(importRelativePathWithExt)
 			if ext2 == "" {
@@ -1779,11 +1778,42 @@ func FindComponentDependenciesLegacy(
 	return unique, nil
 }
 
+// resolveRelativePath checks if a path is relative to the current directory and if so,
+// resolves it relative to the current file's directory. It ensures the resolved path
+// exists within the base path.
+func resolveRelativePath(path string, currentFilePath string) string {
+	if path == "" {
+		return path
+	}
+
+	// Convert all paths to use forward slashes for consistency in processing
+	normalizedPath := filepath.ToSlash(path)
+	normalizedCurrentFilePath := filepath.ToSlash(currentFilePath)
+
+	// Atmos import paths are generally relative paths, however, there are two types of relative paths:
+	//   1. Paths relative to the base path (most common) - e.g. "mixins/region/us-east-2"
+	//   2. Paths relative to the current file's directory (less common) - e.g. "./_defaults" imports will be relative to `./`
+	//
+	// Here we check if the path starts with "." or ".." to identify if it's relative to the current file.
+	// If it is, we'll convert it to be relative to the file doing the import, rather than the `base_path`.
+	parts := strings.Split(normalizedPath, "/")
+	firstElement := filepath.Clean(parts[0])
+	if firstElement == "." || firstElement == ".." {
+		// Join the current local path with the current stack file path
+		baseDir := filepath.Dir(normalizedCurrentFilePath)
+		relativePath := filepath.Join(baseDir, normalizedPath)
+		// Return in original format, OS-specific
+		return filepath.FromSlash(relativePath)
+	}
+	// For non-relative paths, return as-is in original format
+	return path
+}
+
 // ProcessImportSection processes the `import` section in stack manifests
-// The `import` section` can be of the following types:
-// 1. list of `StackImport` structs
-// 2. list of strings
-// 3. List of strings and `StackImport` structs in the same file
+// The `import` section can contain:
+// 1. Project-relative paths (e.g. "mixins/region/us-east-2")
+// 2. Paths relative to the current stack file (e.g. "./_defaults")
+// 3. StackImport structs containing either of the above path types (e.g. "path: mixins/region/us-east-2")
 func ProcessImportSection(stackMap map[string]any, filePath string) ([]schema.StackImport, error) {
 	stackImports, ok := stackMap[cfg.ImportSectionName]
 
@@ -1809,6 +1839,7 @@ func ProcessImportSection(stackMap map[string]any, filePath string) ([]schema.St
 		var importObj schema.StackImport
 		err := mapstructure.Decode(imp, &importObj)
 		if err == nil {
+			importObj.Path = resolveRelativePath(importObj.Path, filePath)
 			result = append(result, importObj)
 			continue
 		}
@@ -1822,6 +1853,7 @@ func ProcessImportSection(stackMap map[string]any, filePath string) ([]schema.St
 			return nil, fmt.Errorf("invalid empty import in the file '%s'", filePath)
 		}
 
+		s = resolveRelativePath(s, filePath)
 		result = append(result, schema.StackImport{Path: s})
 	}
 

pkg/stack/stack_processor_test.go

@@ -193,3 +193,55 @@ func TestStackProcessor(t *testing.T) {
 	assert.Nil(t, err)
 	t.Log(string(yamlConfig))
 }
+
+func TestStackProcessorRelativePaths(t *testing.T) {
+	stacksBasePath := "../../tests/fixtures/scenarios/relative-paths/stacks"
+	terraformComponentsBasePath := "../../tests/fixtures/components/terraform"
+
+	filePaths := []string{
+		"../../tests/fixtures/scenarios/relative-paths/stacks/orgs/acme/platform/dev.yaml",
+		"../../tests/fixtures/scenarios/relative-paths/stacks/orgs/acme/platform/prod.yaml",
+	}
+
+	atmosConfig := schema.AtmosConfiguration{
+		Templates: schema.Templates{
+			Settings: schema.TemplatesSettings{
+				Enabled: true,
+				Sprig: schema.TemplatesSettingsSprig{
+					Enabled: true,
+				},
+				Gomplate: schema.TemplatesSettingsGomplate{
+					Enabled: true,
+				},
+			},
+		},
+	}
+
+	listResult, mapResult, _, err := ProcessYAMLConfigFiles(
+		atmosConfig,
+		stacksBasePath,
+		terraformComponentsBasePath,
+		"",
+		filePaths,
+		true,
+		true,
+		false,
+	)
+
+	assert.Nil(t, err)
+	assert.Equal(t, 2, len(listResult))
+	assert.Equal(t, 2, len(mapResult))
+
+	mapResultKeys := u.StringKeysFromMap(mapResult)
+	assert.Equal(t, "orgs/acme/platform/dev", mapResultKeys[0])
+	assert.Equal(t, "orgs/acme/platform/prod", mapResultKeys[1])
+
+	mapConfig1, err := u.UnmarshalYAML[schema.AtmosSectionMapType](listResult[0])
+	assert.Nil(t, err)
+
+	components := mapConfig1["components"].(map[string]any)
+	terraformComponents := components["terraform"].(map[string]any)
+
+	myappComponent := terraformComponents["myapp"].(map[string]any)
+	assert.NotNil(t, myappComponent)
+}

tests/fixtures/scenarios/relative-paths/atmos.yaml

@@ -0,0 +1,21 @@
+base_path: "./"
+
+components:
+  terraform:
+    base_path: "components/terraform"
+    apply_auto_approve: false
+    deploy_run_init: true
+    init_run_reconfigure: true
+    auto_generate_backend_file: false
+
+stacks:
+  base_path: "stacks"
+  included_paths:
+    - "orgs/**/*"
+  excluded_paths:
+    - "**/_defaults.yaml"
+  name_pattern: "{namespace}-{tenant}-{stage}"
+
+logs:
+  file: "/dev/stderr"
+  level: Info

tests/fixtures/scenarios/relative-paths/components/terraform/myapp/README.md

@@ -0,0 +1,50 @@
+# Example Terraform Weather Component
+
+This Terraform "root" module fetches weather information for a specified location with custom display options.
+It queries data from the [`wttr.in`](https://wttr.in) weather service and stores the result in a local file (`cache.txt`). 
+It also provides several outputs like weather information, request URL, stage, location, language, and units of measurement.
+
+## Features
+
+- Fetch weather updates for a location using HTTP request.
+- Write the obtained weather data in a local file.
+- Customizable display options.
+- View the request URL.
+- Get informed about the stage, location, language, and units in the metadata.
+
+## Usage
+
+To include this module in your [Atmos Stacks](https://atmos.tools/core-concepts/stacks) configuration:
+
+```yaml
+components:
+  terraform:
+    weather:
+      vars:
+        stage: dev
+        location: New York
+        options: 0T
+        format: v2
+        lang: en
+        units: m
+```
+
+### Inputs
+- `stage`: Stage where it will be deployed.
+- `location`: Location for which the weather is reported. Default is "Los Angeles".
+- `options`: Options to customize the output. Default is "0T".
+- `format`: Specifies the output format. Default is "v2".
+- `lang`: Language in which the weather will be displayed. Default is "en".
+- `units`: Units in which the weather will be displayed. Default is "m".
+
+### Outputs
+- `weather`: The fetched weather data.
+- `url`: Requested URL.
+- `stage`: Stage of deployment.
+- `location`: Location of the reported weather.
+- `lang`: Language used for weather data.
+- `units`: Units of measurement for the weather data.
+
+Please note, this module requires Terraform version >=1.0.0, and you need to specify no other required providers.
+
+Happy Weather Tracking!

tests/fixtures/scenarios/relative-paths/components/terraform/myapp/main.tf

@@ -0,0 +1,22 @@
+locals {
+  url = format("https://wttr.in/%v?%v&format=%v&lang=%v&u=%v",
+    urlencode(var.location),
+    urlencode(var.options),
+    urlencode(var.format),
+    urlencode(var.lang),
+    urlencode(var.units),
+  )
+}
+
+data "http" "weather" {
+  url = local.url
+  request_headers = {
+    User-Agent = "curl"
+  }
+}
+
+# Now write this to a file (as an example of a resource)
+resource "local_file" "cache" {
+  filename = "cache.${var.stage}.txt"
+  content  = data.http.weather.response_body
+}

tests/fixtures/scenarios/relative-paths/components/terraform/myapp/outputs.tf

@@ -0,0 +1,27 @@
+output "weather" {
+  value = data.http.weather.response_body
+}
+
+output "url" {
+  value = local.url
+}
+
+output "stage" {
+  value       = var.stage
+  description = "Stage where it was deployed"
+}
+
+output "location" {
+  value       = var.location
+  description = "Location of the weather report."
+}
+
+output "lang" {
+  value       = var.lang
+  description = "Language which the weather is displayed."
+}
+
+output "units" {
+  value       = var.units
+  description = "Units the weather is displayed."
+}

tests/fixtures/scenarios/relative-paths/components/terraform/myapp/variables.tf

@@ -0,0 +1,34 @@
+variable "stage" {
+  description = "Stage where it will be deployed"
+  type        = string
+}
+
+variable "location" {
+  description = "Location for which the weather."
+  type        = string
+  default     = "Los Angeles"
+}
+
+variable "options" {
+  description = "Options to customize the output."
+  type        = string
+  default     = "0T"
+}
+
+variable "format" {
+  description = "Format of the output."
+  type        = string
+  default     = "v2"
+}
+
+variable "lang" {
+  description = "Language in which the weather is displayed."
+  type        = string
+  default     = "en"
+}
+
+variable "units" {
+  description = "Units in which the weather is displayed."
+  type        = string
+  default     = "m"
+}

tests/fixtures/scenarios/relative-paths/components/terraform/myapp/versions.tf

@@ -0,0 +1,5 @@
+terraform {
+  required_version = ">= 1.0.0"
+
+  required_providers {}
+}

tests/fixtures/scenarios/relative-paths/stacks/catalog/myapp.yaml

@@ -0,0 +1,11 @@
+# yaml-language-server: $schema=https://atmos.tools/schemas/atmos/atmos-manifest/1.0/atmos-manifest.json
+
+components:
+  terraform:
+    myapp:
+      vars:
+        location: Los Angeles
+        lang: en
+        format: ''
+        options: '0'
+        units: m

tests/fixtures/scenarios/relative-paths/stacks/orgs/acme/_defaults.yaml

@@ -0,0 +1,4 @@
+# yaml-language-server: $schema=https://atmos.tools/schemas/atmos/atmos-manifest/1.0/atmos-manifest.json
+
+vars:
+  namespace: acme

tests/fixtures/scenarios/relative-paths/stacks/orgs/acme/platform/_defaults.yaml

@@ -0,0 +1,7 @@
+# yaml-language-server: $schema=https://atmos.tools/schemas/atmos/atmos-manifest/1.0/atmos-manifest.json
+
+import:
+  - ../_defaults
+
+vars:
+  tenant: platform

tests/fixtures/scenarios/relative-paths/stacks/orgs/acme/platform/dev.yaml

@@ -0,0 +1,15 @@
+# yaml-language-server: $schema=https://atmos.tools/schemas/atmos/atmos-manifest/1.0/atmos-manifest.json
+
+vars:
+  stage: dev
+
+import:
+  - ./_defaults
+  - catalog/myapp
+
+components:
+  terraform:
+    myapp:
+      vars:
+        location: Stockholm
+        lang: se

tests/fixtures/scenarios/relative-paths/stacks/orgs/acme/platform/prod.yaml

@@ -0,0 +1,15 @@
+# yaml-language-server: $schema=https://atmos.tools/schemas/atmos/atmos-manifest/1.0/atmos-manifest.json
+
+vars:
+  stage: prod
+
+import:
+  - ./_defaults
+  - catalog/myapp
+
+components:
+  terraform:
+    myapp:
+      vars:
+        location: Los Angeles
+        lang: en