From bde802586ecb0c403ac94e4ecd6dba6855a263ec Mon Sep 17 00:00:00 2001 From: zazulam Date: Mon, 1 Jul 2024 23:29:37 -0400 Subject: [PATCH 1/7] feat(backend): move comp logic to workflow params Signed-off-by: zazulam Co-authored-by: droctothorpe Co-authored-by: andreafehrman Co-authored-by: MonicaZhang1 Co-authored-by: kylekaminky Co-authored-by: CarterFendley Signed-off-by: zazulam --- backend/src/v2/compiler/argocompiler/argo.go | 83 +++++++++++------ .../src/v2/compiler/argocompiler/argo_test.go | 27 ++++++ .../src/v2/compiler/argocompiler/container.go | 6 +- .../create_mount_delete_dynamic_pvc.yaml | 88 +++++++++--------- .../testdata/create_pod_metadata.yaml | 90 ++++++++++--------- .../argocompiler/testdata/hello_world.yaml | 29 +++--- .../argocompiler/testdata/importer.yaml | 19 ++-- 7 files changed, 206 insertions(+), 136 deletions(-) diff --git a/backend/src/v2/compiler/argocompiler/argo.go b/backend/src/v2/compiler/argocompiler/argo.go index a5cfed5faef..faf5b2b6984 100644 --- a/backend/src/v2/compiler/argocompiler/argo.go +++ b/backend/src/v2/compiler/argocompiler/argo.go @@ -16,6 +16,7 @@ package argocompiler import ( "fmt" + "strconv" "strings" wfapi "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" @@ -63,7 +64,7 @@ func Compile(jobArg *pipelinespec.PipelineJob, kubernetesSpecArg *pipelinespec.S if err != nil { return nil, err } - // fill root component default paramters to PipelineJob + // fill root component default parameters to PipelineJob specParams := spec.GetRoot().GetInputDefinitions().GetParameters() for name, param := range specParams { _, ok := job.RuntimeConfig.ParameterValues[name] @@ -108,6 +109,9 @@ func Compile(jobArg *pipelinespec.PipelineJob, kubernetesSpecArg *pipelinespec.S "pipelines.kubeflow.org/v2_component": "true", }, }, + Arguments: wfapi.Arguments{ + Parameters: []wfapi.Parameter{}, + }, ServiceAccountName: "pipeline-runner", Entrypoint: tmplEntrypoint, }, @@ -180,69 +184,96 @@ func (c *workflowCompiler) templateName(componentName string) string { return componentName } -// WIP: store component spec, task spec and executor spec in annotations - const ( - annotationComponents = "pipelines.kubeflow.org/components-" - annotationContainers = "pipelines.kubeflow.org/implementations-" - annotationKubernetesSpec = "pipelines.kubeflow.org/kubernetes-" + argumentsComponents = "components-" + argumentsContainers = "implementations-" + argumentsKubernetesSpec = "kubernetes-" ) func (c *workflowCompiler) saveComponentSpec(name string, spec *pipelinespec.ComponentSpec) error { - return c.saveProtoToAnnotation(annotationComponents+name, spec) + baseComponentName := ExtractBaseComponentName(argumentsComponents + name) + return c.saveProtoToArguments(baseComponentName, spec) } // useComponentSpec returns a placeholder we can refer to the component spec // in argo workflow fields. func (c *workflowCompiler) useComponentSpec(name string) (string, error) { - return c.annotationPlaceholder(annotationComponents + name) + baseComponentName := ExtractBaseComponentName(argumentsComponents + name) + return c.argumentsPlaceholder(baseComponentName) } func (c *workflowCompiler) saveComponentImpl(name string, msg proto.Message) error { - return c.saveProtoToAnnotation(annotationContainers+name, msg) + baseComponentName := ExtractBaseComponentName(argumentsContainers + name) + return c.saveProtoToArguments(baseComponentName, msg) } func (c *workflowCompiler) useComponentImpl(name string) (string, error) { - return c.annotationPlaceholder(annotationContainers + name) + baseComponentName := ExtractBaseComponentName(argumentsContainers + name) + return c.argumentsPlaceholder(baseComponentName) } func (c *workflowCompiler) saveKubernetesSpec(name string, spec *structpb.Struct) error { - return c.saveProtoToAnnotation(annotationKubernetesSpec+name, spec) + return c.saveProtoToArguments(argumentsKubernetesSpec+name, spec) } func (c *workflowCompiler) useKubernetesImpl(name string) (string, error) { - return c.annotationPlaceholder(annotationKubernetesSpec + name) + return c.argumentsPlaceholder(argumentsKubernetesSpec + name) } -// TODO(Bobgy): sanitize component name -func (c *workflowCompiler) saveProtoToAnnotation(name string, msg proto.Message) error { +// saveProtoToArguments saves a proto message to the workflow arguments. The +// message is serialized to JSON and stored in the workflow arguments and then +// referenced by the workflow templates using AWF templating syntax. The reason +// for storing it in the workflow arguments is because there is a 1-many +// relationship between components and tasks that reference them. The workflow +// arguments allow us to deduplicate the component logic (implementation & spec +// in IR), significantly reducing the size of the argo workflow manifest. +func (c *workflowCompiler) saveProtoToArguments(componentName string, msg proto.Message) error { if c == nil { return fmt.Errorf("compiler is nil") } - if c.wf.Annotations == nil { - c.wf.Annotations = make(map[string]string) + if c.wf.Spec.Arguments.Parameters == nil { + c.wf.Spec.Arguments = wfapi.Arguments{Parameters: []wfapi.Parameter{}} } - if _, alreadyExists := c.wf.Annotations[name]; alreadyExists { - return fmt.Errorf("annotation %q already exists", name) + if c.wf.Spec.Arguments.GetParameterByName(componentName) != nil { + return nil } json, err := stablyMarshalJSON(msg) if err != nil { - return fmt.Errorf("saving component spec of %q to annotations: %w", name, err) + return fmt.Errorf("saving component spec of %q to arguments: %w", componentName, err) } - // TODO(Bobgy): verify name adheres to Kubernetes annotation restrictions: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set - c.wf.Annotations[name] = json + c.wf.Spec.Arguments.Parameters = append(c.wf.Spec.Arguments.Parameters, wfapi.Parameter{ + Name: componentName, + Value: wfapi.AnyStringPtr(json), + }) return nil } -func (c *workflowCompiler) annotationPlaceholder(name string) (string, error) { +// argumentsPlaceholder checks for the unique component name within the workflow +// arguments and returns a template tag that references the component in the +// workflow arguments. +func (c *workflowCompiler) argumentsPlaceholder(componentName string) (string, error) { if c == nil { return "", fmt.Errorf("compiler is nil") } - if _, exists := c.wf.Annotations[name]; !exists { - return "", fmt.Errorf("using component spec: failed to find annotation %q", name) + if c.wf.Spec.Arguments.GetParameterByName(componentName) == nil { + return "", fmt.Errorf("using component spec: failed to find workflow parameter %q", componentName) } - // Reference: https://argoproj.github.io/argo-workflows/variables/ - return fmt.Sprintf("{{workflow.annotations.%s}}", name), nil + + return workflowParameter(componentName), nil +} + +// extractBaseComponentName removes the iteration suffix that the IR compiler +// adds to the component name. +func ExtractBaseComponentName(componentName string) string { + baseComponentName := componentName + componentNameArray := strings.Split(componentName, "-") + + if _, err := strconv.Atoi(componentNameArray[len(componentNameArray)-1]); err == nil { + baseComponentName = strings.Join(componentNameArray[:len(componentNameArray)-1], "-") + + } + + return baseComponentName } const ( diff --git a/backend/src/v2/compiler/argocompiler/argo_test.go b/backend/src/v2/compiler/argocompiler/argo_test.go index 6c92e54574c..f3bb1fdcb1c 100644 --- a/backend/src/v2/compiler/argocompiler/argo_test.go +++ b/backend/src/v2/compiler/argocompiler/argo_test.go @@ -137,3 +137,30 @@ func load(t *testing.T, path string, platformSpecPath string) (*pipelinespec.Pip } return job, nil } + +func Test_extractBaseComponentName(t *testing.T) { + tests := []struct { + name string + componentName string + expectedBaseName string + }{ + { + name: "With dash and int", + componentName: "component-2", + expectedBaseName: "component", + }, + { + name: "Without dash and int", + componentName: "component", + expectedBaseName: "component", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := argocompiler.ExtractBaseComponentName(tt.componentName) + if result != tt.expectedBaseName { + t.Errorf("Expected: %s, Got: %s", tt.expectedBaseName, result) + } + }) + } +} diff --git a/backend/src/v2/compiler/argocompiler/container.go b/backend/src/v2/compiler/argocompiler/container.go index 14ed4f70679..b04adc58f8a 100644 --- a/backend/src/v2/compiler/argocompiler/container.go +++ b/backend/src/v2/compiler/argocompiler/container.go @@ -358,9 +358,11 @@ func (c *workflowCompiler) addContainerExecutorTemplate(refName string) string { }, } // Update pod metadata if it defined in the Kubernetes Spec - if kubernetesConfigString, ok := c.wf.Annotations[annotationKubernetesSpec+refName]; ok { + kubernetesConfigParam := c.wf.Spec.Arguments.GetParameterByName(argumentsKubernetesSpec + refName) + + if kubernetesConfigParam != nil { k8sExecCfg := &kubernetesplatform.KubernetesExecutorConfig{} - if err := jsonpb.UnmarshalString(kubernetesConfigString, k8sExecCfg); err == nil { + if err := jsonpb.UnmarshalString(string(*kubernetesConfigParam.Value), k8sExecCfg); err == nil { extendPodMetadata(&executor.Metadata, k8sExecCfg) } } diff --git a/backend/src/v2/compiler/argocompiler/testdata/create_mount_delete_dynamic_pvc.yaml b/backend/src/v2/compiler/argocompiler/testdata/create_mount_delete_dynamic_pvc.yaml index e3b427d2455..45ba6039ee0 100644 --- a/backend/src/v2/compiler/argocompiler/testdata/create_mount_delete_dynamic_pvc.yaml +++ b/backend/src/v2/compiler/argocompiler/testdata/create_mount_delete_dynamic_pvc.yaml @@ -1,36 +1,36 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - annotations: - pipelines.kubeflow.org/components-comp-comp: '{"executorLabel":"exec-comp"}' - pipelines.kubeflow.org/components-comp-comp-2: '{"executorLabel":"exec-comp-2"}' - pipelines.kubeflow.org/components-comp-createpvc: '{"executorLabel":"exec-createpvc","inputDefinitions":{"parameters":{"access_modes":{"parameterType":"LIST"},"annotations":{"isOptional":true,"parameterType":"STRUCT"},"pvc_name":{"isOptional":true,"parameterType":"STRING"},"pvc_name_suffix":{"isOptional":true,"parameterType":"STRING"},"size":{"parameterType":"STRING"},"storage_class_name":{"defaultValue":"","isOptional":true,"parameterType":"STRING"},"volume_name":{"isOptional":true,"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"name":{"parameterType":"STRING"}}}}' - pipelines.kubeflow.org/components-comp-deletepvc: '{"executorLabel":"exec-deletepvc","inputDefinitions":{"parameters":{"pvc_name":{"parameterType":"STRING"}}}}' - pipelines.kubeflow.org/components-root: '{"dag":{"tasks":{"comp":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-comp"},"dependentTasks":["createpvc"],"taskInfo":{"name":"comp"}},"comp-2":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-comp-2"},"dependentTasks":["comp","createpvc"],"taskInfo":{"name":"comp-2"}},"createpvc":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-createpvc"},"inputs":{"parameters":{"access_modes":{"runtimeValue":{"constant":["ReadWriteOnce"]}},"pvc_name_suffix":{"runtimeValue":{"constant":"-my-pvc"}},"size":{"runtimeValue":{"constant":"5Gi"}},"storage_class_name":{"runtimeValue":{"constant":"standard"}}}},"taskInfo":{"name":"createpvc"}},"deletepvc":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-deletepvc"},"dependentTasks":["comp-2","createpvc"],"inputs":{"parameters":{"pvc_name":{"taskOutputParameter":{"outputParameterKey":"name","producerTask":"createpvc"}}}},"taskInfo":{"name":"deletepvc"}}}}}' - pipelines.kubeflow.org/implementations-comp-comp: '{"args":["--executor_input","{{$}}","--function_to_execute","comp"],"command":["sh","-c","\nif - ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m - ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 - python3 -m pip install --quiet --no-warn-script-location ''kfp==2.0.0-beta.16'' - \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d) printf \"%s\" - \"$0\" \u003e \"$program_path/ephemeral_component.py\" python3 -m kfp.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\" - ","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import - *\n\ndef comp():\n pass\n\n"],"image":"python:3.7"}' - pipelines.kubeflow.org/implementations-comp-comp-2: '{"args":["--executor_input","{{$}}","--function_to_execute","comp"],"command":["sh","-c","\nif - ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m - ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 - python3 -m pip install --quiet --no-warn-script-location ''kfp==2.0.0-beta.16'' - \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d) printf \"%s\" - \"$0\" \u003e \"$program_path/ephemeral_component.py\" python3 -m kfp.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\" - ","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import - *\n\ndef comp():\n pass\n\n"],"image":"python:3.7"}' - pipelines.kubeflow.org/implementations-comp-createpvc: '{"image":"argostub/createpvc"}' - pipelines.kubeflow.org/implementations-comp-deletepvc: '{"image":"argostub/deletepvc"}' - pipelines.kubeflow.org/kubernetes-comp-comp: '{"pvcMount":[{"mountPath":"/data","taskOutputParameter":{"outputParameterKey":"name","producerTask":"createpvc"}}]}' - pipelines.kubeflow.org/kubernetes-comp-comp-2: '{"pvcMount":[{"mountPath":"/reused_data","taskOutputParameter":{"outputParameterKey":"name","producerTask":"createpvc"}}]}' creationTimestamp: null generateName: my-pipeline- spec: - arguments: {} + arguments: + parameters: + - name: kubernetes-comp-comp + value: '{"pvcMount":[{"mountPath":"/data","taskOutputParameter":{"outputParameterKey":"name","producerTask":"createpvc"}}]}' + - name: components-comp-comp + value: '{"executorLabel":"exec-comp"}' + - name: implementations-comp-comp + value: '{"args":["--executor_input","{{$}}","--function_to_execute","comp"],"command":["sh","-c","\nif + ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 + -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 + python3 -m pip install --quiet --no-warn-script-location ''kfp==2.0.0-beta.16'' + \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d) printf + \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\" python3 -m kfp.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\" + ","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import + *\n\ndef comp():\n pass\n\n"],"image":"python:3.7"}' + - name: kubernetes-comp-comp-2 + value: '{"pvcMount":[{"mountPath":"/reused_data","taskOutputParameter":{"outputParameterKey":"name","producerTask":"createpvc"}}]}' + - name: components-comp-createpvc + value: '{"executorLabel":"exec-createpvc","inputDefinitions":{"parameters":{"access_modes":{"parameterType":"LIST"},"annotations":{"isOptional":true,"parameterType":"STRUCT"},"pvc_name":{"isOptional":true,"parameterType":"STRING"},"pvc_name_suffix":{"isOptional":true,"parameterType":"STRING"},"size":{"parameterType":"STRING"},"storage_class_name":{"defaultValue":"","isOptional":true,"parameterType":"STRING"},"volume_name":{"isOptional":true,"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"name":{"parameterType":"STRING"}}}}' + - name: implementations-comp-createpvc + value: '{"image":"argostub/createpvc"}' + - name: components-comp-deletepvc + value: '{"executorLabel":"exec-deletepvc","inputDefinitions":{"parameters":{"pvc_name":{"parameterType":"STRING"}}}}' + - name: implementations-comp-deletepvc + value: '{"image":"argostub/deletepvc"}' + - name: components-root + value: '{"dag":{"tasks":{"comp":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-comp"},"dependentTasks":["createpvc"],"taskInfo":{"name":"comp"}},"comp-2":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-comp-2"},"dependentTasks":["comp","createpvc"],"taskInfo":{"name":"comp-2"}},"createpvc":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-createpvc"},"inputs":{"parameters":{"access_modes":{"runtimeValue":{"constant":["ReadWriteOnce"]}},"pvc_name_suffix":{"runtimeValue":{"constant":"-my-pvc"}},"size":{"runtimeValue":{"constant":"5Gi"}},"storage_class_name":{"runtimeValue":{"constant":"standard"}}}},"taskInfo":{"name":"createpvc"}},"deletepvc":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-deletepvc"},"dependentTasks":["comp-2","createpvc"],"inputs":{"parameters":{"pvc_name":{"taskOutputParameter":{"outputParameterKey":"name","producerTask":"createpvc"}}}},"taskInfo":{"name":"deletepvc"}}}}}' entrypoint: entrypoint podMetadata: annotations: @@ -180,32 +180,32 @@ spec: volumes: - emptyDir: {} name: kfp-launcher - - emptyDir: { } + - emptyDir: {} name: gcs-scratch - - emptyDir: { } + - emptyDir: {} name: s3-scratch - - emptyDir: { } + - emptyDir: {} name: minio-scratch - - emptyDir: { } + - emptyDir: {} name: dot-local-scratch - - emptyDir: { } + - emptyDir: {} name: dot-cache-scratch - - emptyDir: { } + - emptyDir: {} name: dot-config-scratch - dag: tasks: - arguments: parameters: - name: component - value: '{{workflow.annotations.pipelines.kubeflow.org/components-comp-comp}}' + value: '{{workflow.parameters.components-comp-comp}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-comp"},"dependentTasks":["createpvc"],"taskInfo":{"name":"comp"}}' - name: container - value: '{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-comp}}' + value: '{{workflow.parameters.implementations-comp-comp}}' - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' - name: kubernetes-config - value: '{{workflow.annotations.pipelines.kubeflow.org/kubernetes-comp-comp}}' + value: '{{workflow.parameters.kubernetes-comp-comp}}' depends: createpvc.Succeeded name: comp-driver template: system-container-driver @@ -222,15 +222,15 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.annotations.pipelines.kubeflow.org/components-comp-comp-2}}' + value: '{{workflow.parameters.components-comp-comp}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-comp-2"},"dependentTasks":["comp","createpvc"],"taskInfo":{"name":"comp-2"}}' - name: container - value: '{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-comp-2}}' + value: '{{workflow.parameters.implementations-comp-comp}}' - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' - name: kubernetes-config - value: '{{workflow.annotations.pipelines.kubeflow.org/kubernetes-comp-comp-2}}' + value: '{{workflow.parameters.kubernetes-comp-comp-2}}' depends: comp.Succeeded && createpvc.Succeeded name: comp-2-driver template: system-container-driver @@ -247,11 +247,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.annotations.pipelines.kubeflow.org/components-comp-createpvc}}' + value: '{{workflow.parameters.components-comp-createpvc}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-createpvc"},"inputs":{"parameters":{"access_modes":{"runtimeValue":{"constant":["ReadWriteOnce"]}},"pvc_name_suffix":{"runtimeValue":{"constant":"-my-pvc"}},"size":{"runtimeValue":{"constant":"5Gi"}},"storage_class_name":{"runtimeValue":{"constant":"standard"}}}},"taskInfo":{"name":"createpvc"}}' - name: container - value: '{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-createpvc}}' + value: '{{workflow.parameters.implementations-comp-createpvc}}' - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' name: createpvc @@ -259,11 +259,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.annotations.pipelines.kubeflow.org/components-comp-deletepvc}}' + value: '{{workflow.parameters.components-comp-deletepvc}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-deletepvc"},"dependentTasks":["comp-2","createpvc"],"inputs":{"parameters":{"pvc_name":{"taskOutputParameter":{"outputParameterKey":"name","producerTask":"createpvc"}}}},"taskInfo":{"name":"deletepvc"}}' - name: container - value: '{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-deletepvc}}' + value: '{{workflow.parameters.implementations-comp-deletepvc}}' - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' depends: comp-2.Succeeded && createpvc.Succeeded @@ -343,7 +343,7 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.annotations.pipelines.kubeflow.org/components-root}}' + value: '{{workflow.parameters.components-root}}' - name: runtime-config value: '{}' - name: driver-type diff --git a/backend/src/v2/compiler/argocompiler/testdata/create_pod_metadata.yaml b/backend/src/v2/compiler/argocompiler/testdata/create_pod_metadata.yaml index 450aed6aeca..1aa1a3fee46 100644 --- a/backend/src/v2/compiler/argocompiler/testdata/create_pod_metadata.yaml +++ b/backend/src/v2/compiler/argocompiler/testdata/create_pod_metadata.yaml @@ -1,20 +1,24 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - annotations: - pipelines.kubeflow.org/components-comp-hello-world: '{"executorLabel":"exec-hello-world","inputDefinitions":{"parameters":{"text":{"type":"STRING"}}}}' - pipelines.kubeflow.org/components-root: '{"dag":{"tasks":{"hello-world":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"hello-world"}}}},"inputDefinitions":{"parameters":{"text":{"type":"STRING"}}}}' - pipelines.kubeflow.org/implementations-comp-hello-world: '{"args":["--text","{{$.inputs.parameters[''text'']}}"],"command":["sh","-ec","program_path=$(mktemp)\nprintf - \"%s\" \"$0\" \u003e \"$program_path\"\npython3 -u \"$program_path\" \"$@\"\n","def - hello_world(text):\n print(text)\n return text\n\nimport argparse\n_parser - = argparse.ArgumentParser(prog=''Hello world'', description='''')\n_parser.add_argument(\"--text\", - dest=\"text\", type=str, required=True, default=argparse.SUPPRESS)\n_parsed_args - = vars(_parser.parse_args())\n\n_outputs = hello_world(**_parsed_args)\n"],"image":"python:3.7"}' - pipelines.kubeflow.org/kubernetes-comp-hello-world: '{"podMetadata":{"annotations":{"experiment_id":"234567","run_id":"123456"},"labels":{"kubeflow.com/common":"test","kubeflow.com/kfp":"pipeline-node"}}}' creationTimestamp: null generateName: hello-world- spec: - arguments: {} + arguments: + parameters: + - name: kubernetes-comp-hello-world + value: '{"podMetadata":{"annotations":{"experiment_id":"234567","run_id":"123456"},"labels":{"kubeflow.com/common":"test","kubeflow.com/kfp":"pipeline-node"}}}' + - name: components-comp-hello-world + value: '{"executorLabel":"exec-hello-world","inputDefinitions":{"parameters":{"text":{"type":"STRING"}}}}' + - name: implementations-comp-hello-world + value: '{"args":["--text","{{$.inputs.parameters[''text'']}}"],"command":["sh","-ec","program_path=$(mktemp)\nprintf + \"%s\" \"$0\" \u003e \"$program_path\"\npython3 -u \"$program_path\" \"$@\"\n","def + hello_world(text):\n print(text)\n return text\n\nimport argparse\n_parser + = argparse.ArgumentParser(prog=''Hello world'', description='''')\n_parser.add_argument(\"--text\", + dest=\"text\", type=str, required=True, default=argparse.SUPPRESS)\n_parsed_args + = vars(_parser.parse_args())\n\n_outputs = hello_world(**_parsed_args)\n"],"image":"python:3.7"}' + - name: components-root + value: '{"dag":{"tasks":{"hello-world":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"hello-world"}}}},"inputDefinitions":{"parameters":{"text":{"type":"STRING"}}}}' entrypoint: entrypoint podMetadata: annotations: @@ -124,20 +128,20 @@ spec: name: "" resources: {} volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - - mountPath: /gcs - name: gcs-scratch - - mountPath: /s3 - name: s3-scratch - - mountPath: /minio - name: minio-scratch - - mountPath: /.local - name: dot-local-scratch - - mountPath: /.cache - name: dot-cache-scratch - - mountPath: /.config - name: dot-config-scratch + - mountPath: /kfp-launcher + name: kfp-launcher + - mountPath: /gcs + name: gcs-scratch + - mountPath: /s3 + name: s3-scratch + - mountPath: /minio + name: minio-scratch + - mountPath: /.local + name: dot-local-scratch + - mountPath: /.cache + name: dot-cache-scratch + - mountPath: /.config + name: dot-config-scratch initContainers: - command: - launcher-v2 @@ -168,34 +172,34 @@ spec: outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' volumes: - - emptyDir: {} - name: kfp-launcher - - emptyDir: { } - name: gcs-scratch - - emptyDir: { } - name: s3-scratch - - emptyDir: { } - name: minio-scratch - - emptyDir: { } - name: dot-local-scratch - - emptyDir: { } - name: dot-cache-scratch - - emptyDir: { } - name: dot-config-scratch + - emptyDir: {} + name: kfp-launcher + - emptyDir: {} + name: gcs-scratch + - emptyDir: {} + name: s3-scratch + - emptyDir: {} + name: minio-scratch + - emptyDir: {} + name: dot-local-scratch + - emptyDir: {} + name: dot-cache-scratch + - emptyDir: {} + name: dot-config-scratch - dag: tasks: - arguments: parameters: - name: component - value: '{{workflow.annotations.pipelines.kubeflow.org/components-comp-hello-world}}' + value: '{{workflow.parameters.components-comp-hello-world}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"hello-world"}}' - name: container - value: '{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-hello-world}}' + value: '{{workflow.parameters.implementations-comp-hello-world}}' - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' - name: kubernetes-config - value: '{{workflow.annotations.pipelines.kubeflow.org/kubernetes-comp-hello-world}}' + value: '{{workflow.parameters.kubernetes-comp-hello-world}}' name: hello-world-driver template: system-container-driver - arguments: @@ -282,7 +286,7 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.annotations.pipelines.kubeflow.org/components-root}}' + value: '{{workflow.parameters.components-root}}' - name: runtime-config value: '{"parameters":{"text":{"stringValue":"hi there"}}}' - name: driver-type diff --git a/backend/src/v2/compiler/argocompiler/testdata/hello_world.yaml b/backend/src/v2/compiler/argocompiler/testdata/hello_world.yaml index 5685ece5de5..d59959225e5 100644 --- a/backend/src/v2/compiler/argocompiler/testdata/hello_world.yaml +++ b/backend/src/v2/compiler/argocompiler/testdata/hello_world.yaml @@ -1,19 +1,22 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - annotations: - pipelines.kubeflow.org/components-comp-hello-world: '{"executorLabel":"exec-hello-world","inputDefinitions":{"parameters":{"text":{"type":"STRING"}}}}' - pipelines.kubeflow.org/components-root: '{"dag":{"tasks":{"hello-world":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"hello-world"}}}},"inputDefinitions":{"parameters":{"text":{"type":"STRING"}}}}' - pipelines.kubeflow.org/implementations-comp-hello-world: '{"args":["--text","{{$.inputs.parameters[''text'']}}"],"command":["sh","-ec","program_path=$(mktemp)\nprintf - \"%s\" \"$0\" \u003e \"$program_path\"\npython3 -u \"$program_path\" \"$@\"\n","def - hello_world(text):\n print(text)\n return text\n\nimport argparse\n_parser - = argparse.ArgumentParser(prog=''Hello world'', description='''')\n_parser.add_argument(\"--text\", - dest=\"text\", type=str, required=True, default=argparse.SUPPRESS)\n_parsed_args - = vars(_parser.parse_args())\n\n_outputs = hello_world(**_parsed_args)\n"],"image":"python:3.7"}' creationTimestamp: null generateName: hello-world- spec: - arguments: {} + arguments: + parameters: + - name: components-comp-hello-world + value: '{"executorLabel":"exec-hello-world","inputDefinitions":{"parameters":{"text":{"type":"STRING"}}}}' + - name: implementations-comp-hello-world + value: '{"args":["--text","{{$.inputs.parameters[''text'']}}"],"command":["sh","-ec","program_path=$(mktemp)\nprintf + \"%s\" \"$0\" \u003e \"$program_path\"\npython3 -u \"$program_path\" \"$@\"\n","def + hello_world(text):\n print(text)\n return text\n\nimport argparse\n_parser + = argparse.ArgumentParser(prog=''Hello world'', description='''')\n_parser.add_argument(\"--text\", + dest=\"text\", type=str, required=True, default=argparse.SUPPRESS)\n_parsed_args + = vars(_parser.parse_args())\n\n_outputs = hello_world(**_parsed_args)\n"],"image":"python:3.7"}' + - name: components-root + value: '{"dag":{"tasks":{"hello-world":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"hello-world"}}}},"inputDefinitions":{"parameters":{"text":{"type":"STRING"}}}}' entrypoint: entrypoint podMetadata: annotations: @@ -180,11 +183,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.annotations.pipelines.kubeflow.org/components-comp-hello-world}}' + value: '{{workflow.parameters.components-comp-hello-world}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"hello-world"}}' - name: container - value: '{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-hello-world}}' + value: '{{workflow.parameters.implementations-comp-hello-world}}' - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' name: hello-world-driver @@ -273,7 +276,7 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.annotations.pipelines.kubeflow.org/components-root}}' + value: '{{workflow.parameters.components-root}}' - name: runtime-config value: '{"parameters":{"text":{"stringValue":"hi there"}}}' - name: driver-type diff --git a/backend/src/v2/compiler/argocompiler/testdata/importer.yaml b/backend/src/v2/compiler/argocompiler/testdata/importer.yaml index d0e6ef6eaae..23ec461c307 100644 --- a/backend/src/v2/compiler/argocompiler/testdata/importer.yaml +++ b/backend/src/v2/compiler/argocompiler/testdata/importer.yaml @@ -1,14 +1,17 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - annotations: - pipelines.kubeflow.org/components-comp-importer: '{"executorLabel":"exec-importer","inputDefinitions":{"parameters":{"uri":{"type":"STRING"}}},"outputDefinitions":{"artifacts":{"artifact":{"artifactType":{"schemaTitle":"system.Dataset"}}}}}' - pipelines.kubeflow.org/components-root: '{"dag":{"tasks":{"importer":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-importer"},"inputs":{"parameters":{"uri":{"runtimeValue":{"constantValue":{"stringValue":"gs://ml-pipeline-playground/shakespeare1.txt"}}}}},"taskInfo":{"name":"importer"}}}},"inputDefinitions":{"parameters":{"dataset2":{"type":"STRING"}}}}' - pipelines.kubeflow.org/implementations-comp-importer: '{"artifactUri":{"constantValue":{"stringValue":"gs://ml-pipeline-playground/shakespeare1.txt"}},"typeSchema":{"schemaTitle":"system.Dataset"}}' creationTimestamp: null generateName: pipeline-with-importer- spec: - arguments: {} + arguments: + parameters: + - name: components-comp-importer + value: '{"executorLabel":"exec-importer","inputDefinitions":{"parameters":{"uri":{"type":"STRING"}}},"outputDefinitions":{"artifacts":{"artifact":{"artifactType":{"schemaTitle":"system.Dataset"}}}}}' + - name: implementations-comp-importer + value: '{"artifactUri":{"constantValue":{"stringValue":"gs://ml-pipeline-playground/shakespeare1.txt"}},"typeSchema":{"schemaTitle":"system.Dataset"}}' + - name: components-root + value: '{"dag":{"tasks":{"importer":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-importer"},"inputs":{"parameters":{"uri":{"runtimeValue":{"constantValue":{"stringValue":"gs://ml-pipeline-playground/shakespeare1.txt"}}}}},"taskInfo":{"name":"importer"}}}},"inputDefinitions":{"parameters":{"dataset2":{"type":"STRING"}}}}' entrypoint: entrypoint podMetadata: annotations: @@ -81,9 +84,9 @@ spec: - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-importer"},"inputs":{"parameters":{"uri":{"runtimeValue":{"constantValue":{"stringValue":"gs://ml-pipeline-playground/shakespeare1.txt"}}}}},"taskInfo":{"name":"importer"}}' - name: component - value: '{{workflow.annotations.pipelines.kubeflow.org/components-comp-importer}}' + value: '{{workflow.parameters.components-comp-importer}}' - name: importer - value: '{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-importer}}' + value: '{{workflow.parameters.implementations-comp-importer}}' - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' name: importer @@ -162,7 +165,7 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.annotations.pipelines.kubeflow.org/components-root}}' + value: '{{workflow.parameters.components-root}}' - name: runtime-config value: '{}' - name: driver-type From 15aa78081143789a3f82259b708e63a9668fa765 Mon Sep 17 00:00:00 2001 From: zazulam Date: Mon, 15 Jul 2024 17:08:13 -0400 Subject: [PATCH 2/7] address pr comments Signed-off-by: zazulam --- backend/src/v2/compiler/argocompiler/argo.go | 3 +- .../src/v2/compiler/argocompiler/argo_test.go | 15 ++ sdk/python/kfp/compiler/compiler_test.py | 28 +-- sdk/python/kfp/dsl/component_factory.py | 7 + sdk/python/kfp/dsl/component_factory_test.py | 24 +++ .../pipelines/if_elif_else_complex.py | 4 +- .../pipelines/if_elif_else_complex.yaml | 140 ++++++------- .../pipeline_with_parallelfor_parallelism.py | 18 +- ...pipeline_with_parallelfor_parallelism.yaml | 196 +++++++++--------- 9 files changed, 240 insertions(+), 195 deletions(-) diff --git a/backend/src/v2/compiler/argocompiler/argo.go b/backend/src/v2/compiler/argocompiler/argo.go index faf5b2b6984..b6899f9b6c6 100644 --- a/backend/src/v2/compiler/argocompiler/argo.go +++ b/backend/src/v2/compiler/argocompiler/argo.go @@ -262,7 +262,7 @@ func (c *workflowCompiler) argumentsPlaceholder(componentName string) (string, e return workflowParameter(componentName), nil } -// extractBaseComponentName removes the iteration suffix that the IR compiler +// ExtractBaseComponentName removes the iteration suffix that the IR compiler // adds to the component name. func ExtractBaseComponentName(componentName string) string { baseComponentName := componentName @@ -270,7 +270,6 @@ func ExtractBaseComponentName(componentName string) string { if _, err := strconv.Atoi(componentNameArray[len(componentNameArray)-1]); err == nil { baseComponentName = strings.Join(componentNameArray[:len(componentNameArray)-1], "-") - } return baseComponentName diff --git a/backend/src/v2/compiler/argocompiler/argo_test.go b/backend/src/v2/compiler/argocompiler/argo_test.go index f3bb1fdcb1c..3f6ccb88afe 100644 --- a/backend/src/v2/compiler/argocompiler/argo_test.go +++ b/backend/src/v2/compiler/argocompiler/argo_test.go @@ -154,6 +154,21 @@ func Test_extractBaseComponentName(t *testing.T) { componentName: "component", expectedBaseName: "component", }, + { + name: "Last char is int", + componentName: "component-v2", + expectedBaseName: "component-v2", + }, + { + name: "Multiple dashes, ends with int", + componentName: "service-api-v2", + expectedBaseName: "service-api-v2", + }, + { + name: "Multiple dashes and ints", + componentName: "module-1-2-3", + expectedBaseName: "module-1-2", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/sdk/python/kfp/compiler/compiler_test.py b/sdk/python/kfp/compiler/compiler_test.py index d417d9eec19..3051dca17fd 100644 --- a/sdk/python/kfp/compiler/compiler_test.py +++ b/sdk/python/kfp/compiler/compiler_test.py @@ -129,7 +129,7 @@ def comp(): @dsl.component -def return_1() -> int: +def return_one() -> int: return 1 @@ -3369,43 +3369,43 @@ def test_cpu_memory_optional(self): @dsl.pipeline def simple_pipeline(): - return_1() - return_1().set_cpu_limit('5') - return_1().set_memory_limit('50G') - return_1().set_cpu_request('2').set_cpu_limit( + return_one() + return_one().set_cpu_limit('5') + return_one().set_memory_limit('50G') + return_one().set_cpu_request('2').set_cpu_limit( '5').set_memory_request('4G').set_memory_limit('50G') dict_format = json_format.MessageToDict(simple_pipeline.pipeline_spec) self.assertNotIn( 'resources', dict_format['deploymentSpec']['executors'] - ['exec-return-1']['container']) + ['exec-return-one']['container']) self.assertEqual( - 5, dict_format['deploymentSpec']['executors']['exec-return-1-2'] + 5, dict_format['deploymentSpec']['executors']['exec-return-one-2'] ['container']['resources']['cpuLimit']) self.assertNotIn( 'memoryLimit', dict_format['deploymentSpec']['executors'] - ['exec-return-1-2']['container']['resources']) + ['exec-return-one-2']['container']['resources']) self.assertEqual( - 50, dict_format['deploymentSpec']['executors']['exec-return-1-3'] + 50, dict_format['deploymentSpec']['executors']['exec-return-one-3'] ['container']['resources']['memoryLimit']) self.assertNotIn( 'cpuLimit', dict_format['deploymentSpec']['executors'] - ['exec-return-1-3']['container']['resources']) + ['exec-return-one-3']['container']['resources']) self.assertEqual( - 2, dict_format['deploymentSpec']['executors']['exec-return-1-4'] + 2, dict_format['deploymentSpec']['executors']['exec-return-one-4'] ['container']['resources']['cpuRequest']) self.assertEqual( - 5, dict_format['deploymentSpec']['executors']['exec-return-1-4'] + 5, dict_format['deploymentSpec']['executors']['exec-return-one-4'] ['container']['resources']['cpuLimit']) self.assertEqual( - 4, dict_format['deploymentSpec']['executors']['exec-return-1-4'] + 4, dict_format['deploymentSpec']['executors']['exec-return-one-4'] ['container']['resources']['memoryRequest']) self.assertEqual( - 50, dict_format['deploymentSpec']['executors']['exec-return-1-4'] + 50, dict_format['deploymentSpec']['executors']['exec-return-one-4'] ['container']['resources']['memoryLimit']) diff --git a/sdk/python/kfp/dsl/component_factory.py b/sdk/python/kfp/dsl/component_factory.py index 1af26d80bfc..e6d9656f89a 100644 --- a/sdk/python/kfp/dsl/component_factory.py +++ b/sdk/python/kfp/dsl/component_factory.py @@ -66,6 +66,13 @@ class ComponentInfo(): def _python_function_name_to_component_name(name): name_with_spaces = re.sub(' +', ' ', name.replace('_', ' ')).strip(' ') + name_list = name_with_spaces.split(' ') + + if name_list[-1].isdigit(): + raise ValueError( + f'Invalid function name "{name}". The function name must not end in `_`.' + ) + return name_with_spaces[0].upper() + name_with_spaces[1:] diff --git a/sdk/python/kfp/dsl/component_factory_test.py b/sdk/python/kfp/dsl/component_factory_test.py index b602be241fd..7f8770cbed5 100644 --- a/sdk/python/kfp/dsl/component_factory_test.py +++ b/sdk/python/kfp/dsl/component_factory_test.py @@ -174,6 +174,30 @@ def comp(Output: OutputPath(str), text: str) -> str: pass +class TestPythonFunctionName(unittest.TestCase): + + def test_invalid_function_name(self): + + with self.assertRaisesRegex( + ValueError, + r'Invalid function name "comp_2". The function name must not end in `_`.' + ): + + @component + def comp_2(text: str) -> str: + pass + + def test_valid_function_name(self): + + @component + def comp_v2(text: str) -> str: + pass + + @component + def comp_(text: str) -> str: + pass + + class TestExtractComponentInterfaceListofArtifacts(unittest.TestCase): def test_python_component_input(self): diff --git a/sdk/python/test_data/pipelines/if_elif_else_complex.py b/sdk/python/test_data/pipelines/if_elif_else_complex.py index ea616d9bdf1..c7ebdc804bf 100644 --- a/sdk/python/test_data/pipelines/if_elif_else_complex.py +++ b/sdk/python/test_data/pipelines/if_elif_else_complex.py @@ -18,7 +18,7 @@ @dsl.component -def int_0_to_9999() -> int: +def int_0_to_9999_func() -> int: import random return random.randint(0, 9999) @@ -49,7 +49,7 @@ def lucky_number_pipeline(add_drumroll: bool = True, repeat_if_lucky_number: bool = True, trials: List[int] = [1, 2, 3]): with dsl.ParallelFor(trials) as trial: - int_task = int_0_to_9999().set_caching_options(False) + int_task = int_0_to_9999_func().set_caching_options(False) with dsl.If(add_drumroll == True): with dsl.If(trial == 3): print_and_return(text='Adding drumroll on last trial!') diff --git a/sdk/python/test_data/pipelines/if_elif_else_complex.yaml b/sdk/python/test_data/pipelines/if_elif_else_complex.yaml index 9950a23713c..7e9a1296746 100644 --- a/sdk/python/test_data/pipelines/if_elif_else_complex.yaml +++ b/sdk/python/test_data/pipelines/if_elif_else_complex.yaml @@ -28,7 +28,7 @@ components: name: print-and-return-5 inputDefinitions: parameters: - pipelinechannel--int-0-to-9999-Output: + pipelinechannel--int-0-to-9999-func-Output: parameterType: NUMBER_INTEGER pipelinechannel--is-even-or-odd-2-Output: parameterType: STRING @@ -59,7 +59,7 @@ components: name: print-and-return-6 inputDefinitions: parameters: - pipelinechannel--int-0-to-9999-Output: + pipelinechannel--int-0-to-9999-func-Output: parameterType: NUMBER_INTEGER pipelinechannel--is-even-or-odd-2-Output: parameterType: STRING @@ -75,8 +75,8 @@ components: name: comp-condition-14 inputs: parameters: - pipelinechannel--int-0-to-9999-Output: - componentInputParameter: pipelinechannel--int-0-to-9999-Output + pipelinechannel--int-0-to-9999-func-Output: + componentInputParameter: pipelinechannel--int-0-to-9999-func-Output pipelinechannel--repeat_if_lucky_number: componentInputParameter: pipelinechannel--repeat_if_lucky_number taskInfo: @@ -99,7 +99,7 @@ components: name: print-and-return-8 inputDefinitions: parameters: - pipelinechannel--int-0-to-9999-Output: + pipelinechannel--int-0-to-9999-func-Output: parameterType: NUMBER_INTEGER pipelinechannel--repeat_if_lucky_number: parameterType: BOOLEAN @@ -111,8 +111,8 @@ components: name: comp-for-loop-16 inputs: parameters: - pipelinechannel--int-0-to-9999-Output: - componentInputParameter: pipelinechannel--int-0-to-9999-Output + pipelinechannel--int-0-to-9999-func-Output: + componentInputParameter: pipelinechannel--int-0-to-9999-func-Output pipelinechannel--repeat_if_lucky_number: componentInputParameter: pipelinechannel--repeat_if_lucky_number parameterIterator: @@ -123,7 +123,7 @@ components: name: for-loop-16 inputDefinitions: parameters: - pipelinechannel--int-0-to-9999-Output: + pipelinechannel--int-0-to-9999-func-Output: parameterType: NUMBER_INTEGER pipelinechannel--repeat_if_lucky_number: parameterType: BOOLEAN @@ -194,7 +194,7 @@ components: name: print-and-return-2 inputDefinitions: parameters: - pipelinechannel--int-0-to-9999-Output: + pipelinechannel--int-0-to-9999-func-Output: parameterType: NUMBER_INTEGER pipelinechannel--is-even-or-odd-Output: parameterType: STRING @@ -225,7 +225,7 @@ components: name: print-and-return-3 inputDefinitions: parameters: - pipelinechannel--int-0-to-9999-Output: + pipelinechannel--int-0-to-9999-func-Output: parameterType: NUMBER_INTEGER pipelinechannel--is-even-or-odd-Output: parameterType: STRING @@ -243,8 +243,8 @@ components: - is-even-or-odd inputs: parameters: - pipelinechannel--int-0-to-9999-Output: - componentInputParameter: pipelinechannel--int-0-to-9999-Output + pipelinechannel--int-0-to-9999-func-Output: + componentInputParameter: pipelinechannel--int-0-to-9999-func-Output pipelinechannel--is-even-or-odd-Output: taskOutputParameter: outputParameterKey: Output @@ -259,7 +259,7 @@ components: inputs: parameters: num: - componentInputParameter: pipelinechannel--int-0-to-9999-Output + componentInputParameter: pipelinechannel--int-0-to-9999-func-Output taskInfo: name: is-even-or-odd print-and-return-4: @@ -279,7 +279,7 @@ components: name: print-and-return-4 inputDefinitions: parameters: - pipelinechannel--int-0-to-9999-Output: + pipelinechannel--int-0-to-9999-func-Output: parameterType: NUMBER_INTEGER comp-condition-9: dag: @@ -291,8 +291,8 @@ components: - is-even-or-odd-2 inputs: parameters: - pipelinechannel--int-0-to-9999-Output: - componentInputParameter: pipelinechannel--int-0-to-9999-Output + pipelinechannel--int-0-to-9999-func-Output: + componentInputParameter: pipelinechannel--int-0-to-9999-func-Output pipelinechannel--is-even-or-odd-2-Output: taskOutputParameter: outputParameterKey: Output @@ -307,7 +307,7 @@ components: inputs: parameters: num: - componentInputParameter: pipelinechannel--int-0-to-9999-Output + componentInputParameter: pipelinechannel--int-0-to-9999-func-Output taskInfo: name: is-even-or-odd-2 print-and-return-7: @@ -327,7 +327,7 @@ components: name: print-and-return-7 inputDefinitions: parameters: - pipelinechannel--int-0-to-9999-Output: + pipelinechannel--int-0-to-9999-func-Output: parameterType: NUMBER_INTEGER comp-condition-branches-10: dag: @@ -346,8 +346,8 @@ components: name: comp-condition-11 inputs: parameters: - pipelinechannel--int-0-to-9999-Output: - componentInputParameter: pipelinechannel--int-0-to-9999-Output + pipelinechannel--int-0-to-9999-func-Output: + componentInputParameter: pipelinechannel--int-0-to-9999-func-Output pipelinechannel--is-even-or-odd-2-Output: componentInputParameter: pipelinechannel--is-even-or-odd-2-Output taskInfo: @@ -360,8 +360,8 @@ components: name: comp-condition-12 inputs: parameters: - pipelinechannel--int-0-to-9999-Output: - componentInputParameter: pipelinechannel--int-0-to-9999-Output + pipelinechannel--int-0-to-9999-func-Output: + componentInputParameter: pipelinechannel--int-0-to-9999-func-Output pipelinechannel--is-even-or-odd-2-Output: componentInputParameter: pipelinechannel--is-even-or-odd-2-Output taskInfo: @@ -371,7 +371,7 @@ components: == ''even'')' inputDefinitions: parameters: - pipelinechannel--int-0-to-9999-Output: + pipelinechannel--int-0-to-9999-func-Output: parameterType: NUMBER_INTEGER pipelinechannel--is-even-or-odd-2-Output: parameterType: STRING @@ -387,44 +387,44 @@ components: name: comp-condition-13 inputs: parameters: - pipelinechannel--int-0-to-9999-Output: - componentInputParameter: pipelinechannel--int-0-to-9999-Output + pipelinechannel--int-0-to-9999-func-Output: + componentInputParameter: pipelinechannel--int-0-to-9999-func-Output pipelinechannel--repeat_if_lucky_number: componentInputParameter: pipelinechannel--repeat_if_lucky_number taskInfo: name: condition-13 triggerPolicy: - condition: '!(int(inputs.parameter_values[''pipelinechannel--int-0-to-9999-Output'']) - < 5000) && !(int(inputs.parameter_values[''pipelinechannel--int-0-to-9999-Output'']) + condition: '!(int(inputs.parameter_values[''pipelinechannel--int-0-to-9999-func-Output'']) + < 5000) && !(int(inputs.parameter_values[''pipelinechannel--int-0-to-9999-func-Output'']) > 5000)' condition-8: componentRef: name: comp-condition-8 inputs: parameters: - pipelinechannel--int-0-to-9999-Output: - componentInputParameter: pipelinechannel--int-0-to-9999-Output + pipelinechannel--int-0-to-9999-func-Output: + componentInputParameter: pipelinechannel--int-0-to-9999-func-Output taskInfo: name: condition-8 triggerPolicy: - condition: int(inputs.parameter_values['pipelinechannel--int-0-to-9999-Output']) + condition: int(inputs.parameter_values['pipelinechannel--int-0-to-9999-func-Output']) < 5000 condition-9: componentRef: name: comp-condition-9 inputs: parameters: - pipelinechannel--int-0-to-9999-Output: - componentInputParameter: pipelinechannel--int-0-to-9999-Output + pipelinechannel--int-0-to-9999-func-Output: + componentInputParameter: pipelinechannel--int-0-to-9999-func-Output taskInfo: name: condition-9 triggerPolicy: - condition: '!(int(inputs.parameter_values[''pipelinechannel--int-0-to-9999-Output'']) - < 5000) && int(inputs.parameter_values[''pipelinechannel--int-0-to-9999-Output'']) + condition: '!(int(inputs.parameter_values[''pipelinechannel--int-0-to-9999-func-Output'']) + < 5000) && int(inputs.parameter_values[''pipelinechannel--int-0-to-9999-func-Output'']) > 5000' inputDefinitions: parameters: - pipelinechannel--int-0-to-9999-Output: + pipelinechannel--int-0-to-9999-func-Output: parameterType: NUMBER_INTEGER pipelinechannel--repeat_if_lucky_number: parameterType: BOOLEAN @@ -445,8 +445,8 @@ components: name: comp-condition-6 inputs: parameters: - pipelinechannel--int-0-to-9999-Output: - componentInputParameter: pipelinechannel--int-0-to-9999-Output + pipelinechannel--int-0-to-9999-func-Output: + componentInputParameter: pipelinechannel--int-0-to-9999-func-Output pipelinechannel--is-even-or-odd-Output: componentInputParameter: pipelinechannel--is-even-or-odd-Output taskInfo: @@ -459,8 +459,8 @@ components: name: comp-condition-7 inputs: parameters: - pipelinechannel--int-0-to-9999-Output: - componentInputParameter: pipelinechannel--int-0-to-9999-Output + pipelinechannel--int-0-to-9999-func-Output: + componentInputParameter: pipelinechannel--int-0-to-9999-func-Output pipelinechannel--is-even-or-odd-Output: componentInputParameter: pipelinechannel--is-even-or-odd-Output taskInfo: @@ -470,7 +470,7 @@ components: == ''even'')' inputDefinitions: parameters: - pipelinechannel--int-0-to-9999-Output: + pipelinechannel--int-0-to-9999-func-Output: parameterType: NUMBER_INTEGER pipelinechannel--is-even-or-odd-Output: parameterType: STRING @@ -482,10 +482,10 @@ components: dag: outputs: parameters: - pipelinechannel--int-0-to-9999-Output: + pipelinechannel--int-0-to-9999-func-Output: valueFromParameter: outputParameterKey: Output - producerSubtask: int-0-to-9999 + producerSubtask: int-0-to-9999-func tasks: condition-2: componentRef: @@ -505,23 +505,23 @@ components: componentRef: name: comp-condition-branches-4 dependentTasks: - - int-0-to-9999 + - int-0-to-9999-func inputs: parameters: - pipelinechannel--int-0-to-9999-Output: + pipelinechannel--int-0-to-9999-func-Output: taskOutputParameter: outputParameterKey: Output - producerTask: int-0-to-9999 + producerTask: int-0-to-9999-func pipelinechannel--repeat_if_lucky_number: componentInputParameter: pipelinechannel--repeat_if_lucky_number taskInfo: name: condition-branches-4 - int-0-to-9999: + int-0-to-9999-func: cachingOptions: {} componentRef: - name: comp-int-0-to-9999 + name: comp-int-0-to-9999-func taskInfo: - name: int-0-to-9999 + name: int-0-to-9999-func inputDefinitions: parameters: pipelinechannel--add_drumroll: @@ -534,7 +534,7 @@ components: parameterType: NUMBER_INTEGER outputDefinitions: parameters: - pipelinechannel--int-0-to-9999-Output: + pipelinechannel--int-0-to-9999-func-Output: parameterType: LIST comp-for-loop-16: dag: @@ -554,14 +554,14 @@ components: name: print-and-return-9 inputDefinitions: parameters: - pipelinechannel--int-0-to-9999-Output: + pipelinechannel--int-0-to-9999-func-Output: parameterType: NUMBER_INTEGER pipelinechannel--loop-item-param-15: parameterType: NUMBER_INTEGER pipelinechannel--repeat_if_lucky_number: parameterType: BOOLEAN - comp-int-0-to-9999: - executorLabel: exec-int-0-to-9999 + comp-int-0-to-9999-func: + executorLabel: exec-int-0-to-9999-func outputDefinitions: parameters: Output: @@ -684,19 +684,19 @@ components: parameterType: LIST deploymentSpec: executors: - exec-int-0-to-9999: + exec-int-0-to-9999-func: container: args: - --executor_input - '{{$}}' - --function_to_execute - - int_0_to_9999 + - int_0_to_9999_func command: - sh - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -710,7 +710,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ - \ *\n\ndef int_0_to_9999() -> int:\n import random\n return random.randint(0,\ + \ *\n\ndef int_0_to_9999_func() -> int:\n import random\n return random.randint(0,\ \ 9999)\n\n" image: python:3.8 exec-is-even-or-odd: @@ -725,7 +725,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -754,7 +754,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -783,7 +783,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -812,7 +812,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -841,7 +841,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -870,7 +870,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -899,7 +899,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -928,7 +928,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -957,7 +957,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -986,7 +986,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -1015,7 +1015,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -1044,7 +1044,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -1093,7 +1093,7 @@ root: parameters: ints: taskOutputParameter: - outputParameterKey: pipelinechannel--int-0-to-9999-Output + outputParameterKey: pipelinechannel--int-0-to-9999-func-Output producerTask: for-loop-1 taskInfo: name: print-ints @@ -1115,4 +1115,4 @@ root: isOptional: true parameterType: LIST schemaVersion: 2.1.0 -sdkVersion: kfp-2.7.0 +sdkVersion: kfp-2.8.0 diff --git a/sdk/python/test_data/pipelines/pipeline_with_parallelfor_parallelism.py b/sdk/python/test_data/pipelines/pipeline_with_parallelfor_parallelism.py index f477767dd64..05d50c32041 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_parallelfor_parallelism.py +++ b/sdk/python/test_data/pipelines/pipeline_with_parallelfor_parallelism.py @@ -33,7 +33,7 @@ def print_int(x: int): @component -def list_dict_maker_0() -> List[Dict[str, int]]: +def list_dict_maker_v0() -> List[Dict[str, int]]: """Enforces strict type checking - returns a list of dictionaries where keys are strings and values are integers. For testing type handling during compilation.""" @@ -41,7 +41,7 @@ def list_dict_maker_0() -> List[Dict[str, int]]: @component -def list_dict_maker_1() -> List[Dict]: +def list_dict_maker_v1() -> List[Dict]: """Utilizes generic dictionary typing (no enforcement of specific key or value types). @@ -51,7 +51,7 @@ def list_dict_maker_1() -> List[Dict]: @component -def list_dict_maker_2() -> List[dict]: +def list_dict_maker_v2() -> List[dict]: """Returns a list of dictionaries without type enforcement. Tests flexibility in type handling. @@ -60,7 +60,7 @@ def list_dict_maker_2() -> List[dict]: @component -def list_dict_maker_3() -> List: +def list_dict_maker_v3() -> List: """Returns a basic list (no typing or structure guarantees). Tests the limits of compiler type handling. @@ -71,7 +71,7 @@ def list_dict_maker_3() -> List: with tempfile.TemporaryDirectory() as tmpdir: pipeline_package_path = os.path.join(tmpdir, 'upstream_component.yaml') compiler.Compiler().compile( - pipeline_func=list_dict_maker_1, + pipeline_func=list_dict_maker_v1, package_path=pipeline_package_path, ) @@ -113,19 +113,19 @@ def my_pipeline(loop_parameter: List[str]): print_int(x=item.a) # Loop argument that coming from the upstream component. - t_0 = list_dict_maker_0() + t_0 = list_dict_maker_v0() with dsl.ParallelFor(items=t_0.output) as item: print_int(x=item.a) - t_1 = list_dict_maker_1() + t_1 = list_dict_maker_v1() with dsl.ParallelFor(items=t_1.output) as item: print_int(x=item.a) - t_2 = list_dict_maker_2() + t_2 = list_dict_maker_v2() with dsl.ParallelFor(items=t_2.output) as item: print_int(x=item.a) - t_3 = list_dict_maker_3() + t_3 = list_dict_maker_v3() with dsl.ParallelFor(items=t_3.output) as item: print_int(x=item.a) diff --git a/sdk/python/test_data/pipelines/pipeline_with_parallelfor_parallelism.yaml b/sdk/python/test_data/pipelines/pipeline_with_parallelfor_parallelism.yaml index a33fc357221..ae52688dc01 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_parallelfor_parallelism.yaml +++ b/sdk/python/test_data/pipelines/pipeline_with_parallelfor_parallelism.yaml @@ -47,15 +47,15 @@ components: inputs: parameters: x: - componentInputParameter: pipelinechannel--list-dict-maker-1-Output-loop-item + componentInputParameter: pipelinechannel--list-dict-maker-v1-Output-loop-item parameterExpressionSelector: parseJson(string_value)["a"] taskInfo: name: print-int-3 inputDefinitions: parameters: - pipelinechannel--list-dict-maker-1-Output: + pipelinechannel--list-dict-maker-v1-Output: parameterType: LIST - pipelinechannel--list-dict-maker-1-Output-loop-item: + pipelinechannel--list-dict-maker-v1-Output-loop-item: parameterType: STRUCT comp-for-loop-11: dag: @@ -68,15 +68,15 @@ components: inputs: parameters: x: - componentInputParameter: pipelinechannel--list-dict-maker-2-Output-loop-item + componentInputParameter: pipelinechannel--list-dict-maker-v2-Output-loop-item parameterExpressionSelector: parseJson(string_value)["a"] taskInfo: name: print-int-4 inputDefinitions: parameters: - pipelinechannel--list-dict-maker-2-Output: + pipelinechannel--list-dict-maker-v2-Output: parameterType: LIST - pipelinechannel--list-dict-maker-2-Output-loop-item: + pipelinechannel--list-dict-maker-v2-Output-loop-item: parameterType: STRUCT comp-for-loop-12: dag: @@ -89,15 +89,15 @@ components: inputs: parameters: x: - componentInputParameter: pipelinechannel--list-dict-maker-3-Output-loop-item + componentInputParameter: pipelinechannel--list-dict-maker-v3-Output-loop-item parameterExpressionSelector: parseJson(string_value)["a"] taskInfo: name: print-int-5 inputDefinitions: parameters: - pipelinechannel--list-dict-maker-3-Output: + pipelinechannel--list-dict-maker-v3-Output: parameterType: LIST - pipelinechannel--list-dict-maker-3-Output-loop-item: + pipelinechannel--list-dict-maker-v3-Output-loop-item: parameterType: STRING comp-for-loop-13: dag: @@ -110,15 +110,15 @@ components: inputs: parameters: x: - componentInputParameter: pipelinechannel--list-dict-maker-1-2-Output-loop-item + componentInputParameter: pipelinechannel--list-dict-maker-v1-2-Output-loop-item parameterExpressionSelector: parseJson(string_value)["a"] taskInfo: name: print-int-6 inputDefinitions: parameters: - pipelinechannel--list-dict-maker-1-2-Output: + pipelinechannel--list-dict-maker-v1-2-Output: parameterType: LIST - pipelinechannel--list-dict-maker-1-2-Output-loop-item: + pipelinechannel--list-dict-maker-v1-2-Output-loop-item: parameterType: STRING comp-for-loop-2: dag: @@ -243,42 +243,42 @@ components: inputs: parameters: x: - componentInputParameter: pipelinechannel--list-dict-maker-0-Output-loop-item + componentInputParameter: pipelinechannel--list-dict-maker-v0-Output-loop-item parameterExpressionSelector: parseJson(string_value)["a"] taskInfo: name: print-int-2 inputDefinitions: parameters: - pipelinechannel--list-dict-maker-0-Output: + pipelinechannel--list-dict-maker-v0-Output: parameterType: LIST - pipelinechannel--list-dict-maker-0-Output-loop-item: + pipelinechannel--list-dict-maker-v0-Output-loop-item: parameterType: STRUCT - comp-list-dict-maker-0: - executorLabel: exec-list-dict-maker-0 + comp-list-dict-maker-v0: + executorLabel: exec-list-dict-maker-v0 outputDefinitions: parameters: Output: parameterType: LIST - comp-list-dict-maker-1: - executorLabel: exec-list-dict-maker-1 + comp-list-dict-maker-v1: + executorLabel: exec-list-dict-maker-v1 outputDefinitions: parameters: Output: parameterType: LIST - comp-list-dict-maker-1-2: - executorLabel: exec-list-dict-maker-1-2 + comp-list-dict-maker-v1-2: + executorLabel: exec-list-dict-maker-v1-2 outputDefinitions: parameters: Output: parameterType: LIST - comp-list-dict-maker-2: - executorLabel: exec-list-dict-maker-2 + comp-list-dict-maker-v2: + executorLabel: exec-list-dict-maker-v2 outputDefinitions: parameters: Output: parameterType: LIST - comp-list-dict-maker-3: - executorLabel: exec-list-dict-maker-3 + comp-list-dict-maker-v3: + executorLabel: exec-list-dict-maker-v3 outputDefinitions: parameters: Output: @@ -357,19 +357,19 @@ components: parameterType: STRING deploymentSpec: executors: - exec-list-dict-maker-0: + exec-list-dict-maker-v0: container: args: - --executor_input - '{{$}}' - --function_to_execute - - list_dict_maker_0 + - list_dict_maker_v0 command: - sh - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -383,25 +383,25 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ - \ *\n\ndef list_dict_maker_0() -> List[Dict[str, int]]:\n \"\"\"Enforces\ + \ *\n\ndef list_dict_maker_v0() -> List[Dict[str, int]]:\n \"\"\"Enforces\ \ strict type checking - returns a list of dictionaries \n where keys\ \ are strings and values are integers. For testing type \n handling during\ \ compilation.\"\"\"\n return [{'a': 1, 'b': 2}, {'a': 2, 'b': 3}, {'a':\ \ 3, 'b': 4}]\n\n" image: python:3.8 - exec-list-dict-maker-1: + exec-list-dict-maker-v1: container: args: - --executor_input - '{{$}}' - --function_to_execute - - list_dict_maker_1 + - list_dict_maker_v1 command: - sh - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -415,24 +415,24 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ - \ *\n\ndef list_dict_maker_1() -> List[Dict]:\n \"\"\"Utilizes generic\ + \ *\n\ndef list_dict_maker_v1() -> List[Dict]:\n \"\"\"Utilizes generic\ \ dictionary typing (no enforcement of specific key or\n value types).\n\ \n Tests flexibility in type handling.\n \"\"\"\n return [{'a':\ \ 1, 'b': 2}, {'a': 2, 'b': 3}, {'a': 3, 'b': 4}]\n\n" image: python:3.8 - exec-list-dict-maker-1-2: + exec-list-dict-maker-v1-2: container: args: - --executor_input - '{{$}}' - --function_to_execute - - list_dict_maker_1 + - list_dict_maker_v1 command: - sh - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -446,24 +446,24 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ - \ *\n\ndef list_dict_maker_1() -> List[Dict]:\n \"\"\"Utilizes generic\ + \ *\n\ndef list_dict_maker_v1() -> List[Dict]:\n \"\"\"Utilizes generic\ \ dictionary typing (no enforcement of specific key or\n value types).\n\ \n Tests flexibility in type handling.\n \"\"\"\n return [{'a':\ \ 1, 'b': 2}, {'a': 2, 'b': 3}, {'a': 3, 'b': 4}]\n\n" image: python:3.8 - exec-list-dict-maker-2: + exec-list-dict-maker-v2: container: args: - --executor_input - '{{$}}' - --function_to_execute - - list_dict_maker_2 + - list_dict_maker_v2 command: - sh - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -477,24 +477,24 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ - \ *\n\ndef list_dict_maker_2() -> List[dict]:\n \"\"\"Returns a list\ + \ *\n\ndef list_dict_maker_v2() -> List[dict]:\n \"\"\"Returns a list\ \ of dictionaries without type enforcement.\n\n Tests flexibility in\ \ type handling.\n \"\"\"\n return [{'a': 1, 'b': 2}, {'a': 2, 'b':\ \ 3}, {'a': 3, 'b': 4}]\n\n" image: python:3.8 - exec-list-dict-maker-3: + exec-list-dict-maker-v3: container: args: - --executor_input - '{{$}}' - --function_to_execute - - list_dict_maker_3 + - list_dict_maker_v3 command: - sh - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -508,7 +508,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ - \ *\n\ndef list_dict_maker_3() -> List:\n \"\"\"Returns a basic list\ + \ *\n\ndef list_dict_maker_v3() -> List:\n \"\"\"Returns a basic list\ \ (no typing or structure guarantees).\n\n Tests the limits of compiler\ \ type handling.\n \"\"\"\n return [{'a': 1, 'b': 2}, {'a': 2, 'b':\ \ 3}, {'a': 3, 'b': 4}]\n\n" @@ -525,7 +525,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -553,7 +553,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -581,7 +581,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -609,7 +609,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -637,7 +637,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -665,7 +665,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -693,7 +693,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -721,7 +721,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -749,7 +749,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -777,7 +777,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -805,7 +805,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -833,7 +833,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -873,68 +873,68 @@ root: componentRef: name: comp-for-loop-10 dependentTasks: - - list-dict-maker-1 + - list-dict-maker-v1 inputs: parameters: - pipelinechannel--list-dict-maker-1-Output: + pipelinechannel--list-dict-maker-v1-Output: taskOutputParameter: outputParameterKey: Output - producerTask: list-dict-maker-1 + producerTask: list-dict-maker-v1 parameterIterator: - itemInput: pipelinechannel--list-dict-maker-1-Output-loop-item + itemInput: pipelinechannel--list-dict-maker-v1-Output-loop-item items: - inputParameter: pipelinechannel--list-dict-maker-1-Output + inputParameter: pipelinechannel--list-dict-maker-v1-Output taskInfo: name: for-loop-10 for-loop-11: componentRef: name: comp-for-loop-11 dependentTasks: - - list-dict-maker-2 + - list-dict-maker-v2 inputs: parameters: - pipelinechannel--list-dict-maker-2-Output: + pipelinechannel--list-dict-maker-v2-Output: taskOutputParameter: outputParameterKey: Output - producerTask: list-dict-maker-2 + producerTask: list-dict-maker-v2 parameterIterator: - itemInput: pipelinechannel--list-dict-maker-2-Output-loop-item + itemInput: pipelinechannel--list-dict-maker-v2-Output-loop-item items: - inputParameter: pipelinechannel--list-dict-maker-2-Output + inputParameter: pipelinechannel--list-dict-maker-v2-Output taskInfo: name: for-loop-11 for-loop-12: componentRef: name: comp-for-loop-12 dependentTasks: - - list-dict-maker-3 + - list-dict-maker-v3 inputs: parameters: - pipelinechannel--list-dict-maker-3-Output: + pipelinechannel--list-dict-maker-v3-Output: taskOutputParameter: outputParameterKey: Output - producerTask: list-dict-maker-3 + producerTask: list-dict-maker-v3 parameterIterator: - itemInput: pipelinechannel--list-dict-maker-3-Output-loop-item + itemInput: pipelinechannel--list-dict-maker-v3-Output-loop-item items: - inputParameter: pipelinechannel--list-dict-maker-3-Output + inputParameter: pipelinechannel--list-dict-maker-v3-Output taskInfo: name: for-loop-12 for-loop-13: componentRef: name: comp-for-loop-13 dependentTasks: - - list-dict-maker-1-2 + - list-dict-maker-v1-2 inputs: parameters: - pipelinechannel--list-dict-maker-1-2-Output: + pipelinechannel--list-dict-maker-v1-2-Output: taskOutputParameter: outputParameterKey: Output - producerTask: list-dict-maker-1-2 + producerTask: list-dict-maker-v1-2 parameterIterator: - itemInput: pipelinechannel--list-dict-maker-1-2-Output-loop-item + itemInput: pipelinechannel--list-dict-maker-v1-2-Output-loop-item items: - inputParameter: pipelinechannel--list-dict-maker-1-2-Output + inputParameter: pipelinechannel--list-dict-maker-v1-2-Output taskInfo: name: for-loop-13 for-loop-4: @@ -961,57 +961,57 @@ root: componentRef: name: comp-for-loop-9 dependentTasks: - - list-dict-maker-0 + - list-dict-maker-v0 inputs: parameters: - pipelinechannel--list-dict-maker-0-Output: + pipelinechannel--list-dict-maker-v0-Output: taskOutputParameter: outputParameterKey: Output - producerTask: list-dict-maker-0 + producerTask: list-dict-maker-v0 parameterIterator: - itemInput: pipelinechannel--list-dict-maker-0-Output-loop-item + itemInput: pipelinechannel--list-dict-maker-v0-Output-loop-item items: - inputParameter: pipelinechannel--list-dict-maker-0-Output + inputParameter: pipelinechannel--list-dict-maker-v0-Output taskInfo: name: for-loop-9 - list-dict-maker-0: + list-dict-maker-v0: cachingOptions: enableCache: true componentRef: - name: comp-list-dict-maker-0 + name: comp-list-dict-maker-v0 taskInfo: - name: list-dict-maker-0 - list-dict-maker-1: + name: list-dict-maker-v0 + list-dict-maker-v1: cachingOptions: enableCache: true componentRef: - name: comp-list-dict-maker-1 + name: comp-list-dict-maker-v1 taskInfo: - name: list-dict-maker-1 - list-dict-maker-1-2: + name: list-dict-maker-v1 + list-dict-maker-v1-2: cachingOptions: enableCache: true componentRef: - name: comp-list-dict-maker-1-2 + name: comp-list-dict-maker-v1-2 taskInfo: - name: list-dict-maker-1-2 - list-dict-maker-2: + name: list-dict-maker-v1-2 + list-dict-maker-v2: cachingOptions: enableCache: true componentRef: - name: comp-list-dict-maker-2 + name: comp-list-dict-maker-v2 taskInfo: - name: list-dict-maker-2 - list-dict-maker-3: + name: list-dict-maker-v2 + list-dict-maker-v3: cachingOptions: enableCache: true componentRef: - name: comp-list-dict-maker-3 + name: comp-list-dict-maker-v3 taskInfo: - name: list-dict-maker-3 + name: list-dict-maker-v3 inputDefinitions: parameters: loop_parameter: parameterType: LIST schemaVersion: 2.1.0 -sdkVersion: kfp-2.7.0 +sdkVersion: kfp-2.8.0 From 65ce19f5827c658ee1d36cd8abbb262991384c8e Mon Sep 17 00:00:00 2001 From: droctothorpe Date: Thu, 1 Aug 2024 14:58:24 -0400 Subject: [PATCH 3/7] Use function name instead of base name and address edge cases Signed-off-by: droctothorpe Co-authored-by: zazulam --- backend/src/v2/compiler/argocompiler/argo.go | 57 +++-- .../src/v2/compiler/argocompiler/argo_test.go | 42 ---- sdk/python/kfp/compiler/compiler_test.py | 28 +-- sdk/python/kfp/dsl/component_factory.py | 7 - sdk/python/kfp/dsl/component_factory_test.py | 24 --- .../pipelines/if_elif_else_complex.py | 4 +- .../pipelines/if_elif_else_complex.yaml | 140 ++++++------- .../pipeline_with_parallelfor_parallelism.py | 18 +- ...pipeline_with_parallelfor_parallelism.yaml | 196 +++++++++--------- 9 files changed, 232 insertions(+), 284 deletions(-) diff --git a/backend/src/v2/compiler/argocompiler/argo.go b/backend/src/v2/compiler/argocompiler/argo.go index b6899f9b6c6..cfa4477d0c7 100644 --- a/backend/src/v2/compiler/argocompiler/argo.go +++ b/backend/src/v2/compiler/argocompiler/argo.go @@ -16,12 +16,12 @@ package argocompiler import ( "fmt" - "strconv" "strings" wfapi "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" "github.com/kubeflow/pipelines/backend/src/v2/compiler" + log "github.com/sirupsen/logrus" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/structpb" k8score "k8s.io/api/core/v1" @@ -191,25 +191,29 @@ const ( ) func (c *workflowCompiler) saveComponentSpec(name string, spec *pipelinespec.ComponentSpec) error { - baseComponentName := ExtractBaseComponentName(argumentsComponents + name) - return c.saveProtoToArguments(baseComponentName, spec) + functionName := c.extractFunctionName(name) + + return c.saveProtoToArguments(argumentsComponents+functionName, spec) } // useComponentSpec returns a placeholder we can refer to the component spec // in argo workflow fields. func (c *workflowCompiler) useComponentSpec(name string) (string, error) { - baseComponentName := ExtractBaseComponentName(argumentsComponents + name) - return c.argumentsPlaceholder(baseComponentName) + functionName := c.extractFunctionName(name) + + return c.argumentsPlaceholder(argumentsComponents + functionName) } func (c *workflowCompiler) saveComponentImpl(name string, msg proto.Message) error { - baseComponentName := ExtractBaseComponentName(argumentsContainers + name) - return c.saveProtoToArguments(baseComponentName, msg) + functionName := c.extractFunctionName(name) + + return c.saveProtoToArguments(argumentsContainers+functionName, msg) } func (c *workflowCompiler) useComponentImpl(name string) (string, error) { - baseComponentName := ExtractBaseComponentName(argumentsContainers + name) - return c.argumentsPlaceholder(baseComponentName) + functionName := c.extractFunctionName(name) + + return c.argumentsPlaceholder(argumentsContainers + functionName) } func (c *workflowCompiler) saveKubernetesSpec(name string, spec *structpb.Struct) error { @@ -262,17 +266,34 @@ func (c *workflowCompiler) argumentsPlaceholder(componentName string) (string, e return workflowParameter(componentName), nil } -// ExtractBaseComponentName removes the iteration suffix that the IR compiler -// adds to the component name. -func ExtractBaseComponentName(componentName string) string { - baseComponentName := componentName - componentNameArray := strings.Split(componentName, "-") - - if _, err := strconv.Atoi(componentNameArray[len(componentNameArray)-1]); err == nil { - baseComponentName = strings.Join(componentNameArray[:len(componentNameArray)-1], "-") +// extractFunctionName extracts the function name of a component by looking it +// up in the pipeline spec. +func (c *workflowCompiler) extractFunctionName(componentName string) string { + log.Debug("componentName: ", componentName) + // The root component is a DAG and therefore doesn't have a corresponding + // executor or function name. The final return statement in this function + // would cover this edge case, but this saves us some unecessary iteration. + if componentName == "root" { + return componentName + } + executorLabel := c.spec.Components[componentName].GetExecutorLabel() + log.Debug("executorLabel: ", executorLabel) + for executorName, executorValue := range c.executors { + log.Debug("executorName: ", executorName) + if executorName == executorLabel { + args := executorValue.GetContainer().Args + componentFunctionName := args[len(args)-1] + log.Debug("componentFunctionName: ", componentFunctionName) + + return componentFunctionName + } } - return baseComponentName + log.Debug("No corresponding executor for component: ", componentName) + // We could theoretically return an error here, but since the only + // consequence of not finding a matching executor is reduced deduplication, + // this doesn't result in application failure and we therefore continue. + return componentName } const ( diff --git a/backend/src/v2/compiler/argocompiler/argo_test.go b/backend/src/v2/compiler/argocompiler/argo_test.go index 3f6ccb88afe..6c92e54574c 100644 --- a/backend/src/v2/compiler/argocompiler/argo_test.go +++ b/backend/src/v2/compiler/argocompiler/argo_test.go @@ -137,45 +137,3 @@ func load(t *testing.T, path string, platformSpecPath string) (*pipelinespec.Pip } return job, nil } - -func Test_extractBaseComponentName(t *testing.T) { - tests := []struct { - name string - componentName string - expectedBaseName string - }{ - { - name: "With dash and int", - componentName: "component-2", - expectedBaseName: "component", - }, - { - name: "Without dash and int", - componentName: "component", - expectedBaseName: "component", - }, - { - name: "Last char is int", - componentName: "component-v2", - expectedBaseName: "component-v2", - }, - { - name: "Multiple dashes, ends with int", - componentName: "service-api-v2", - expectedBaseName: "service-api-v2", - }, - { - name: "Multiple dashes and ints", - componentName: "module-1-2-3", - expectedBaseName: "module-1-2", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := argocompiler.ExtractBaseComponentName(tt.componentName) - if result != tt.expectedBaseName { - t.Errorf("Expected: %s, Got: %s", tt.expectedBaseName, result) - } - }) - } -} diff --git a/sdk/python/kfp/compiler/compiler_test.py b/sdk/python/kfp/compiler/compiler_test.py index 3051dca17fd..d417d9eec19 100644 --- a/sdk/python/kfp/compiler/compiler_test.py +++ b/sdk/python/kfp/compiler/compiler_test.py @@ -129,7 +129,7 @@ def comp(): @dsl.component -def return_one() -> int: +def return_1() -> int: return 1 @@ -3369,43 +3369,43 @@ def test_cpu_memory_optional(self): @dsl.pipeline def simple_pipeline(): - return_one() - return_one().set_cpu_limit('5') - return_one().set_memory_limit('50G') - return_one().set_cpu_request('2').set_cpu_limit( + return_1() + return_1().set_cpu_limit('5') + return_1().set_memory_limit('50G') + return_1().set_cpu_request('2').set_cpu_limit( '5').set_memory_request('4G').set_memory_limit('50G') dict_format = json_format.MessageToDict(simple_pipeline.pipeline_spec) self.assertNotIn( 'resources', dict_format['deploymentSpec']['executors'] - ['exec-return-one']['container']) + ['exec-return-1']['container']) self.assertEqual( - 5, dict_format['deploymentSpec']['executors']['exec-return-one-2'] + 5, dict_format['deploymentSpec']['executors']['exec-return-1-2'] ['container']['resources']['cpuLimit']) self.assertNotIn( 'memoryLimit', dict_format['deploymentSpec']['executors'] - ['exec-return-one-2']['container']['resources']) + ['exec-return-1-2']['container']['resources']) self.assertEqual( - 50, dict_format['deploymentSpec']['executors']['exec-return-one-3'] + 50, dict_format['deploymentSpec']['executors']['exec-return-1-3'] ['container']['resources']['memoryLimit']) self.assertNotIn( 'cpuLimit', dict_format['deploymentSpec']['executors'] - ['exec-return-one-3']['container']['resources']) + ['exec-return-1-3']['container']['resources']) self.assertEqual( - 2, dict_format['deploymentSpec']['executors']['exec-return-one-4'] + 2, dict_format['deploymentSpec']['executors']['exec-return-1-4'] ['container']['resources']['cpuRequest']) self.assertEqual( - 5, dict_format['deploymentSpec']['executors']['exec-return-one-4'] + 5, dict_format['deploymentSpec']['executors']['exec-return-1-4'] ['container']['resources']['cpuLimit']) self.assertEqual( - 4, dict_format['deploymentSpec']['executors']['exec-return-one-4'] + 4, dict_format['deploymentSpec']['executors']['exec-return-1-4'] ['container']['resources']['memoryRequest']) self.assertEqual( - 50, dict_format['deploymentSpec']['executors']['exec-return-one-4'] + 50, dict_format['deploymentSpec']['executors']['exec-return-1-4'] ['container']['resources']['memoryLimit']) diff --git a/sdk/python/kfp/dsl/component_factory.py b/sdk/python/kfp/dsl/component_factory.py index e6d9656f89a..1af26d80bfc 100644 --- a/sdk/python/kfp/dsl/component_factory.py +++ b/sdk/python/kfp/dsl/component_factory.py @@ -66,13 +66,6 @@ class ComponentInfo(): def _python_function_name_to_component_name(name): name_with_spaces = re.sub(' +', ' ', name.replace('_', ' ')).strip(' ') - name_list = name_with_spaces.split(' ') - - if name_list[-1].isdigit(): - raise ValueError( - f'Invalid function name "{name}". The function name must not end in `_`.' - ) - return name_with_spaces[0].upper() + name_with_spaces[1:] diff --git a/sdk/python/kfp/dsl/component_factory_test.py b/sdk/python/kfp/dsl/component_factory_test.py index 7f8770cbed5..b602be241fd 100644 --- a/sdk/python/kfp/dsl/component_factory_test.py +++ b/sdk/python/kfp/dsl/component_factory_test.py @@ -174,30 +174,6 @@ def comp(Output: OutputPath(str), text: str) -> str: pass -class TestPythonFunctionName(unittest.TestCase): - - def test_invalid_function_name(self): - - with self.assertRaisesRegex( - ValueError, - r'Invalid function name "comp_2". The function name must not end in `_`.' - ): - - @component - def comp_2(text: str) -> str: - pass - - def test_valid_function_name(self): - - @component - def comp_v2(text: str) -> str: - pass - - @component - def comp_(text: str) -> str: - pass - - class TestExtractComponentInterfaceListofArtifacts(unittest.TestCase): def test_python_component_input(self): diff --git a/sdk/python/test_data/pipelines/if_elif_else_complex.py b/sdk/python/test_data/pipelines/if_elif_else_complex.py index c7ebdc804bf..ea616d9bdf1 100644 --- a/sdk/python/test_data/pipelines/if_elif_else_complex.py +++ b/sdk/python/test_data/pipelines/if_elif_else_complex.py @@ -18,7 +18,7 @@ @dsl.component -def int_0_to_9999_func() -> int: +def int_0_to_9999() -> int: import random return random.randint(0, 9999) @@ -49,7 +49,7 @@ def lucky_number_pipeline(add_drumroll: bool = True, repeat_if_lucky_number: bool = True, trials: List[int] = [1, 2, 3]): with dsl.ParallelFor(trials) as trial: - int_task = int_0_to_9999_func().set_caching_options(False) + int_task = int_0_to_9999().set_caching_options(False) with dsl.If(add_drumroll == True): with dsl.If(trial == 3): print_and_return(text='Adding drumroll on last trial!') diff --git a/sdk/python/test_data/pipelines/if_elif_else_complex.yaml b/sdk/python/test_data/pipelines/if_elif_else_complex.yaml index 7e9a1296746..9950a23713c 100644 --- a/sdk/python/test_data/pipelines/if_elif_else_complex.yaml +++ b/sdk/python/test_data/pipelines/if_elif_else_complex.yaml @@ -28,7 +28,7 @@ components: name: print-and-return-5 inputDefinitions: parameters: - pipelinechannel--int-0-to-9999-func-Output: + pipelinechannel--int-0-to-9999-Output: parameterType: NUMBER_INTEGER pipelinechannel--is-even-or-odd-2-Output: parameterType: STRING @@ -59,7 +59,7 @@ components: name: print-and-return-6 inputDefinitions: parameters: - pipelinechannel--int-0-to-9999-func-Output: + pipelinechannel--int-0-to-9999-Output: parameterType: NUMBER_INTEGER pipelinechannel--is-even-or-odd-2-Output: parameterType: STRING @@ -75,8 +75,8 @@ components: name: comp-condition-14 inputs: parameters: - pipelinechannel--int-0-to-9999-func-Output: - componentInputParameter: pipelinechannel--int-0-to-9999-func-Output + pipelinechannel--int-0-to-9999-Output: + componentInputParameter: pipelinechannel--int-0-to-9999-Output pipelinechannel--repeat_if_lucky_number: componentInputParameter: pipelinechannel--repeat_if_lucky_number taskInfo: @@ -99,7 +99,7 @@ components: name: print-and-return-8 inputDefinitions: parameters: - pipelinechannel--int-0-to-9999-func-Output: + pipelinechannel--int-0-to-9999-Output: parameterType: NUMBER_INTEGER pipelinechannel--repeat_if_lucky_number: parameterType: BOOLEAN @@ -111,8 +111,8 @@ components: name: comp-for-loop-16 inputs: parameters: - pipelinechannel--int-0-to-9999-func-Output: - componentInputParameter: pipelinechannel--int-0-to-9999-func-Output + pipelinechannel--int-0-to-9999-Output: + componentInputParameter: pipelinechannel--int-0-to-9999-Output pipelinechannel--repeat_if_lucky_number: componentInputParameter: pipelinechannel--repeat_if_lucky_number parameterIterator: @@ -123,7 +123,7 @@ components: name: for-loop-16 inputDefinitions: parameters: - pipelinechannel--int-0-to-9999-func-Output: + pipelinechannel--int-0-to-9999-Output: parameterType: NUMBER_INTEGER pipelinechannel--repeat_if_lucky_number: parameterType: BOOLEAN @@ -194,7 +194,7 @@ components: name: print-and-return-2 inputDefinitions: parameters: - pipelinechannel--int-0-to-9999-func-Output: + pipelinechannel--int-0-to-9999-Output: parameterType: NUMBER_INTEGER pipelinechannel--is-even-or-odd-Output: parameterType: STRING @@ -225,7 +225,7 @@ components: name: print-and-return-3 inputDefinitions: parameters: - pipelinechannel--int-0-to-9999-func-Output: + pipelinechannel--int-0-to-9999-Output: parameterType: NUMBER_INTEGER pipelinechannel--is-even-or-odd-Output: parameterType: STRING @@ -243,8 +243,8 @@ components: - is-even-or-odd inputs: parameters: - pipelinechannel--int-0-to-9999-func-Output: - componentInputParameter: pipelinechannel--int-0-to-9999-func-Output + pipelinechannel--int-0-to-9999-Output: + componentInputParameter: pipelinechannel--int-0-to-9999-Output pipelinechannel--is-even-or-odd-Output: taskOutputParameter: outputParameterKey: Output @@ -259,7 +259,7 @@ components: inputs: parameters: num: - componentInputParameter: pipelinechannel--int-0-to-9999-func-Output + componentInputParameter: pipelinechannel--int-0-to-9999-Output taskInfo: name: is-even-or-odd print-and-return-4: @@ -279,7 +279,7 @@ components: name: print-and-return-4 inputDefinitions: parameters: - pipelinechannel--int-0-to-9999-func-Output: + pipelinechannel--int-0-to-9999-Output: parameterType: NUMBER_INTEGER comp-condition-9: dag: @@ -291,8 +291,8 @@ components: - is-even-or-odd-2 inputs: parameters: - pipelinechannel--int-0-to-9999-func-Output: - componentInputParameter: pipelinechannel--int-0-to-9999-func-Output + pipelinechannel--int-0-to-9999-Output: + componentInputParameter: pipelinechannel--int-0-to-9999-Output pipelinechannel--is-even-or-odd-2-Output: taskOutputParameter: outputParameterKey: Output @@ -307,7 +307,7 @@ components: inputs: parameters: num: - componentInputParameter: pipelinechannel--int-0-to-9999-func-Output + componentInputParameter: pipelinechannel--int-0-to-9999-Output taskInfo: name: is-even-or-odd-2 print-and-return-7: @@ -327,7 +327,7 @@ components: name: print-and-return-7 inputDefinitions: parameters: - pipelinechannel--int-0-to-9999-func-Output: + pipelinechannel--int-0-to-9999-Output: parameterType: NUMBER_INTEGER comp-condition-branches-10: dag: @@ -346,8 +346,8 @@ components: name: comp-condition-11 inputs: parameters: - pipelinechannel--int-0-to-9999-func-Output: - componentInputParameter: pipelinechannel--int-0-to-9999-func-Output + pipelinechannel--int-0-to-9999-Output: + componentInputParameter: pipelinechannel--int-0-to-9999-Output pipelinechannel--is-even-or-odd-2-Output: componentInputParameter: pipelinechannel--is-even-or-odd-2-Output taskInfo: @@ -360,8 +360,8 @@ components: name: comp-condition-12 inputs: parameters: - pipelinechannel--int-0-to-9999-func-Output: - componentInputParameter: pipelinechannel--int-0-to-9999-func-Output + pipelinechannel--int-0-to-9999-Output: + componentInputParameter: pipelinechannel--int-0-to-9999-Output pipelinechannel--is-even-or-odd-2-Output: componentInputParameter: pipelinechannel--is-even-or-odd-2-Output taskInfo: @@ -371,7 +371,7 @@ components: == ''even'')' inputDefinitions: parameters: - pipelinechannel--int-0-to-9999-func-Output: + pipelinechannel--int-0-to-9999-Output: parameterType: NUMBER_INTEGER pipelinechannel--is-even-or-odd-2-Output: parameterType: STRING @@ -387,44 +387,44 @@ components: name: comp-condition-13 inputs: parameters: - pipelinechannel--int-0-to-9999-func-Output: - componentInputParameter: pipelinechannel--int-0-to-9999-func-Output + pipelinechannel--int-0-to-9999-Output: + componentInputParameter: pipelinechannel--int-0-to-9999-Output pipelinechannel--repeat_if_lucky_number: componentInputParameter: pipelinechannel--repeat_if_lucky_number taskInfo: name: condition-13 triggerPolicy: - condition: '!(int(inputs.parameter_values[''pipelinechannel--int-0-to-9999-func-Output'']) - < 5000) && !(int(inputs.parameter_values[''pipelinechannel--int-0-to-9999-func-Output'']) + condition: '!(int(inputs.parameter_values[''pipelinechannel--int-0-to-9999-Output'']) + < 5000) && !(int(inputs.parameter_values[''pipelinechannel--int-0-to-9999-Output'']) > 5000)' condition-8: componentRef: name: comp-condition-8 inputs: parameters: - pipelinechannel--int-0-to-9999-func-Output: - componentInputParameter: pipelinechannel--int-0-to-9999-func-Output + pipelinechannel--int-0-to-9999-Output: + componentInputParameter: pipelinechannel--int-0-to-9999-Output taskInfo: name: condition-8 triggerPolicy: - condition: int(inputs.parameter_values['pipelinechannel--int-0-to-9999-func-Output']) + condition: int(inputs.parameter_values['pipelinechannel--int-0-to-9999-Output']) < 5000 condition-9: componentRef: name: comp-condition-9 inputs: parameters: - pipelinechannel--int-0-to-9999-func-Output: - componentInputParameter: pipelinechannel--int-0-to-9999-func-Output + pipelinechannel--int-0-to-9999-Output: + componentInputParameter: pipelinechannel--int-0-to-9999-Output taskInfo: name: condition-9 triggerPolicy: - condition: '!(int(inputs.parameter_values[''pipelinechannel--int-0-to-9999-func-Output'']) - < 5000) && int(inputs.parameter_values[''pipelinechannel--int-0-to-9999-func-Output'']) + condition: '!(int(inputs.parameter_values[''pipelinechannel--int-0-to-9999-Output'']) + < 5000) && int(inputs.parameter_values[''pipelinechannel--int-0-to-9999-Output'']) > 5000' inputDefinitions: parameters: - pipelinechannel--int-0-to-9999-func-Output: + pipelinechannel--int-0-to-9999-Output: parameterType: NUMBER_INTEGER pipelinechannel--repeat_if_lucky_number: parameterType: BOOLEAN @@ -445,8 +445,8 @@ components: name: comp-condition-6 inputs: parameters: - pipelinechannel--int-0-to-9999-func-Output: - componentInputParameter: pipelinechannel--int-0-to-9999-func-Output + pipelinechannel--int-0-to-9999-Output: + componentInputParameter: pipelinechannel--int-0-to-9999-Output pipelinechannel--is-even-or-odd-Output: componentInputParameter: pipelinechannel--is-even-or-odd-Output taskInfo: @@ -459,8 +459,8 @@ components: name: comp-condition-7 inputs: parameters: - pipelinechannel--int-0-to-9999-func-Output: - componentInputParameter: pipelinechannel--int-0-to-9999-func-Output + pipelinechannel--int-0-to-9999-Output: + componentInputParameter: pipelinechannel--int-0-to-9999-Output pipelinechannel--is-even-or-odd-Output: componentInputParameter: pipelinechannel--is-even-or-odd-Output taskInfo: @@ -470,7 +470,7 @@ components: == ''even'')' inputDefinitions: parameters: - pipelinechannel--int-0-to-9999-func-Output: + pipelinechannel--int-0-to-9999-Output: parameterType: NUMBER_INTEGER pipelinechannel--is-even-or-odd-Output: parameterType: STRING @@ -482,10 +482,10 @@ components: dag: outputs: parameters: - pipelinechannel--int-0-to-9999-func-Output: + pipelinechannel--int-0-to-9999-Output: valueFromParameter: outputParameterKey: Output - producerSubtask: int-0-to-9999-func + producerSubtask: int-0-to-9999 tasks: condition-2: componentRef: @@ -505,23 +505,23 @@ components: componentRef: name: comp-condition-branches-4 dependentTasks: - - int-0-to-9999-func + - int-0-to-9999 inputs: parameters: - pipelinechannel--int-0-to-9999-func-Output: + pipelinechannel--int-0-to-9999-Output: taskOutputParameter: outputParameterKey: Output - producerTask: int-0-to-9999-func + producerTask: int-0-to-9999 pipelinechannel--repeat_if_lucky_number: componentInputParameter: pipelinechannel--repeat_if_lucky_number taskInfo: name: condition-branches-4 - int-0-to-9999-func: + int-0-to-9999: cachingOptions: {} componentRef: - name: comp-int-0-to-9999-func + name: comp-int-0-to-9999 taskInfo: - name: int-0-to-9999-func + name: int-0-to-9999 inputDefinitions: parameters: pipelinechannel--add_drumroll: @@ -534,7 +534,7 @@ components: parameterType: NUMBER_INTEGER outputDefinitions: parameters: - pipelinechannel--int-0-to-9999-func-Output: + pipelinechannel--int-0-to-9999-Output: parameterType: LIST comp-for-loop-16: dag: @@ -554,14 +554,14 @@ components: name: print-and-return-9 inputDefinitions: parameters: - pipelinechannel--int-0-to-9999-func-Output: + pipelinechannel--int-0-to-9999-Output: parameterType: NUMBER_INTEGER pipelinechannel--loop-item-param-15: parameterType: NUMBER_INTEGER pipelinechannel--repeat_if_lucky_number: parameterType: BOOLEAN - comp-int-0-to-9999-func: - executorLabel: exec-int-0-to-9999-func + comp-int-0-to-9999: + executorLabel: exec-int-0-to-9999 outputDefinitions: parameters: Output: @@ -684,19 +684,19 @@ components: parameterType: LIST deploymentSpec: executors: - exec-int-0-to-9999-func: + exec-int-0-to-9999: container: args: - --executor_input - '{{$}}' - --function_to_execute - - int_0_to_9999_func + - int_0_to_9999 command: - sh - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -710,7 +710,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ - \ *\n\ndef int_0_to_9999_func() -> int:\n import random\n return random.randint(0,\ + \ *\n\ndef int_0_to_9999() -> int:\n import random\n return random.randint(0,\ \ 9999)\n\n" image: python:3.8 exec-is-even-or-odd: @@ -725,7 +725,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -754,7 +754,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -783,7 +783,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -812,7 +812,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -841,7 +841,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -870,7 +870,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -899,7 +899,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -928,7 +928,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -957,7 +957,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -986,7 +986,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -1015,7 +1015,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -1044,7 +1044,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -1093,7 +1093,7 @@ root: parameters: ints: taskOutputParameter: - outputParameterKey: pipelinechannel--int-0-to-9999-func-Output + outputParameterKey: pipelinechannel--int-0-to-9999-Output producerTask: for-loop-1 taskInfo: name: print-ints @@ -1115,4 +1115,4 @@ root: isOptional: true parameterType: LIST schemaVersion: 2.1.0 -sdkVersion: kfp-2.8.0 +sdkVersion: kfp-2.7.0 diff --git a/sdk/python/test_data/pipelines/pipeline_with_parallelfor_parallelism.py b/sdk/python/test_data/pipelines/pipeline_with_parallelfor_parallelism.py index 05d50c32041..f477767dd64 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_parallelfor_parallelism.py +++ b/sdk/python/test_data/pipelines/pipeline_with_parallelfor_parallelism.py @@ -33,7 +33,7 @@ def print_int(x: int): @component -def list_dict_maker_v0() -> List[Dict[str, int]]: +def list_dict_maker_0() -> List[Dict[str, int]]: """Enforces strict type checking - returns a list of dictionaries where keys are strings and values are integers. For testing type handling during compilation.""" @@ -41,7 +41,7 @@ def list_dict_maker_v0() -> List[Dict[str, int]]: @component -def list_dict_maker_v1() -> List[Dict]: +def list_dict_maker_1() -> List[Dict]: """Utilizes generic dictionary typing (no enforcement of specific key or value types). @@ -51,7 +51,7 @@ def list_dict_maker_v1() -> List[Dict]: @component -def list_dict_maker_v2() -> List[dict]: +def list_dict_maker_2() -> List[dict]: """Returns a list of dictionaries without type enforcement. Tests flexibility in type handling. @@ -60,7 +60,7 @@ def list_dict_maker_v2() -> List[dict]: @component -def list_dict_maker_v3() -> List: +def list_dict_maker_3() -> List: """Returns a basic list (no typing or structure guarantees). Tests the limits of compiler type handling. @@ -71,7 +71,7 @@ def list_dict_maker_v3() -> List: with tempfile.TemporaryDirectory() as tmpdir: pipeline_package_path = os.path.join(tmpdir, 'upstream_component.yaml') compiler.Compiler().compile( - pipeline_func=list_dict_maker_v1, + pipeline_func=list_dict_maker_1, package_path=pipeline_package_path, ) @@ -113,19 +113,19 @@ def my_pipeline(loop_parameter: List[str]): print_int(x=item.a) # Loop argument that coming from the upstream component. - t_0 = list_dict_maker_v0() + t_0 = list_dict_maker_0() with dsl.ParallelFor(items=t_0.output) as item: print_int(x=item.a) - t_1 = list_dict_maker_v1() + t_1 = list_dict_maker_1() with dsl.ParallelFor(items=t_1.output) as item: print_int(x=item.a) - t_2 = list_dict_maker_v2() + t_2 = list_dict_maker_2() with dsl.ParallelFor(items=t_2.output) as item: print_int(x=item.a) - t_3 = list_dict_maker_v3() + t_3 = list_dict_maker_3() with dsl.ParallelFor(items=t_3.output) as item: print_int(x=item.a) diff --git a/sdk/python/test_data/pipelines/pipeline_with_parallelfor_parallelism.yaml b/sdk/python/test_data/pipelines/pipeline_with_parallelfor_parallelism.yaml index ae52688dc01..a33fc357221 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_parallelfor_parallelism.yaml +++ b/sdk/python/test_data/pipelines/pipeline_with_parallelfor_parallelism.yaml @@ -47,15 +47,15 @@ components: inputs: parameters: x: - componentInputParameter: pipelinechannel--list-dict-maker-v1-Output-loop-item + componentInputParameter: pipelinechannel--list-dict-maker-1-Output-loop-item parameterExpressionSelector: parseJson(string_value)["a"] taskInfo: name: print-int-3 inputDefinitions: parameters: - pipelinechannel--list-dict-maker-v1-Output: + pipelinechannel--list-dict-maker-1-Output: parameterType: LIST - pipelinechannel--list-dict-maker-v1-Output-loop-item: + pipelinechannel--list-dict-maker-1-Output-loop-item: parameterType: STRUCT comp-for-loop-11: dag: @@ -68,15 +68,15 @@ components: inputs: parameters: x: - componentInputParameter: pipelinechannel--list-dict-maker-v2-Output-loop-item + componentInputParameter: pipelinechannel--list-dict-maker-2-Output-loop-item parameterExpressionSelector: parseJson(string_value)["a"] taskInfo: name: print-int-4 inputDefinitions: parameters: - pipelinechannel--list-dict-maker-v2-Output: + pipelinechannel--list-dict-maker-2-Output: parameterType: LIST - pipelinechannel--list-dict-maker-v2-Output-loop-item: + pipelinechannel--list-dict-maker-2-Output-loop-item: parameterType: STRUCT comp-for-loop-12: dag: @@ -89,15 +89,15 @@ components: inputs: parameters: x: - componentInputParameter: pipelinechannel--list-dict-maker-v3-Output-loop-item + componentInputParameter: pipelinechannel--list-dict-maker-3-Output-loop-item parameterExpressionSelector: parseJson(string_value)["a"] taskInfo: name: print-int-5 inputDefinitions: parameters: - pipelinechannel--list-dict-maker-v3-Output: + pipelinechannel--list-dict-maker-3-Output: parameterType: LIST - pipelinechannel--list-dict-maker-v3-Output-loop-item: + pipelinechannel--list-dict-maker-3-Output-loop-item: parameterType: STRING comp-for-loop-13: dag: @@ -110,15 +110,15 @@ components: inputs: parameters: x: - componentInputParameter: pipelinechannel--list-dict-maker-v1-2-Output-loop-item + componentInputParameter: pipelinechannel--list-dict-maker-1-2-Output-loop-item parameterExpressionSelector: parseJson(string_value)["a"] taskInfo: name: print-int-6 inputDefinitions: parameters: - pipelinechannel--list-dict-maker-v1-2-Output: + pipelinechannel--list-dict-maker-1-2-Output: parameterType: LIST - pipelinechannel--list-dict-maker-v1-2-Output-loop-item: + pipelinechannel--list-dict-maker-1-2-Output-loop-item: parameterType: STRING comp-for-loop-2: dag: @@ -243,42 +243,42 @@ components: inputs: parameters: x: - componentInputParameter: pipelinechannel--list-dict-maker-v0-Output-loop-item + componentInputParameter: pipelinechannel--list-dict-maker-0-Output-loop-item parameterExpressionSelector: parseJson(string_value)["a"] taskInfo: name: print-int-2 inputDefinitions: parameters: - pipelinechannel--list-dict-maker-v0-Output: + pipelinechannel--list-dict-maker-0-Output: parameterType: LIST - pipelinechannel--list-dict-maker-v0-Output-loop-item: + pipelinechannel--list-dict-maker-0-Output-loop-item: parameterType: STRUCT - comp-list-dict-maker-v0: - executorLabel: exec-list-dict-maker-v0 + comp-list-dict-maker-0: + executorLabel: exec-list-dict-maker-0 outputDefinitions: parameters: Output: parameterType: LIST - comp-list-dict-maker-v1: - executorLabel: exec-list-dict-maker-v1 + comp-list-dict-maker-1: + executorLabel: exec-list-dict-maker-1 outputDefinitions: parameters: Output: parameterType: LIST - comp-list-dict-maker-v1-2: - executorLabel: exec-list-dict-maker-v1-2 + comp-list-dict-maker-1-2: + executorLabel: exec-list-dict-maker-1-2 outputDefinitions: parameters: Output: parameterType: LIST - comp-list-dict-maker-v2: - executorLabel: exec-list-dict-maker-v2 + comp-list-dict-maker-2: + executorLabel: exec-list-dict-maker-2 outputDefinitions: parameters: Output: parameterType: LIST - comp-list-dict-maker-v3: - executorLabel: exec-list-dict-maker-v3 + comp-list-dict-maker-3: + executorLabel: exec-list-dict-maker-3 outputDefinitions: parameters: Output: @@ -357,19 +357,19 @@ components: parameterType: STRING deploymentSpec: executors: - exec-list-dict-maker-v0: + exec-list-dict-maker-0: container: args: - --executor_input - '{{$}}' - --function_to_execute - - list_dict_maker_v0 + - list_dict_maker_0 command: - sh - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -383,25 +383,25 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ - \ *\n\ndef list_dict_maker_v0() -> List[Dict[str, int]]:\n \"\"\"Enforces\ + \ *\n\ndef list_dict_maker_0() -> List[Dict[str, int]]:\n \"\"\"Enforces\ \ strict type checking - returns a list of dictionaries \n where keys\ \ are strings and values are integers. For testing type \n handling during\ \ compilation.\"\"\"\n return [{'a': 1, 'b': 2}, {'a': 2, 'b': 3}, {'a':\ \ 3, 'b': 4}]\n\n" image: python:3.8 - exec-list-dict-maker-v1: + exec-list-dict-maker-1: container: args: - --executor_input - '{{$}}' - --function_to_execute - - list_dict_maker_v1 + - list_dict_maker_1 command: - sh - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -415,24 +415,24 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ - \ *\n\ndef list_dict_maker_v1() -> List[Dict]:\n \"\"\"Utilizes generic\ + \ *\n\ndef list_dict_maker_1() -> List[Dict]:\n \"\"\"Utilizes generic\ \ dictionary typing (no enforcement of specific key or\n value types).\n\ \n Tests flexibility in type handling.\n \"\"\"\n return [{'a':\ \ 1, 'b': 2}, {'a': 2, 'b': 3}, {'a': 3, 'b': 4}]\n\n" image: python:3.8 - exec-list-dict-maker-v1-2: + exec-list-dict-maker-1-2: container: args: - --executor_input - '{{$}}' - --function_to_execute - - list_dict_maker_v1 + - list_dict_maker_1 command: - sh - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -446,24 +446,24 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ - \ *\n\ndef list_dict_maker_v1() -> List[Dict]:\n \"\"\"Utilizes generic\ + \ *\n\ndef list_dict_maker_1() -> List[Dict]:\n \"\"\"Utilizes generic\ \ dictionary typing (no enforcement of specific key or\n value types).\n\ \n Tests flexibility in type handling.\n \"\"\"\n return [{'a':\ \ 1, 'b': 2}, {'a': 2, 'b': 3}, {'a': 3, 'b': 4}]\n\n" image: python:3.8 - exec-list-dict-maker-v2: + exec-list-dict-maker-2: container: args: - --executor_input - '{{$}}' - --function_to_execute - - list_dict_maker_v2 + - list_dict_maker_2 command: - sh - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -477,24 +477,24 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ - \ *\n\ndef list_dict_maker_v2() -> List[dict]:\n \"\"\"Returns a list\ + \ *\n\ndef list_dict_maker_2() -> List[dict]:\n \"\"\"Returns a list\ \ of dictionaries without type enforcement.\n\n Tests flexibility in\ \ type handling.\n \"\"\"\n return [{'a': 1, 'b': 2}, {'a': 2, 'b':\ \ 3}, {'a': 3, 'b': 4}]\n\n" image: python:3.8 - exec-list-dict-maker-v3: + exec-list-dict-maker-3: container: args: - --executor_input - '{{$}}' - --function_to_execute - - list_dict_maker_v3 + - list_dict_maker_3 command: - sh - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -508,7 +508,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ - \ *\n\ndef list_dict_maker_v3() -> List:\n \"\"\"Returns a basic list\ + \ *\n\ndef list_dict_maker_3() -> List:\n \"\"\"Returns a basic list\ \ (no typing or structure guarantees).\n\n Tests the limits of compiler\ \ type handling.\n \"\"\"\n return [{'a': 1, 'b': 2}, {'a': 2, 'b':\ \ 3}, {'a': 3, 'b': 4}]\n\n" @@ -525,7 +525,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -553,7 +553,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -581,7 +581,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -609,7 +609,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -637,7 +637,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -665,7 +665,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -693,7 +693,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -721,7 +721,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -749,7 +749,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -777,7 +777,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -805,7 +805,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -833,7 +833,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -873,68 +873,68 @@ root: componentRef: name: comp-for-loop-10 dependentTasks: - - list-dict-maker-v1 + - list-dict-maker-1 inputs: parameters: - pipelinechannel--list-dict-maker-v1-Output: + pipelinechannel--list-dict-maker-1-Output: taskOutputParameter: outputParameterKey: Output - producerTask: list-dict-maker-v1 + producerTask: list-dict-maker-1 parameterIterator: - itemInput: pipelinechannel--list-dict-maker-v1-Output-loop-item + itemInput: pipelinechannel--list-dict-maker-1-Output-loop-item items: - inputParameter: pipelinechannel--list-dict-maker-v1-Output + inputParameter: pipelinechannel--list-dict-maker-1-Output taskInfo: name: for-loop-10 for-loop-11: componentRef: name: comp-for-loop-11 dependentTasks: - - list-dict-maker-v2 + - list-dict-maker-2 inputs: parameters: - pipelinechannel--list-dict-maker-v2-Output: + pipelinechannel--list-dict-maker-2-Output: taskOutputParameter: outputParameterKey: Output - producerTask: list-dict-maker-v2 + producerTask: list-dict-maker-2 parameterIterator: - itemInput: pipelinechannel--list-dict-maker-v2-Output-loop-item + itemInput: pipelinechannel--list-dict-maker-2-Output-loop-item items: - inputParameter: pipelinechannel--list-dict-maker-v2-Output + inputParameter: pipelinechannel--list-dict-maker-2-Output taskInfo: name: for-loop-11 for-loop-12: componentRef: name: comp-for-loop-12 dependentTasks: - - list-dict-maker-v3 + - list-dict-maker-3 inputs: parameters: - pipelinechannel--list-dict-maker-v3-Output: + pipelinechannel--list-dict-maker-3-Output: taskOutputParameter: outputParameterKey: Output - producerTask: list-dict-maker-v3 + producerTask: list-dict-maker-3 parameterIterator: - itemInput: pipelinechannel--list-dict-maker-v3-Output-loop-item + itemInput: pipelinechannel--list-dict-maker-3-Output-loop-item items: - inputParameter: pipelinechannel--list-dict-maker-v3-Output + inputParameter: pipelinechannel--list-dict-maker-3-Output taskInfo: name: for-loop-12 for-loop-13: componentRef: name: comp-for-loop-13 dependentTasks: - - list-dict-maker-v1-2 + - list-dict-maker-1-2 inputs: parameters: - pipelinechannel--list-dict-maker-v1-2-Output: + pipelinechannel--list-dict-maker-1-2-Output: taskOutputParameter: outputParameterKey: Output - producerTask: list-dict-maker-v1-2 + producerTask: list-dict-maker-1-2 parameterIterator: - itemInput: pipelinechannel--list-dict-maker-v1-2-Output-loop-item + itemInput: pipelinechannel--list-dict-maker-1-2-Output-loop-item items: - inputParameter: pipelinechannel--list-dict-maker-v1-2-Output + inputParameter: pipelinechannel--list-dict-maker-1-2-Output taskInfo: name: for-loop-13 for-loop-4: @@ -961,57 +961,57 @@ root: componentRef: name: comp-for-loop-9 dependentTasks: - - list-dict-maker-v0 + - list-dict-maker-0 inputs: parameters: - pipelinechannel--list-dict-maker-v0-Output: + pipelinechannel--list-dict-maker-0-Output: taskOutputParameter: outputParameterKey: Output - producerTask: list-dict-maker-v0 + producerTask: list-dict-maker-0 parameterIterator: - itemInput: pipelinechannel--list-dict-maker-v0-Output-loop-item + itemInput: pipelinechannel--list-dict-maker-0-Output-loop-item items: - inputParameter: pipelinechannel--list-dict-maker-v0-Output + inputParameter: pipelinechannel--list-dict-maker-0-Output taskInfo: name: for-loop-9 - list-dict-maker-v0: + list-dict-maker-0: cachingOptions: enableCache: true componentRef: - name: comp-list-dict-maker-v0 + name: comp-list-dict-maker-0 taskInfo: - name: list-dict-maker-v0 - list-dict-maker-v1: + name: list-dict-maker-0 + list-dict-maker-1: cachingOptions: enableCache: true componentRef: - name: comp-list-dict-maker-v1 + name: comp-list-dict-maker-1 taskInfo: - name: list-dict-maker-v1 - list-dict-maker-v1-2: + name: list-dict-maker-1 + list-dict-maker-1-2: cachingOptions: enableCache: true componentRef: - name: comp-list-dict-maker-v1-2 + name: comp-list-dict-maker-1-2 taskInfo: - name: list-dict-maker-v1-2 - list-dict-maker-v2: + name: list-dict-maker-1-2 + list-dict-maker-2: cachingOptions: enableCache: true componentRef: - name: comp-list-dict-maker-v2 + name: comp-list-dict-maker-2 taskInfo: - name: list-dict-maker-v2 - list-dict-maker-v3: + name: list-dict-maker-2 + list-dict-maker-3: cachingOptions: enableCache: true componentRef: - name: comp-list-dict-maker-v3 + name: comp-list-dict-maker-3 taskInfo: - name: list-dict-maker-v3 + name: list-dict-maker-3 inputDefinitions: parameters: loop_parameter: parameterType: LIST schemaVersion: 2.1.0 -sdkVersion: kfp-2.8.0 +sdkVersion: kfp-2.7.0 From f962ef3297b5e04b1603c79c2d57f35bb3a3b6b3 Mon Sep 17 00:00:00 2001 From: droctothorpe Date: Fri, 2 Aug 2024 09:59:00 -0400 Subject: [PATCH 4/7] Improve logic and update tests Signed-off-by: droctothorpe Co-authored-by: zazulam --- backend/src/v2/compiler/argocompiler/argo.go | 27 +++++++++++++------ .../create_mount_delete_dynamic_pvc.yaml | 12 ++++----- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/backend/src/v2/compiler/argocompiler/argo.go b/backend/src/v2/compiler/argocompiler/argo.go index cfa4477d0c7..f4c83d554fc 100644 --- a/backend/src/v2/compiler/argocompiler/argo.go +++ b/backend/src/v2/compiler/argocompiler/argo.go @@ -278,14 +278,25 @@ func (c *workflowCompiler) extractFunctionName(componentName string) string { } executorLabel := c.spec.Components[componentName].GetExecutorLabel() log.Debug("executorLabel: ", executorLabel) - for executorName, executorValue := range c.executors { - log.Debug("executorName: ", executorName) - if executorName == executorLabel { - args := executorValue.GetContainer().Args - componentFunctionName := args[len(args)-1] - log.Debug("componentFunctionName: ", componentFunctionName) - - return componentFunctionName + // There are more nested conditionals here than we would prefer, but we + // don't want to make any assumptions about the presence of specific fields + // in the IR. + if c.executors != nil { + for executorName, executorValue := range c.executors { + log.Debug("executorName: ", executorName) + if executorName == executorLabel { + args := executorValue.GetContainer().GetArgs() + if args != nil { + if len(args) > 1 { + penultimateArg := args[len(args)-2] + if penultimateArg == "--function_to_execute" { + componentFunctionName := args[len(args)-1] + log.Debug("componentFunctionName: ", componentFunctionName) + return componentFunctionName + } + } + } + } } } diff --git a/backend/src/v2/compiler/argocompiler/testdata/create_mount_delete_dynamic_pvc.yaml b/backend/src/v2/compiler/argocompiler/testdata/create_mount_delete_dynamic_pvc.yaml index 45ba6039ee0..0ef25ba74c9 100644 --- a/backend/src/v2/compiler/argocompiler/testdata/create_mount_delete_dynamic_pvc.yaml +++ b/backend/src/v2/compiler/argocompiler/testdata/create_mount_delete_dynamic_pvc.yaml @@ -8,9 +8,9 @@ spec: parameters: - name: kubernetes-comp-comp value: '{"pvcMount":[{"mountPath":"/data","taskOutputParameter":{"outputParameterKey":"name","producerTask":"createpvc"}}]}' - - name: components-comp-comp + - name: components-comp value: '{"executorLabel":"exec-comp"}' - - name: implementations-comp-comp + - name: implementations-comp value: '{"args":["--executor_input","{{$}}","--function_to_execute","comp"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -197,11 +197,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-comp-comp}}' + value: '{{workflow.parameters.components-comp}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-comp"},"dependentTasks":["createpvc"],"taskInfo":{"name":"comp"}}' - name: container - value: '{{workflow.parameters.implementations-comp-comp}}' + value: '{{workflow.parameters.implementations-comp}}' - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' - name: kubernetes-config @@ -222,11 +222,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-comp-comp}}' + value: '{{workflow.parameters.components-comp}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-comp-2"},"dependentTasks":["comp","createpvc"],"taskInfo":{"name":"comp-2"}}' - name: container - value: '{{workflow.parameters.implementations-comp-comp}}' + value: '{{workflow.parameters.implementations-comp}}' - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' - name: kubernetes-config From 4c513ebbbe0cf9938ac3a1bf3c8a13047da78247 Mon Sep 17 00:00:00 2001 From: droctothorpe Date: Tue, 6 Aug 2024 16:44:52 -0400 Subject: [PATCH 5/7] POC hashing command and args Signed-off-by: droctothorpe Co-authored-by: zazulam --- backend/src/v2/compiler/argocompiler/argo.go | 57 ++++++++++--------- .../create_mount_delete_dynamic_pvc.yaml | 12 ++-- .../testdata/create_pod_metadata.yaml | 8 +-- .../argocompiler/testdata/hello_world.yaml | 8 +-- 4 files changed, 43 insertions(+), 42 deletions(-) diff --git a/backend/src/v2/compiler/argocompiler/argo.go b/backend/src/v2/compiler/argocompiler/argo.go index f4c83d554fc..b1798d522fd 100644 --- a/backend/src/v2/compiler/argocompiler/argo.go +++ b/backend/src/v2/compiler/argocompiler/argo.go @@ -15,6 +15,8 @@ package argocompiler import ( + "crypto/sha256" + "encoding/hex" "fmt" "strings" @@ -191,29 +193,29 @@ const ( ) func (c *workflowCompiler) saveComponentSpec(name string, spec *pipelinespec.ComponentSpec) error { - functionName := c.extractFunctionName(name) + hashedComponent := c.hashComponentCommandAndArgs(name) - return c.saveProtoToArguments(argumentsComponents+functionName, spec) + return c.saveProtoToArguments(argumentsComponents+hashedComponent, spec) } // useComponentSpec returns a placeholder we can refer to the component spec // in argo workflow fields. func (c *workflowCompiler) useComponentSpec(name string) (string, error) { - functionName := c.extractFunctionName(name) + hashedComponent := c.hashComponentCommandAndArgs(name) - return c.argumentsPlaceholder(argumentsComponents + functionName) + return c.argumentsPlaceholder(argumentsComponents + hashedComponent) } func (c *workflowCompiler) saveComponentImpl(name string, msg proto.Message) error { - functionName := c.extractFunctionName(name) + hashedComponent := c.hashComponentCommandAndArgs(name) - return c.saveProtoToArguments(argumentsContainers+functionName, msg) + return c.saveProtoToArguments(argumentsContainers+hashedComponent, msg) } func (c *workflowCompiler) useComponentImpl(name string) (string, error) { - functionName := c.extractFunctionName(name) + hashedComponent := c.hashComponentCommandAndArgs(name) - return c.argumentsPlaceholder(argumentsContainers + functionName) + return c.argumentsPlaceholder(argumentsContainers + hashedComponent) } func (c *workflowCompiler) saveKubernetesSpec(name string, spec *structpb.Struct) error { @@ -266,9 +268,9 @@ func (c *workflowCompiler) argumentsPlaceholder(componentName string) (string, e return workflowParameter(componentName), nil } -// extractFunctionName extracts the function name of a component by looking it -// up in the pipeline spec. -func (c *workflowCompiler) extractFunctionName(componentName string) string { +// hashComponentCommandAndArgs combines and hashes command and args fields of a +// given component. +func (c *workflowCompiler) hashComponentCommandAndArgs(componentName string) string { log.Debug("componentName: ", componentName) // The root component is a DAG and therefore doesn't have a corresponding // executor or function name. The final return statement in this function @@ -278,35 +280,34 @@ func (c *workflowCompiler) extractFunctionName(componentName string) string { } executorLabel := c.spec.Components[componentName].GetExecutorLabel() log.Debug("executorLabel: ", executorLabel) - // There are more nested conditionals here than we would prefer, but we - // don't want to make any assumptions about the presence of specific fields - // in the IR. if c.executors != nil { for executorName, executorValue := range c.executors { log.Debug("executorName: ", executorName) if executorName == executorLabel { - args := executorValue.GetContainer().GetArgs() - if args != nil { - if len(args) > 1 { - penultimateArg := args[len(args)-2] - if penultimateArg == "--function_to_execute" { - componentFunctionName := args[len(args)-1] - log.Debug("componentFunctionName: ", componentFunctionName) - return componentFunctionName - } - } + commandList := executorValue.GetContainer().GetCommand() + argList := executorValue.GetContainer().GetArgs() + if commandList == nil && argList == nil { + return componentName + } + stringToHash := strings.Join(commandList, " ") + if argList != nil { + stringToHash += strings.Join(argList, " ") + return hashString(stringToHash) } } } } - log.Debug("No corresponding executor for component: ", componentName) - // We could theoretically return an error here, but since the only - // consequence of not finding a matching executor is reduced deduplication, - // this doesn't result in application failure and we therefore continue. return componentName } +func hashString(s string) string { + h := sha256.New() + h.Write([]byte(s)) + + return hex.EncodeToString(h.Sum(nil)) +} + const ( paramComponent = "component" // component spec paramTask = "task" // task spec diff --git a/backend/src/v2/compiler/argocompiler/testdata/create_mount_delete_dynamic_pvc.yaml b/backend/src/v2/compiler/argocompiler/testdata/create_mount_delete_dynamic_pvc.yaml index 0ef25ba74c9..d51c50da07a 100644 --- a/backend/src/v2/compiler/argocompiler/testdata/create_mount_delete_dynamic_pvc.yaml +++ b/backend/src/v2/compiler/argocompiler/testdata/create_mount_delete_dynamic_pvc.yaml @@ -8,9 +8,9 @@ spec: parameters: - name: kubernetes-comp-comp value: '{"pvcMount":[{"mountPath":"/data","taskOutputParameter":{"outputParameterKey":"name","producerTask":"createpvc"}}]}' - - name: components-comp + - name: components-db3197b81c484742322b041296c85f7998bd2442d4ffa0459f23dc73c57b9ef1 value: '{"executorLabel":"exec-comp"}' - - name: implementations-comp + - name: implementations-db3197b81c484742322b041296c85f7998bd2442d4ffa0459f23dc73c57b9ef1 value: '{"args":["--executor_input","{{$}}","--function_to_execute","comp"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -197,11 +197,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-comp}}' + value: '{{workflow.parameters.components-db3197b81c484742322b041296c85f7998bd2442d4ffa0459f23dc73c57b9ef1}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-comp"},"dependentTasks":["createpvc"],"taskInfo":{"name":"comp"}}' - name: container - value: '{{workflow.parameters.implementations-comp}}' + value: '{{workflow.parameters.implementations-db3197b81c484742322b041296c85f7998bd2442d4ffa0459f23dc73c57b9ef1}}' - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' - name: kubernetes-config @@ -222,11 +222,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-comp}}' + value: '{{workflow.parameters.components-db3197b81c484742322b041296c85f7998bd2442d4ffa0459f23dc73c57b9ef1}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-comp-2"},"dependentTasks":["comp","createpvc"],"taskInfo":{"name":"comp-2"}}' - name: container - value: '{{workflow.parameters.implementations-comp}}' + value: '{{workflow.parameters.implementations-db3197b81c484742322b041296c85f7998bd2442d4ffa0459f23dc73c57b9ef1}}' - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' - name: kubernetes-config diff --git a/backend/src/v2/compiler/argocompiler/testdata/create_pod_metadata.yaml b/backend/src/v2/compiler/argocompiler/testdata/create_pod_metadata.yaml index 1aa1a3fee46..7e70001c9de 100644 --- a/backend/src/v2/compiler/argocompiler/testdata/create_pod_metadata.yaml +++ b/backend/src/v2/compiler/argocompiler/testdata/create_pod_metadata.yaml @@ -8,9 +8,9 @@ spec: parameters: - name: kubernetes-comp-hello-world value: '{"podMetadata":{"annotations":{"experiment_id":"234567","run_id":"123456"},"labels":{"kubeflow.com/common":"test","kubeflow.com/kfp":"pipeline-node"}}}' - - name: components-comp-hello-world + - name: components-285f834445312aa2162cf75d155ac477a6149126369484809c864b8d1943533b value: '{"executorLabel":"exec-hello-world","inputDefinitions":{"parameters":{"text":{"type":"STRING"}}}}' - - name: implementations-comp-hello-world + - name: implementations-285f834445312aa2162cf75d155ac477a6149126369484809c864b8d1943533b value: '{"args":["--text","{{$.inputs.parameters[''text'']}}"],"command":["sh","-ec","program_path=$(mktemp)\nprintf \"%s\" \"$0\" \u003e \"$program_path\"\npython3 -u \"$program_path\" \"$@\"\n","def hello_world(text):\n print(text)\n return text\n\nimport argparse\n_parser @@ -191,11 +191,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-comp-hello-world}}' + value: '{{workflow.parameters.components-285f834445312aa2162cf75d155ac477a6149126369484809c864b8d1943533b}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"hello-world"}}' - name: container - value: '{{workflow.parameters.implementations-comp-hello-world}}' + value: '{{workflow.parameters.implementations-285f834445312aa2162cf75d155ac477a6149126369484809c864b8d1943533b}}' - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' - name: kubernetes-config diff --git a/backend/src/v2/compiler/argocompiler/testdata/hello_world.yaml b/backend/src/v2/compiler/argocompiler/testdata/hello_world.yaml index d59959225e5..b710aa48a0b 100644 --- a/backend/src/v2/compiler/argocompiler/testdata/hello_world.yaml +++ b/backend/src/v2/compiler/argocompiler/testdata/hello_world.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-comp-hello-world + - name: components-285f834445312aa2162cf75d155ac477a6149126369484809c864b8d1943533b value: '{"executorLabel":"exec-hello-world","inputDefinitions":{"parameters":{"text":{"type":"STRING"}}}}' - - name: implementations-comp-hello-world + - name: implementations-285f834445312aa2162cf75d155ac477a6149126369484809c864b8d1943533b value: '{"args":["--text","{{$.inputs.parameters[''text'']}}"],"command":["sh","-ec","program_path=$(mktemp)\nprintf \"%s\" \"$0\" \u003e \"$program_path\"\npython3 -u \"$program_path\" \"$@\"\n","def hello_world(text):\n print(text)\n return text\n\nimport argparse\n_parser @@ -183,11 +183,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-comp-hello-world}}' + value: '{{workflow.parameters.components-285f834445312aa2162cf75d155ac477a6149126369484809c864b8d1943533b}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"hello-world"}}' - name: container - value: '{{workflow.parameters.implementations-comp-hello-world}}' + value: '{{workflow.parameters.implementations-285f834445312aa2162cf75d155ac477a6149126369484809c864b8d1943533b}}' - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' name: hello-world-driver From 68d4c0bd5d85ee6cf21b69f10aa8eb7d2cb6439f Mon Sep 17 00:00:00 2001 From: droctothorpe Date: Tue, 6 Aug 2024 22:31:05 -0400 Subject: [PATCH 6/7] Add comments to clarify the logic Signed-off-by: droctothorpe Co-authored-by: zazulam --- backend/src/v2/compiler/argocompiler/argo.go | 35 +++++++++++--------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/backend/src/v2/compiler/argocompiler/argo.go b/backend/src/v2/compiler/argocompiler/argo.go index b1798d522fd..5a261619373 100644 --- a/backend/src/v2/compiler/argocompiler/argo.go +++ b/backend/src/v2/compiler/argocompiler/argo.go @@ -268,31 +268,36 @@ func (c *workflowCompiler) argumentsPlaceholder(componentName string) (string, e return workflowParameter(componentName), nil } -// hashComponentCommandAndArgs combines and hashes command and args fields of a +// hashComponentCommandAndArgs combines and hashes the command and args fields of a // given component. func (c *workflowCompiler) hashComponentCommandAndArgs(componentName string) string { log.Debug("componentName: ", componentName) - // The root component is a DAG and therefore doesn't have a corresponding - // executor or function name. The final return statement in this function - // would cover this edge case, but this saves us some unecessary iteration. + // Return early for root component since it has no command and args. if componentName == "root" { return componentName } - executorLabel := c.spec.Components[componentName].GetExecutorLabel() - log.Debug("executorLabel: ", executorLabel) - if c.executors != nil { + if c.executors != nil { // Don't bother if there are no executors in the pipeline spec. + // Look up the executorLabel for the component in question. + executorLabel := c.spec.Components[componentName].GetExecutorLabel() + log.Debug("executorLabel: ", executorLabel) + // Iterate through the list of executors. for executorName, executorValue := range c.executors { log.Debug("executorName: ", executorName) + // If one of them matches the executorLabel we extracted earlier... if executorName == executorLabel { + // Get the corresponding list of commands. commandList := executorValue.GetContainer().GetCommand() - argList := executorValue.GetContainer().GetArgs() - if commandList == nil && argList == nil { - return componentName - } - stringToHash := strings.Join(commandList, " ") - if argList != nil { - stringToHash += strings.Join(argList, " ") - return hashString(stringToHash) + // And the list of args. + argsList := executorValue.GetContainer().GetArgs() + if !(commandList == nil && argsList == nil) { // Some components have neither command nor args. + // Convert the command list into a hash string. + stringToHash := strings.Join(commandList, " ") + if argsList != nil { + // Convert the args list into a hash string and append it to the command list hash string. + stringToHash += strings.Join(argsList, " ") + + return hashString(stringToHash) + } } } } From a56df166fe091d2dff22a26d410f66d3fccf8c3e Mon Sep 17 00:00:00 2001 From: droctothorpe Date: Wed, 7 Aug 2024 15:56:30 -0400 Subject: [PATCH 7/7] Hash entire PipelineContainerSpec Signed-off-by: droctothorpe Co-authored-by: zazulam --- backend/src/v2/compiler/argocompiler/argo.go | 48 ++++++++++--------- .../create_mount_delete_dynamic_pvc.yaml | 28 +++++------ .../testdata/create_pod_metadata.yaml | 8 ++-- .../argocompiler/testdata/hello_world.yaml | 8 ++-- 4 files changed, 48 insertions(+), 44 deletions(-) diff --git a/backend/src/v2/compiler/argocompiler/argo.go b/backend/src/v2/compiler/argocompiler/argo.go index 5a261619373..1f1c19ed3ec 100644 --- a/backend/src/v2/compiler/argocompiler/argo.go +++ b/backend/src/v2/compiler/argocompiler/argo.go @@ -17,6 +17,7 @@ package argocompiler import ( "crypto/sha256" "encoding/hex" + "encoding/json" "fmt" "strings" @@ -193,7 +194,7 @@ const ( ) func (c *workflowCompiler) saveComponentSpec(name string, spec *pipelinespec.ComponentSpec) error { - hashedComponent := c.hashComponentCommandAndArgs(name) + hashedComponent := c.hashComponentContainer(name) return c.saveProtoToArguments(argumentsComponents+hashedComponent, spec) } @@ -201,19 +202,19 @@ func (c *workflowCompiler) saveComponentSpec(name string, spec *pipelinespec.Com // useComponentSpec returns a placeholder we can refer to the component spec // in argo workflow fields. func (c *workflowCompiler) useComponentSpec(name string) (string, error) { - hashedComponent := c.hashComponentCommandAndArgs(name) + hashedComponent := c.hashComponentContainer(name) return c.argumentsPlaceholder(argumentsComponents + hashedComponent) } func (c *workflowCompiler) saveComponentImpl(name string, msg proto.Message) error { - hashedComponent := c.hashComponentCommandAndArgs(name) + hashedComponent := c.hashComponentContainer(name) return c.saveProtoToArguments(argumentsContainers+hashedComponent, msg) } func (c *workflowCompiler) useComponentImpl(name string) (string, error) { - hashedComponent := c.hashComponentCommandAndArgs(name) + hashedComponent := c.hashComponentContainer(name) return c.argumentsPlaceholder(argumentsContainers + hashedComponent) } @@ -268,9 +269,9 @@ func (c *workflowCompiler) argumentsPlaceholder(componentName string) (string, e return workflowParameter(componentName), nil } -// hashComponentCommandAndArgs combines and hashes the command and args fields of a -// given component. -func (c *workflowCompiler) hashComponentCommandAndArgs(componentName string) string { +// hashComponentContainer serializes and hashes the container field of a given +// component. +func (c *workflowCompiler) hashComponentContainer(componentName string) string { log.Debug("componentName: ", componentName) // Return early for root component since it has no command and args. if componentName == "root" { @@ -285,19 +286,17 @@ func (c *workflowCompiler) hashComponentCommandAndArgs(componentName string) str log.Debug("executorName: ", executorName) // If one of them matches the executorLabel we extracted earlier... if executorName == executorLabel { - // Get the corresponding list of commands. - commandList := executorValue.GetContainer().GetCommand() - // And the list of args. - argsList := executorValue.GetContainer().GetArgs() - if !(commandList == nil && argsList == nil) { // Some components have neither command nor args. - // Convert the command list into a hash string. - stringToHash := strings.Join(commandList, " ") - if argsList != nil { - // Convert the args list into a hash string and append it to the command list hash string. - stringToHash += strings.Join(argsList, " ") - - return hashString(stringToHash) + // Get the corresponding container. + container := executorValue.GetContainer() + if container != nil { + containerHash, err := hashValue(container) + if err != nil { + // Do not bubble up since this is not a breaking error + // and we can just return the componentName in full. + log.Debug("Error hashing container: ", err) } + + return containerHash } } } @@ -306,11 +305,16 @@ func (c *workflowCompiler) hashComponentCommandAndArgs(componentName string) str return componentName } -func hashString(s string) string { +// hashValue serializes and hashes a provided value. +func hashValue(value interface{}) (string, error) { + bytes, err := json.Marshal(value) + if err != nil { + return "", err + } h := sha256.New() - h.Write([]byte(s)) + h.Write([]byte(bytes)) - return hex.EncodeToString(h.Sum(nil)) + return hex.EncodeToString(h.Sum(nil)), nil } const ( diff --git a/backend/src/v2/compiler/argocompiler/testdata/create_mount_delete_dynamic_pvc.yaml b/backend/src/v2/compiler/argocompiler/testdata/create_mount_delete_dynamic_pvc.yaml index d51c50da07a..7a5565595ed 100644 --- a/backend/src/v2/compiler/argocompiler/testdata/create_mount_delete_dynamic_pvc.yaml +++ b/backend/src/v2/compiler/argocompiler/testdata/create_mount_delete_dynamic_pvc.yaml @@ -8,9 +8,9 @@ spec: parameters: - name: kubernetes-comp-comp value: '{"pvcMount":[{"mountPath":"/data","taskOutputParameter":{"outputParameterKey":"name","producerTask":"createpvc"}}]}' - - name: components-db3197b81c484742322b041296c85f7998bd2442d4ffa0459f23dc73c57b9ef1 + - name: components-95f802401136aebf1bf728a6675d7adba5513b53673a3698e00a6d8744638080 value: '{"executorLabel":"exec-comp"}' - - name: implementations-db3197b81c484742322b041296c85f7998bd2442d4ffa0459f23dc73c57b9ef1 + - name: implementations-95f802401136aebf1bf728a6675d7adba5513b53673a3698e00a6d8744638080 value: '{"args":["--executor_input","{{$}}","--function_to_execute","comp"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -21,13 +21,13 @@ spec: *\n\ndef comp():\n pass\n\n"],"image":"python:3.7"}' - name: kubernetes-comp-comp-2 value: '{"pvcMount":[{"mountPath":"/reused_data","taskOutputParameter":{"outputParameterKey":"name","producerTask":"createpvc"}}]}' - - name: components-comp-createpvc + - name: components-98f254581598234b59377784d6cbf209de79e0bcda8013fe4c4397b5d3a26767 value: '{"executorLabel":"exec-createpvc","inputDefinitions":{"parameters":{"access_modes":{"parameterType":"LIST"},"annotations":{"isOptional":true,"parameterType":"STRUCT"},"pvc_name":{"isOptional":true,"parameterType":"STRING"},"pvc_name_suffix":{"isOptional":true,"parameterType":"STRING"},"size":{"parameterType":"STRING"},"storage_class_name":{"defaultValue":"","isOptional":true,"parameterType":"STRING"},"volume_name":{"isOptional":true,"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"name":{"parameterType":"STRING"}}}}' - - name: implementations-comp-createpvc + - name: implementations-98f254581598234b59377784d6cbf209de79e0bcda8013fe4c4397b5d3a26767 value: '{"image":"argostub/createpvc"}' - - name: components-comp-deletepvc + - name: components-ecfc655dce17b0d317707d37fc226fb7de858cc93d45916945122484a13ef725 value: '{"executorLabel":"exec-deletepvc","inputDefinitions":{"parameters":{"pvc_name":{"parameterType":"STRING"}}}}' - - name: implementations-comp-deletepvc + - name: implementations-ecfc655dce17b0d317707d37fc226fb7de858cc93d45916945122484a13ef725 value: '{"image":"argostub/deletepvc"}' - name: components-root value: '{"dag":{"tasks":{"comp":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-comp"},"dependentTasks":["createpvc"],"taskInfo":{"name":"comp"}},"comp-2":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-comp-2"},"dependentTasks":["comp","createpvc"],"taskInfo":{"name":"comp-2"}},"createpvc":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-createpvc"},"inputs":{"parameters":{"access_modes":{"runtimeValue":{"constant":["ReadWriteOnce"]}},"pvc_name_suffix":{"runtimeValue":{"constant":"-my-pvc"}},"size":{"runtimeValue":{"constant":"5Gi"}},"storage_class_name":{"runtimeValue":{"constant":"standard"}}}},"taskInfo":{"name":"createpvc"}},"deletepvc":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-deletepvc"},"dependentTasks":["comp-2","createpvc"],"inputs":{"parameters":{"pvc_name":{"taskOutputParameter":{"outputParameterKey":"name","producerTask":"createpvc"}}}},"taskInfo":{"name":"deletepvc"}}}}}' @@ -197,11 +197,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-db3197b81c484742322b041296c85f7998bd2442d4ffa0459f23dc73c57b9ef1}}' + value: '{{workflow.parameters.components-95f802401136aebf1bf728a6675d7adba5513b53673a3698e00a6d8744638080}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-comp"},"dependentTasks":["createpvc"],"taskInfo":{"name":"comp"}}' - name: container - value: '{{workflow.parameters.implementations-db3197b81c484742322b041296c85f7998bd2442d4ffa0459f23dc73c57b9ef1}}' + value: '{{workflow.parameters.implementations-95f802401136aebf1bf728a6675d7adba5513b53673a3698e00a6d8744638080}}' - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' - name: kubernetes-config @@ -222,11 +222,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-db3197b81c484742322b041296c85f7998bd2442d4ffa0459f23dc73c57b9ef1}}' + value: '{{workflow.parameters.components-95f802401136aebf1bf728a6675d7adba5513b53673a3698e00a6d8744638080}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-comp-2"},"dependentTasks":["comp","createpvc"],"taskInfo":{"name":"comp-2"}}' - name: container - value: '{{workflow.parameters.implementations-db3197b81c484742322b041296c85f7998bd2442d4ffa0459f23dc73c57b9ef1}}' + value: '{{workflow.parameters.implementations-95f802401136aebf1bf728a6675d7adba5513b53673a3698e00a6d8744638080}}' - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' - name: kubernetes-config @@ -247,11 +247,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-comp-createpvc}}' + value: '{{workflow.parameters.components-98f254581598234b59377784d6cbf209de79e0bcda8013fe4c4397b5d3a26767}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-createpvc"},"inputs":{"parameters":{"access_modes":{"runtimeValue":{"constant":["ReadWriteOnce"]}},"pvc_name_suffix":{"runtimeValue":{"constant":"-my-pvc"}},"size":{"runtimeValue":{"constant":"5Gi"}},"storage_class_name":{"runtimeValue":{"constant":"standard"}}}},"taskInfo":{"name":"createpvc"}}' - name: container - value: '{{workflow.parameters.implementations-comp-createpvc}}' + value: '{{workflow.parameters.implementations-98f254581598234b59377784d6cbf209de79e0bcda8013fe4c4397b5d3a26767}}' - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' name: createpvc @@ -259,11 +259,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-comp-deletepvc}}' + value: '{{workflow.parameters.components-ecfc655dce17b0d317707d37fc226fb7de858cc93d45916945122484a13ef725}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-deletepvc"},"dependentTasks":["comp-2","createpvc"],"inputs":{"parameters":{"pvc_name":{"taskOutputParameter":{"outputParameterKey":"name","producerTask":"createpvc"}}}},"taskInfo":{"name":"deletepvc"}}' - name: container - value: '{{workflow.parameters.implementations-comp-deletepvc}}' + value: '{{workflow.parameters.implementations-ecfc655dce17b0d317707d37fc226fb7de858cc93d45916945122484a13ef725}}' - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' depends: comp-2.Succeeded && createpvc.Succeeded diff --git a/backend/src/v2/compiler/argocompiler/testdata/create_pod_metadata.yaml b/backend/src/v2/compiler/argocompiler/testdata/create_pod_metadata.yaml index 7e70001c9de..67308ba33dd 100644 --- a/backend/src/v2/compiler/argocompiler/testdata/create_pod_metadata.yaml +++ b/backend/src/v2/compiler/argocompiler/testdata/create_pod_metadata.yaml @@ -8,9 +8,9 @@ spec: parameters: - name: kubernetes-comp-hello-world value: '{"podMetadata":{"annotations":{"experiment_id":"234567","run_id":"123456"},"labels":{"kubeflow.com/common":"test","kubeflow.com/kfp":"pipeline-node"}}}' - - name: components-285f834445312aa2162cf75d155ac477a6149126369484809c864b8d1943533b + - name: components-34e222d692a0573c88000b8cb02ad24423491a53e061e9bba36d3718dd4c3390 value: '{"executorLabel":"exec-hello-world","inputDefinitions":{"parameters":{"text":{"type":"STRING"}}}}' - - name: implementations-285f834445312aa2162cf75d155ac477a6149126369484809c864b8d1943533b + - name: implementations-34e222d692a0573c88000b8cb02ad24423491a53e061e9bba36d3718dd4c3390 value: '{"args":["--text","{{$.inputs.parameters[''text'']}}"],"command":["sh","-ec","program_path=$(mktemp)\nprintf \"%s\" \"$0\" \u003e \"$program_path\"\npython3 -u \"$program_path\" \"$@\"\n","def hello_world(text):\n print(text)\n return text\n\nimport argparse\n_parser @@ -191,11 +191,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-285f834445312aa2162cf75d155ac477a6149126369484809c864b8d1943533b}}' + value: '{{workflow.parameters.components-34e222d692a0573c88000b8cb02ad24423491a53e061e9bba36d3718dd4c3390}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"hello-world"}}' - name: container - value: '{{workflow.parameters.implementations-285f834445312aa2162cf75d155ac477a6149126369484809c864b8d1943533b}}' + value: '{{workflow.parameters.implementations-34e222d692a0573c88000b8cb02ad24423491a53e061e9bba36d3718dd4c3390}}' - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' - name: kubernetes-config diff --git a/backend/src/v2/compiler/argocompiler/testdata/hello_world.yaml b/backend/src/v2/compiler/argocompiler/testdata/hello_world.yaml index b710aa48a0b..6bc59366d8d 100644 --- a/backend/src/v2/compiler/argocompiler/testdata/hello_world.yaml +++ b/backend/src/v2/compiler/argocompiler/testdata/hello_world.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-285f834445312aa2162cf75d155ac477a6149126369484809c864b8d1943533b + - name: components-34e222d692a0573c88000b8cb02ad24423491a53e061e9bba36d3718dd4c3390 value: '{"executorLabel":"exec-hello-world","inputDefinitions":{"parameters":{"text":{"type":"STRING"}}}}' - - name: implementations-285f834445312aa2162cf75d155ac477a6149126369484809c864b8d1943533b + - name: implementations-34e222d692a0573c88000b8cb02ad24423491a53e061e9bba36d3718dd4c3390 value: '{"args":["--text","{{$.inputs.parameters[''text'']}}"],"command":["sh","-ec","program_path=$(mktemp)\nprintf \"%s\" \"$0\" \u003e \"$program_path\"\npython3 -u \"$program_path\" \"$@\"\n","def hello_world(text):\n print(text)\n return text\n\nimport argparse\n_parser @@ -183,11 +183,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-285f834445312aa2162cf75d155ac477a6149126369484809c864b8d1943533b}}' + value: '{{workflow.parameters.components-34e222d692a0573c88000b8cb02ad24423491a53e061e9bba36d3718dd4c3390}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"hello-world"}}' - name: container - value: '{{workflow.parameters.implementations-285f834445312aa2162cf75d155ac477a6149126369484809c864b8d1943533b}}' + value: '{{workflow.parameters.implementations-34e222d692a0573c88000b8cb02ad24423491a53e061e9bba36d3718dd4c3390}}' - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' name: hello-world-driver