From c4ad778f5011d8d5943b54e45bc681c39f9c6cb5 Mon Sep 17 00:00:00 2001 From: Bernd Verst Date: Thu, 16 May 2019 14:48:52 -0700 Subject: [PATCH 1/3] Use new container registry. --- code/pipeline.py | 10 +++++----- code/preprocess/build.sh | 4 ++-- code/release/build.sh | 4 ++-- code/scoring/build.sh | 4 ++-- code/training/build.sh | 4 ++-- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/code/pipeline.py b/code/pipeline.py index a31813b..79e025c 100644 --- a/code/pipeline.py +++ b/code/pipeline.py @@ -20,7 +20,7 @@ def dogsandcats_train( # preprocess data operations['preprocess'] = dsl.ContainerOp( name='preprocess', - image='tlaloc.azurecr.io/kubeflow/preprocess', + image='kubeflowregistry.azurecr.io/kubeflow/preprocess', command=['python'], arguments=[ '/scripts/data.py', @@ -35,7 +35,7 @@ def dogsandcats_train( # train operations['train'] = dsl.ContainerOp( name='train', - image='tlaloc.azurecr.io/kubeflow/train', + image='kubeflowregistry.azurecr.io/kubeflow/train', command=['python'], arguments=[ '/scripts/train.py', @@ -54,7 +54,7 @@ def dogsandcats_train( # score operations['score'] = dsl.ContainerOp( name='score', - image='tlaloc.azurecr.io/kubeflow/score', + image='kubeflowregistry.azurecr.io/kubeflow/score', command=['python'], arguments=[ '/scripts/score.py', @@ -67,7 +67,7 @@ def dogsandcats_train( #release operations['release'] = dsl.ContainerOp( name='release', - image='tlaloc.azurecr.io/kubeflow/release', + image='kubeflowregistry.azurecr.io/kubeflow/release', command=['python'], arguments=[ '/scripts/release.py', @@ -91,4 +91,4 @@ def dogsandcats_train( if __name__ == '__main__': import kfp.compiler as compiler - compiler.Compiler().compile(dogsandcats_train, __file__ + '.tar.gz') \ No newline at end of file + compiler.Compiler().compile(dogsandcats_train, __file__ + '.tar.gz') diff --git a/code/preprocess/build.sh b/code/preprocess/build.sh index 9d42cb7..6ce4afb 100644 --- a/code/preprocess/build.sh +++ b/code/preprocess/build.sh @@ -1,2 +1,2 @@ -IMAGE=tlaloc.azurecr.io/kubeflow/preprocess -docker build -t $IMAGE . && docker run -it $IMAGE \ No newline at end of file +IMAGE=kubeflowregistry.azurecr.io/kubeflow/preprocess +docker build -t $IMAGE . && docker run -it $IMAGE diff --git a/code/release/build.sh b/code/release/build.sh index 5965434..649878f 100644 --- a/code/release/build.sh +++ b/code/release/build.sh @@ -1,2 +1,2 @@ -IMAGE=tlaloc.azurecr.io/kubeflow/release -docker build -t $IMAGE . && docker run -it --privileged --env-file aml.env $IMAGE \ No newline at end of file +IMAGE=kubeflowregistry.azurecr.io/kubeflow/release +docker build -t $IMAGE . && docker run -it --privileged --env-file aml.env $IMAGE diff --git a/code/scoring/build.sh b/code/scoring/build.sh index 3ef0099..c7241bd 100644 --- a/code/scoring/build.sh +++ b/code/scoring/build.sh @@ -1,2 +1,2 @@ -IMAGE=tlaloc.azurecr.io/kubeflow/score -docker build -t $IMAGE . && docker run -it $IMAGE \ No newline at end of file +IMAGE=kubeflowregistry.azurecr.io/kubeflow/score +docker build -t $IMAGE . && docker run -it $IMAGE diff --git a/code/training/build.sh b/code/training/build.sh index 7e5df89..ad1c9a9 100644 --- a/code/training/build.sh +++ b/code/training/build.sh @@ -1,2 +1,2 @@ -IMAGE=tlaloc.azurecr.io/kubeflow/train -docker build -t $IMAGE . && docker run -it --privileged --env-file blob.env $IMAGE \ No newline at end of file +IMAGE=kubeflowregistry.azurecr.io/kubeflow/train +docker build -t $IMAGE . && docker run -it --privileged --env-file blob.env $IMAGE From 756116ff2cb858c85929391bf9616070c1d08c3b Mon Sep 17 00:00:00 2001 From: Bernd Verst Date: Thu, 16 May 2019 14:53:11 -0700 Subject: [PATCH 2/3] Add K8s persistent volume definition. --- kubernetes/pvc.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 kubernetes/pvc.yaml diff --git a/kubernetes/pvc.yaml b/kubernetes/pvc.yaml new file mode 100644 index 0000000..bb9d6ad --- /dev/null +++ b/kubernetes/pvc.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: azure-managed-disk +spec: + accessModes: + - ReadWriteOnce + storageClassName: managed-premium + resources: + requests: + storage: 300Gi From f12225b9dc264fd0c46e77bf3c1e7a13807b7dbc Mon Sep 17 00:00:00 2001 From: Bernd Verst Date: Thu, 16 May 2019 16:15:02 -0700 Subject: [PATCH 3/3] Adds persistent volume claim to pipeline. --- code/pipeline.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/code/pipeline.py b/code/pipeline.py index 79e025c..78c91c0 100644 --- a/code/pipeline.py +++ b/code/pipeline.py @@ -7,7 +7,7 @@ ) def dogsandcats_train( persistent_volume_name='azure', - persistent_volume_path='/mnt/azure' + persistent_volume_path='/mnt/azure', base_path='/mnt/azure', epochs=5, batch=32, @@ -80,14 +80,16 @@ def dogsandcats_train( for _, op in operations.items(): op.add_volume( - k8s_client.V1Volume( - host_path=k8s_client.V1HostPathVolumeSource( - path=persistent_volume_path), - name=persistent_volume_name) - ) \ - .add_volume_mount(k8s_client.V1VolumeMount( + k8s_client.V1Volume( + name=persistent_volume_name, + persistent_volume_claim=k8s_client.V1PersistentVolumeClaimVolumeSource( + claim_name='azure-managed-disk') + ) + ).add_volume_mount(k8s_client.V1VolumeMount( mount_path=persistent_volume_path, - name=persistent_volume_name)) + name=persistent_volume_name) + ) + if __name__ == '__main__': import kfp.compiler as compiler