From 78ea821a17d085cfa07e8f679b8ca8978a67a97d Mon Sep 17 00:00:00 2001 From: Oliver Steffmann Date: Fri, 18 Dec 2020 11:58:17 -0600 Subject: [PATCH] run strategy on ecs --- 0_Setup/algo-reference.yaml | 2 +- 1_Data/Load_Hist_Data_Daily.ipynb | 2 +- 1_Data/Load_Hist_Data_Daily_Public.ipynb | 4 +- 1_Data/Load_Hist_Data_Intraday.ipynb | 2 +- 2_Strategies/Run_Strategy_ECS.ipynb | 59 ++++++++++++++++++++++++ 2_Strategies/deploy.sh | 51 ++++++++++++++++++++ 2_Strategies/docker-compose.yml.template | 13 ++++++ 2_Strategies/ecs-params.yml.template | 21 +++++++++ 8 files changed, 149 insertions(+), 5 deletions(-) create mode 100644 2_Strategies/Run_Strategy_ECS.ipynb create mode 100644 2_Strategies/deploy.sh create mode 100644 2_Strategies/docker-compose.yml.template create mode 100644 2_Strategies/ecs-params.yml.template diff --git a/0_Setup/algo-reference.yaml b/0_Setup/algo-reference.yaml index 8d4ed21..d00e3e9 100644 --- a/0_Setup/algo-reference.yaml +++ b/0_Setup/algo-reference.yaml @@ -209,7 +209,7 @@ Resources: AlgorithmicTradingInstance: Type: AWS::SageMaker::NotebookInstance Properties: - InstanceType: ml.t2.medium + InstanceType: ml.t2.large DefaultCodeRepository: https://github.com/aws-samples/algorithmic-trading RoleArn: !GetAtt 'SageMakerExecutionRole.Arn' diff --git a/1_Data/Load_Hist_Data_Daily.ipynb b/1_Data/Load_Hist_Data_Daily.ipynb index 346ad21..1b2fd4f 100644 --- a/1_Data/Load_Hist_Data_Daily.ipynb +++ b/1_Data/Load_Hist_Data_Daily.ipynb @@ -27,7 +27,7 @@ "outputs": [], "source": [ "# get S3 bucket\n", - "s3bucket=!(aws s3 ls | grep algotrading-data | awk '{print $3}')\n", + "s3bucket=!(aws s3 ls | grep algotrading- | awk '{print $3}')\n", "s3bucket=s3bucket[0]\n", "s3bucket" ] diff --git a/1_Data/Load_Hist_Data_Daily_Public.ipynb b/1_Data/Load_Hist_Data_Daily_Public.ipynb index 7506ab5..9686851 100644 --- a/1_Data/Load_Hist_Data_Daily_Public.ipynb +++ b/1_Data/Load_Hist_Data_Daily_Public.ipynb @@ -38,7 +38,7 @@ "outputs": [], "source": [ "# get S3 bucket\n", - "s3bucket=!(aws s3 ls | grep algotrading-data | awk '{print $3}')\n", + "s3bucket=!(aws s3 ls | grep algotrading- | awk '{print $3}')\n", "s3bucket=s3bucket[0]\n", "s3bucket" ] @@ -106,4 +106,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/1_Data/Load_Hist_Data_Intraday.ipynb b/1_Data/Load_Hist_Data_Intraday.ipynb index a0064c9..704f44a 100644 --- a/1_Data/Load_Hist_Data_Intraday.ipynb +++ b/1_Data/Load_Hist_Data_Intraday.ipynb @@ -33,7 +33,7 @@ "outputs": [], "source": [ "# get S3 bucket\n", - "s3bucket=!(aws s3 ls | grep algotrading-data | awk '{print $3}')\n", + "s3bucket=!(aws s3 ls | grep algotrading- | awk '{print $3}')\n", "s3bucket=s3bucket[0]\n", "s3bucket" ] diff --git a/2_Strategies/Run_Strategy_ECS.ipynb b/2_Strategies/Run_Strategy_ECS.ipynb new file mode 100644 index 0000000..e5edb06 --- /dev/null +++ b/2_Strategies/Run_Strategy_ECS.ipynb @@ -0,0 +1,59 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Please run through the strategy notebook first. This will create the container image." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "strategy_image=\"algo_daily_breakout\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!./deploy.sh $strategy_image" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!./ecs-cli compose up" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "conda_python3", + "language": "python", + "name": "conda_python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.10" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/2_Strategies/deploy.sh b/2_Strategies/deploy.sh new file mode 100644 index 0000000..dd13e1c --- /dev/null +++ b/2_Strategies/deploy.sh @@ -0,0 +1,51 @@ +#!/bin/bash +image=$1 +service=$1 + +# Get the account number associated with the current IAM credentials +account=$(aws sts get-caller-identity --query Account --output text) + +if [ $? -ne 0 ] +then + exit 255 +fi + +# Get the region defined in the current configuration +region=$(aws configure get region) +region=${region:-us-east-1} + +echo "create docker-compose.yml" + +cp docker-compose.yml.template docker-compose.yml +sed -i "s/\$ENV/${env}/g" docker-compose.yml +sed -i "s/\$REGION/${region}/g" docker-compose.yml +sed -i "s/\$IMAGE/${account}.dkr.ecr.${region}.amazonaws.com\/${image}/g" docker-compose.yml +sed -i "s/\$SERVICE/${service}/g" docker-compose.yml + +AWS_EXPORTS=`aws cloudformation list-exports` + +VPC=`echo $AWS_EXPORTS | jq -r '.Exports[] | select (.Name=="AlgorithmicTrading-VPC") | .Value'` +SUBNET1=`echo $AWS_EXPORTS | jq -r '.Exports[] | select (.Name=="AlgorithmicTrading-PrivateSubnet1") | .Value'` +SUBNET2=`echo $AWS_EXPORTS | jq -r '.Exports[] | select (.Name=="AlgorithmicTrading-PrivateSubnet2") | .Value'` +SG=`echo $AWS_EXPORTS | jq -r '.Exports[] | select (.Name=="AlgorithmicTrading-ECSHostSecurityGroup") | .Value'` +TASK_ROLE=`echo $AWS_EXPORTS | jq -r '.Exports[] | select (.Name=="AlgorithmicTrading-ECSTaskExecutionRole") | .Value'` +ROLE_ARN=`echo $AWS_EXPORTS | jq -r '.Exports[] | select (.Name=="AlgorithmicTrading-AlgoExecutionRole-ARN") | .Value'` +CLUSTER=`echo $AWS_EXPORTS | jq -r '.Exports[] | select (.Name=="AlgorithmicTrading-ECSCluster") | .Value'` + +echo "create ecs-params.yml" + +cp ecs-params.yml.template ecs-params.yml +sed -i "s/\$VPC/${VPC}/g" ecs-params.yml +sed -i "s/\$SUBNET1/${SUBNET1}/g" ecs-params.yml +sed -i "s/\$SUBNET2/${SUBNET2}/g" ecs-params.yml +sed -i "s/\$SG/${SG}/g" ecs-params.yml +sed -i "s/\$TASK_ROLE/${TASK_ROLE}/g" ecs-params.yml +sed -i "s@\$ROLE_ARN@${ROLE_ARN}@g" ecs-params.yml + +if [ ! -f "ecs-cli" ] ; then + curl -Lo ecs-cli https://amazon-ecs-cli.s3.amazonaws.com/ecs-cli-linux-amd64-latest + chmod 777 ecs-cli + echo "ecs-cli installed" +fi + +./ecs-cli configure --region ${region} --cluster ${CLUSTER} --default-launch-type FARGATE \ No newline at end of file diff --git a/2_Strategies/docker-compose.yml.template b/2_Strategies/docker-compose.yml.template new file mode 100644 index 0000000..85557ad --- /dev/null +++ b/2_Strategies/docker-compose.yml.template @@ -0,0 +1,13 @@ +version: "3" +services: + $SERVICE: + image: $IMAGE + environment: + - AWS_DEFAULT_REGION=$REGION + - ALGO_CMD=train + logging: + driver: awslogs + options: + awslogs-group: algo + awslogs-region: $REGION + awslogs-stream-prefix: $SERVICE \ No newline at end of file diff --git a/2_Strategies/ecs-params.yml.template b/2_Strategies/ecs-params.yml.template new file mode 100644 index 0000000..f7dfb88 --- /dev/null +++ b/2_Strategies/ecs-params.yml.template @@ -0,0 +1,21 @@ +version: 1 +task_definition: + task_execution_role: $TASK_ROLE + task_role_arn: $ROLE_ARN + ecs_network_mode: awsvpc + task_size: + mem_limit: 512 + cpu_limit: 256 +run_params: + network_configuration: + awsvpc_configuration: + subnets: + - $SUBNET1 + - $SUBNET2 + security_groups: + - $SG + assign_public_ip: DISABLED + service_discovery: + private_dns_namespace: + vpc: $VPC + name: algo