run strategy on ecs

0_Setup/algo-reference.yaml

@@ -209,7 +209,7 @@ Resources:
   AlgorithmicTradingInstance:
     Type: AWS::SageMaker::NotebookInstance
     Properties:
-      InstanceType: ml.t2.medium
+      InstanceType: ml.t2.large
       DefaultCodeRepository: https://github.com/aws-samples/algorithmic-trading
       RoleArn: !GetAtt 'SageMakerExecutionRole.Arn'
 

1_Data/Load_Hist_Data_Daily.ipynb

@@ -27,7 +27,7 @@
    "outputs": [],
    "source": [
     "# get S3 bucket\n",
-    "s3bucket=!(aws s3 ls | grep algotrading-data | awk  '{print $3}')\n",
+    "s3bucket=!(aws s3 ls | grep algotrading- | awk  '{print $3}')\n",
     "s3bucket=s3bucket[0]\n",
     "s3bucket"
    ]

1_Data/Load_Hist_Data_Daily_Public.ipynb

@@ -38,7 +38,7 @@
    "outputs": [],
    "source": [
     "# get S3 bucket\n",
-    "s3bucket=!(aws s3 ls | grep algotrading-data | awk  '{print $3}')\n",
+    "s3bucket=!(aws s3 ls | grep algotrading- | awk  '{print $3}')\n",
     "s3bucket=s3bucket[0]\n",
     "s3bucket"
    ]
@@ -106,4 +106,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 2
-}
+}

1_Data/Load_Hist_Data_Intraday.ipynb

@@ -33,7 +33,7 @@
    "outputs": [],
    "source": [
     "# get S3 bucket\n",
-    "s3bucket=!(aws s3 ls | grep algotrading-data | awk  '{print $3}')\n",
+    "s3bucket=!(aws s3 ls | grep algotrading- | awk  '{print $3}')\n",
     "s3bucket=s3bucket[0]\n",
     "s3bucket"
    ]

2_Strategies/Run_Strategy_ECS.ipynb

@@ -0,0 +1,59 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Please run through the strategy notebook first. This will create the container image."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "strategy_image=\"algo_daily_breakout\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!./deploy.sh $strategy_image"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!./ecs-cli compose up"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "conda_python3",
+   "language": "python",
+   "name": "conda_python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.10"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

2_Strategies/deploy.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+image=$1
+service=$1
+
+# Get the account number associated with the current IAM credentials
+account=$(aws sts get-caller-identity --query Account --output text)
+
+if [ $? -ne 0 ]
+then
+    exit 255
+fi
+
+# Get the region defined in the current configuration
+region=$(aws configure get region)
+region=${region:-us-east-1}
+
+echo "create docker-compose.yml"
+
+cp docker-compose.yml.template docker-compose.yml
+sed -i "s/\$ENV/${env}/g" docker-compose.yml
+sed -i "s/\$REGION/${region}/g" docker-compose.yml
+sed -i "s/\$IMAGE/${account}.dkr.ecr.${region}.amazonaws.com\/${image}/g" docker-compose.yml
+sed -i "s/\$SERVICE/${service}/g" docker-compose.yml
+
+AWS_EXPORTS=`aws cloudformation list-exports`
+
+VPC=`echo $AWS_EXPORTS | jq -r '.Exports[] | select (.Name=="AlgorithmicTrading-VPC") | .Value'`
+SUBNET1=`echo $AWS_EXPORTS | jq -r '.Exports[] | select (.Name=="AlgorithmicTrading-PrivateSubnet1") | .Value'`
+SUBNET2=`echo $AWS_EXPORTS | jq -r '.Exports[] | select (.Name=="AlgorithmicTrading-PrivateSubnet2") | .Value'`
+SG=`echo $AWS_EXPORTS | jq -r '.Exports[] | select (.Name=="AlgorithmicTrading-ECSHostSecurityGroup") | .Value'`
+TASK_ROLE=`echo $AWS_EXPORTS | jq -r '.Exports[] | select (.Name=="AlgorithmicTrading-ECSTaskExecutionRole") | .Value'`
+ROLE_ARN=`echo $AWS_EXPORTS | jq -r '.Exports[] | select (.Name=="AlgorithmicTrading-AlgoExecutionRole-ARN") | .Value'`
+CLUSTER=`echo $AWS_EXPORTS | jq -r '.Exports[] | select (.Name=="AlgorithmicTrading-ECSCluster") | .Value'`
+
+echo "create ecs-params.yml"
+
+cp ecs-params.yml.template ecs-params.yml
+sed -i "s/\$VPC/${VPC}/g" ecs-params.yml
+sed -i "s/\$SUBNET1/${SUBNET1}/g" ecs-params.yml
+sed -i "s/\$SUBNET2/${SUBNET2}/g" ecs-params.yml
+sed -i "s/\$SG/${SG}/g" ecs-params.yml
+sed -i "s/\$TASK_ROLE/${TASK_ROLE}/g" ecs-params.yml
+sed -i "s@\$ROLE_ARN@${ROLE_ARN}@g" ecs-params.yml
+
+if [ ! -f "ecs-cli" ] ; then
+  curl -Lo ecs-cli https://amazon-ecs-cli.s3.amazonaws.com/ecs-cli-linux-amd64-latest
+  chmod 777 ecs-cli
+  echo "ecs-cli installed"
+fi
+
+./ecs-cli configure --region ${region} --cluster ${CLUSTER} --default-launch-type FARGATE

2_Strategies/docker-compose.yml.template

@@ -0,0 +1,13 @@
+version: "3"
+services:
+  $SERVICE:
+    image: $IMAGE
+    environment:
+      - AWS_DEFAULT_REGION=$REGION
+      - ALGO_CMD=train
+    logging:
+      driver: awslogs
+      options:
+        awslogs-group: algo
+        awslogs-region: $REGION
+        awslogs-stream-prefix: $SERVICE

2_Strategies/ecs-params.yml.template

@@ -0,0 +1,21 @@
+version: 1
+task_definition:
+  task_execution_role: $TASK_ROLE
+  task_role_arn: $ROLE_ARN
+  ecs_network_mode: awsvpc
+  task_size:
+    mem_limit: 512
+    cpu_limit: 256
+run_params:
+  network_configuration:
+    awsvpc_configuration:
+      subnets:
+        - $SUBNET1
+        - $SUBNET2
+      security_groups:
+        - $SG
+      assign_public_ip: DISABLED
+  service_discovery:
+    private_dns_namespace:
+        vpc: $VPC
+        name: algo