-
Notifications
You must be signed in to change notification settings - Fork 2
/
Copy path01_deploy.py
executable file
·164 lines (124 loc) · 7.16 KB
/
01_deploy.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
#!/usr/bin/env python
import yaml
import subprocess
import os
import sys
import argparse
import re
import time
from jinja2 import Environment, Template, FileSystemLoader
# https://docs.python.org/2/library/argparse.html#
parser = argparse.ArgumentParser(description='Deploy Cluster')
parser.add_argument('-i', type=int, dest="instance", nargs=1, required=True, help='the cluster instance')
parser.add_argument('-bp', dest="bp", required=False, help='Install from Blueprint')
args = parser.parse_args()
instance = args.instance[0]
# Change to script file directory
os.chdir(os.path.dirname(os.path.abspath( __file__ )))
# Set the locations
cfg_path = os.environ['HWX_CFG_DIR']+'/config'
host_path = os.environ['HWX_CFG_DIR']+'/hosts'
cfg_file = os.environ['HWX_CFG_DIR']+'/config/' + str(instance) + '.yaml'
host_file = os.environ['HWX_CFG_DIR']+'/hosts/' + str(instance) + '.yaml'
class Deployment(object):
def swarm(self, cfgYaml):
print("Swarm Deployment............." + cfgYaml["env_set"])
# Environment Set which location
env_set = cfgYaml["env_set"]
env_type = cfgYaml["env_type"]
docker_swarm_mngr = os.environ['DOCKER_SWARM_MANAGER']
# TODO: Check to see if the INFRA stack has been deployed.
docker_stack = 'hwx'+str(instance)
print ("Checking if Stack " + docker_stack + " has already been deployed")
check_stack = False
out = subprocess.check_output(['docker', '-H', docker_swarm_mngr, 'stack', 'ls'])
for line in out.splitlines():
# print('Line: '+ line.decode('utf-8'))
if (re.search(docker_stack, str(line))):
check_stack = True
print('Build the Host File for instance: '+ str(instance))
subprocess.call(['./build-host-yaml.py','-i',str(instance)], stderr=subprocess.STDOUT)
if ( not check_stack ):
print('Stack: ' + docker_stack + ' hasn''t been deployed yet. Doing it now.')
# Environment Set which location
# env_set = cfgYaml["env_set"]
# docker_stack = 'hwx' + str(instance)
env = Environment(
loader = FileSystemLoader('../docker/deployment/swarm/stack-compose/'+env_type)
)
template = env.get_template(env_set + '.yaml')
instance_cfg = template.render(cfgYaml)
# With the compose template, create a qualified composefile and save to tmp.
text_file = open('/tmp/resolved_'+env_type+'_'+env_set + '.yaml', 'w')
text_file.write(instance_cfg)
text_file.close()
# Need to assign labels to hosts to match deployment.
config_vars = '@'+os.environ['HWX_CFG_DIR']+'/config/' + str(instance) + '.yaml'
# subprocess.call(['ansible-playbook', '--extra-vars', config_vars, '../infrastructure/docker-node-labels-' + env_set + '.yaml'])
# Now Deploy
print('Deploy Docker Stack '+ docker_stack + ' with compose file (' + env_type + '_' + env_set + ')')
subprocess.call(['docker', '-H', docker_swarm_mngr, 'stack', 'deploy', '--compose-file','/tmp/resolved_'+env_type+'_'+env_set + '.yaml', docker_stack], stderr=subprocess.STDOUT)
#
print('Pause for 25 seconds while the docker services start')
time.sleep(25)
#
# # Populate Deployment readme.md
# print('Set Readme Docs.')
# subprocess.call(['ansible-playbook', '--extra-vars','@'+cfg_file, '-e', 'cfg_path='+cfg_path, '--tags', 'add', '../config/config-dictionary.yaml'], stderr=subprocess.STDOUT)
else:
print('Stack has already been deployed.')
print(' True up configurations...')
# echo "OS Prep"
# Handled in Docker Image
# print('OS Prep')
# subprocess.call(['ansible-playbook', '-i', host_file,'--extra-vars','@'+cfg_file, '../environment/baremetal/os_pre_reqs.yaml'], stderr=subprocess.STDOUT)
if ( env_type == 'hdp' ):
print('Edge Node Config')
subprocess.call(['ansible-playbook', '-i', host_file,'--extra-vars','@'+cfg_file, '../hdp/setup/hadoopcli.yaml', '-vvv'], stderr=subprocess.STDOUT)
print('Ambari Install Playbook')
subprocess.call(['ansible-playbook', '-i', host_file,'--extra-vars','@'+cfg_file, '../hdp/ambari/ambari_install.yaml'], stderr=subprocess.STDOUT)
if (args.bp):
print('Ambari Blueprint Install Playbook')
subprocess.call(['ansible-playbook', '-i', host_file,'--extra-vars','@'+cfg_file, '../hdp/ambari/ambari_blueprint_install.yaml'], stderr=subprocess.STDOUT)
def baremetal(self, cfgYaml):
print ("Bare metal Deployment......... ")
# TODO: Check if cluster is already deployed
check_stack = False
if ( not check_stack ):
# Populate Deployment readme.md
print('Set Readme Docs.')
subprocess.call(['ansible-playbook', '--extra-vars','@'+cfg_file, '-e', 'cfg_path='+cfg_path, '--tags', 'add', '../config/config-dictionary.yaml'], stderr=subprocess.STDOUT)
else:
print('Stack has already been deployed.')
print(' True up configurations...')
print('Build the Host File for instance: '+ str(instance))
subprocess.call(['./build-host-yaml.py','-i',str(instance)], stderr=subprocess.STDOUT)
# Link Directories
print('Switching Data Dir Link /hadoop to /hwx_data/' + str(cfgYaml['env_instance']) + '/' + cfgYaml['hdp_version'])
subprocess.call(['ansible-playbook', '-i', host_file,'--extra-vars','@'+cfg_file, '../environment/baremetal/set_cldr_dirs.yaml'], stderr=subprocess.STDOUT)
# echo "OS Prep"
# print('OS Prep')
# subprocess.call(['ansible-playbook', '-i', host_file,'--extra-vars','@'+cfg_file, '../environment/baremetal/os_pre_reqs.yaml'], stderr=subprocess.STDOUT)
#
# print('Setup Local Accounts')
# subprocess.call(['ansible-playbook', '-i', host_file,'--extra-vars','@'+cfg_file, '../environment/baremetal/local_hwx_users.yaml'], stderr=subprocess.STDOUT)
print('Edge Node Config')
subprocess.call(['ansible-playbook', '-i', host_file,'--extra-vars','@'+cfg_file, '../hdp/setup/hadoopcli.yaml'], stderr=subprocess.STDOUT)
print('Ambari Install Playbook')
subprocess.call(['ansible-playbook', '-i', host_file,'--extra-vars','@'+cfg_file, '../hdp/ambari/ambari_install.yaml'], stderr=subprocess.STDOUT)
if (args.bp):
print('Ambari Blueprint Install Playbook')
subprocess.call(['ansible-playbook', '-i', host_file,'--extra-vars','@'+cfg_file, '../hdp/ambari/ambari_blueprint_install.yaml'], stderr=subprocess.STDOUT)
def k8s(self, cfgYaml):
print ("k8s wip")
def runDeploy(self, argument, cfgYaml):
print("Arg:" + argument)
method = getattr(self, argument, lambda: "No such deployment model")
return method(cfgYaml)
if (os.path.isfile(cfg_file)):
cfgYaml = yaml.load(open(cfg_file))
deployment = Deployment()
deploy_type = cfgYaml["deploy_type"]
deployment.runDeploy(deploy_type, cfgYaml)
else:
print('Could not find config file: ' + cfg_file)