#!/usr/bin/python import os, sys, re, socket, errno, logging, time from subprocess import PIPE, Popen class Server(object): def __init__(self, standby, ws_server, p4debug, p4error): self.TMP = os.path.join(os.environ['HOME'],'tmp') self.debug = p4debug.debug self.error = p4error.exception #self.peeking = peeking self.P4D = 'p4d' self.P4BROKER = 'p4broker' self.P4 = 'p4' self.bin_dir = os.path.join(self.TMP,'bin') self.num_standby = standby self.num_ws_server = ws_server self.MASTER = 'depot-master' self.STANDBY = 'depot-standby' self.WS_SERVER = 'workspace-server' self.ROUTER = 'broker' self.ZOOKEEPER = 'localhost:2181' self.CLUSTER_NAME = os.path.basename(os.environ['HOME']) + '_cluster' self.servers = {} self.service_users = {} self.SUPER_USER = 'super_user' self.SERVER_BIN = 'p4d' self.MASTER_ROOT = os.path.join(self.TMP,self.MASTER) self.STANDBY_ROOT = os.path.join(self.TMP,self.STANDBY) self.WS_SERVER_ROOT = os.path.join(self.TMP,self.WS_SERVER) self.ROUTER_ROOT = os.path.join(self.TMP,self.ROUTER) #self.ZOOKEEPER_ROOT = os.path.join(self.TMP,self.ZOOKEEPER) def build_servers(self): servers = [] servers.append(self.MASTER) for i in range(self.num_standby): servers.append(self.STANDBY) for i in range(self.num_ws_server): servers.append(self.WS_SERVER) return servers def build_map(self, dcs, server_list): s_count = ws_count = 0 map_dict = dict() for each_server in server_list: if each_server == self.MASTER: map_dict = { (lambda x,y: x + '#' + y)(self.CLUSTER_NAME,k):v for k, v in dcs[self.CLUSTER_NAME].iteritems() } tmp_dict = { (lambda x,y: x + '#' + y)(self.MASTER,k):v for k, v in dcs[self.MASTER].iteritems() } map_dict.update(tmp_dict) elif each_server == self.STANDBY: s_count += 1 tmp_dict = { (lambda x,y: x + '#' + y)(self.STANDBY + '_' + str(s_count),k):v for k, v in dcs[self.STANDBY].iteritems() } map_dict.update(tmp_dict) elif each_server == self.WS_SERVER: ws_count += 1 tmp_dict = { (lambda x,y: x + '#' + y)(self.WS_SERVER + '_' + str(ws_count),k):v for k, v in dcs[self.WS_SERVER].iteritems() } map_dict.update(tmp_dict) return [ (lambda x,y: x + '=' + y)(k,str(v)) for k, v in map_dict.iteritems() ] def configure_map(self, server_type, serverid, port): dcs = {} #dcs configuration set against depotMaster dcs[self.CLUSTER_NAME] = { 'monitor':1, 'cluster.target':'localhost:' + str(port), 'P4JOURNAL':'journal', 'journalPrefix':'journal', 'zk.host.port.pairs':self.ZOOKEEPER, 'p4.utils.dir':self.bin_dir, 'rpl.journal.ack':1, 'serviceUser':'dcs_service', 'p4zk.socket.path':os.path.join(self.TMP,'p4zksocket') } dcs[self.MASTER] = { 'cluster':6, 'rpl':2, 'zk':6, 'cluster.id':self.CLUSTER_NAME, 'P4LOG':'', } dcs[self.STANDBY] = { 'cluster':6, 'rpl':2, 'zk':6, 'cluster.id':self.CLUSTER_NAME, 'P4LOG':'', 'P4TARGET':'localhost:' + str(port), 'startup.1':'journalcopy -i 0 -b 1', 'startup.2':'pull -L -i 1 -b 1' } dcs[self.WS_SERVER] = { 'cluster':6, 'rpl':2, 'zk':6, 'P4LOG':'', 'P4TARGET':'localhost:' + str(port), 'cluster.id':self.CLUSTER_NAME, 'startup.1':'pull -i 0 -b 1' } server_list = self.build_servers() map_list = self.build_map(dcs, server_list) configure_cmd = [self.P4,'-p',str(port),'-u',self.SUPER_USER,'configure','set'] for each_conf in map_list: try: configure_cmd.append(each_conf) print("CONF CMD: {0}".format(configure_cmd)) (conf_out, conf_err) = Popen(configure_cmd, stdin=PIPE, stdout=PIPE).communicate() print("CONF OUT: {0}".format(conf_out)) print("CONF ERR: {0}".format(conf_err)) except Exception as e: print e finally: configure_cmd.remove(each_conf) def run_process(self, p4d_cmd, port, server_type, server_root): pid = 0 try: time.sleep(2) p4d_cmd.append('-p') p4d_cmd.append(str(port)) p4d_cmd.append('-r') p4d_cmd.append(server_root) p4d_cmd.append('-L') p4d_cmd.append(server_type + '.log') print('SERVER_CMD: {0}'.format(p4d_cmd)) self.debug('SERVER_CMD: {0}'.format(p4d_cmd)) pid = Popen(p4d_cmd, stdin=PIPE, stdout=PIPE).pid print('P4D_PID: {0}'.format(pid)) print('SERVERID: {0}'.format(os.path.basename(server_root))) print self.debug('P4D_PID: {0}'.format(pid)) self.debug('SERVERID: {0}'.format(os.path.basename(server_root))) except Exception as e: self.error(e) return (os.path.basename(server_root), port, pid) def p4d_process(self,server_type): if server_type == 'depot-master': port = self.get_avail_local_port() self.mkdir_p(self.MASTER_ROOT) p4d_cmd = [self.P4D] (server_id, port, pid) = self.run_process(p4d_cmd, port, server_type, self.MASTER_ROOT) self.servers[server_id + ':' + str(port)] = pid p4d_cmd = [] self.configure_server(server_type, server_id, port, pid) if server_type == 'broker': port = self.get_avail_local_port() self.mkdir_p(self.ROUTER_ROOT) p4d_cmd = [self.P4BROKER] (serverid, port, pid) = self.run_process(p4d_cmd, port, server_type, self.ROUTER_ROOT) self.servers[server_id + ':' + str(port)] = pid p4d_cmd = [] self.configure_server(server_type, server_id, port, pid) bcount = count = 0 if server_type == 'depot-standby': bcount = self.num_standby while(count < bcount): count += 1 port = self.get_avail_local_port() self.mkdir_p(self.STANDBY_ROOT + '_' + str(count)) p4d_cmd = [self.P4D] (server_id, port, pid) = self.run_process(p4d_cmd, port, server_type, self.STANDBY_ROOT + '_' + str(count)) self.servers[server_id + ':' + str(port)] = pid p4d_cmd = [] self.configure_server(server_type, server_id, port, pid) bcount = count = 0 if server_type == 'workspace-server': bcount = self.num_ws_server while(count < bcount): count += 1 port = self.get_avail_local_port() self.mkdir_p(self.WS_SERVER_ROOT + '_' + str(count)) p4d_cmd = [self.P4D] (server_id, port, pid) = self.run_process(p4d_cmd, port, server_type, self.WS_SERVER_ROOT + '_' + str(count)) self.servers[serverid + ':' + str(port)] = pid p4d_cmd = [] self.configure_server(server_type, server_id, port, pid) def configure_server(self, server_type, server_id, port, pid): if server_type == 'depot-master': self.configure_map(server_type, server_id, port) sys.exit(1) self.create_serverid(server_type, server_id, port) self.create_serverspec(server_type, server_id, port) self.create_serviceuser(server_id) self.create_groupuser() self.create_protections(server_id) self.create_configureables(server_type, server_id, port) def create_serverid(self, server_type, server_id, port): id_cmd = [] if server_type == 'depot_master': id_cmd = [self.P4, '-p', str(port), '-u', self.SUPER_USER, 'serverid', server_id] elif server_type == 'depot-standby': id_cmd = [self.P4, '-p', str(port), '-u', self.SUPER_USER, 'serverid', server_id] elif server_type == 'workspace-server': id_cmd = [self.P4, '-p', str(port), '-u', self.SUPER_USER, 'serverid', server_id] try: time.sleep(2) self.debug("SERVER_ID: {0}".format(id_cmd)) (id_out, id_err) = Popen(id_cmd, stdin=PIPE, stdout=PIPE).communicate() print("{0}".format(id_out.strip())) self.debug(id_out.strip()) except Exception as e: self.error(e) def create_serverspec(self, server_type, server_id, port): spec_cmd = [self.P4, '-p', str(self.MASTER_PORT),'-u', self.SUPER_USER, 'server', '-i'] server_spec = 'ServerID:' + server_id + '\n' server_spec += 'Type: server' + '\n' server_spec += 'Name: ' + server_id + '\n' server_spec += 'Address: ' + port + '\n' server_spec += 'Services: ' + server_type + '\n' server_spec += 'Description: ' + server_type + '\n' try: self.debug("SERVER SPEC CMD: {0}".format(spec_cmd)) (spec_out, spec_err) = Popen(spec_cmd, stdin=PIPE, stdout=PIPE).communicate(input=server_spec) print("\nSERVER_SPEC: {0}".format(spec_out.strip())) self.debug("SERVER_SPEC: {0}".format(spec_out.strip())) except Exception as e: self.error(e) finally: spec_cmd.remove('-i') srv = re.compile('^ServerID:\s+(\S+).*$') svc = re.compile('^Services:\s+(\S+).*$') try: spec_cmd.append('-o') spec_cmd.append(server_id) self.debug("ServerSpec CMD: {0}".format(spec_cmd)) (s_out, s_err) = Popen(spec_cmd, stdin=PIPE, stdout=PIPE).communicate() for each_line in s_out.split('\n'): m_srv = srv.match(each_line) m_svc = svc.match(each_line) if m_svc != None: print("SERVER_SPEC saved for Services: {0}".format(str(m_svc.group(1)))) self.debug("SERVER_SPEC saved for Services: {0}".format(str(m_svc.group(1)))) if m_srv != None: print("SERVER_SPEC saved for ServerID: {0}".format(str(m_srv.group(1)))) self.debug("SERVER_SPEC saved for ServerID: {0}".format(str(m_srv.group(1)))) except Exception as e: self.error(e) def create_serviceuser(self, server_id): user_cmd = [self.P4, '-p', str(self.P4PORT), '-u', self.SUPER_USER, 'user', '-f', '-i'] user_spec = 'User: ' + server_id + '_svc' + '\n' user_spec += 'Type: service' + '\n' user_spec += 'Email: ' + server_id + '_svc' + '\n' user_spec += 'Fullname: ' + server_id + '_svc' + '\n' try: self.debug("USER_SPEC CMD: {0} ".format(user_cmd)) (spec_out, spec_err) = Popen(user_cmd, stdin=PIPE, stdout=PIPE).communicate(input=user_spec) print("USER_SPEC: {0} ".format(spec_out.strip())) self.debug("USER_SPEC: {0} ".format(spec_out.strip())) except Exception as e: self.error(e) def create_groupuser(self): group_cmd = [self.P4, '-p', str(self.MASTER_PORT), '-u', self.SUPER_USER, 'group', '-i'] group_spec = 'Group: service_users' + '\n' group_spec += 'Timeout: unlimited' + '\n' group_spec += 'PasswordTimeout: unlimited' + '\n' group_spec += 'Subgroups: ' + '\n' group_spec += 'Owners: ' + '\n' group_spec += 'Users: ' + '\n' group_spec += '\t' + self.SERVICE_USER + '\n' try: self.debug("GROUP SPEC CMD: {0} ".format(group_cmd)) (spec_out, spec_err) = Popen(group_cmd, stdin=PIPE,stdout=PIPE).communicate(input=user_spec) print("GROUP SPEC: {0} ".format(spec_out.strip())) self.debug("GROUP SPEC: {0} ".format(spec_out.strip())) except Exception as e: self.error(e) def create_protections(self): protections_cmd = [self.P4,'-p',str(self.P4PORT),'-u',self.SUPER_USER,'protect','-i'] protect_spec = 'Protections:' + '\n' protect_spec += '\twrite user * * //...' + '\n' protect_spec += '\tsuper user ' + self.SUPER_USER + ' * //...' + '\n' protect_spec += '\tsuper user ' + self.SERVICE_USER + ' * //...' + '\n' try: self.debug("Protect SPEC CMD: {0} ".format(protections_cmd)) (spec_out, spec_err) = Popen(protections_cmd,stdin=PIPE,stdout=PIPE).communicate(input=protect_spec) print("Protect SPEC: {0} ".format(spec_out.strip())) self.debug("Protect SPEC: {0} ".format(spec_out.strip())) except Exception as e: self.error(e) def server_configure(self, server_count, port): configure_cmd = [self.P4,'-p', str(self.P4PORT), '-u', self.SUPER_USER, 'configure', 'set'] if server_count == 0: for each_configurable in ('monitor=2','server=3', 'P4LOG=' + self.MASTER + '.log', 'serviceUser=' + self.SERVICE_USER): each_setting = self.MASTER + '#' + each_configurable configure_cmd.append(each_setting) try: self.debug("CONFIGURE CMD: {0}".format(configure_cmd)) (setting_out,err) = Popen(configure_cmd, stdin=PIPE, stdout=PIPE).communicate() print("SETTING: {0}".format(setting_out.strip())) self.debug("SETTING: {0}".format(setting_out.strip())) sys.stdout.flush() except Exception as e: self.error(e) finally: configure_cmd.remove(each_setting) else: time.sleep(2) for each_configurable in ('P4TARGET=localhost:' + str(self.P4PORT), 'rpl=4','time=1','lbr=3', 'P4LOG=' + self.SLAVE + '_' + str(server_count) + '.log', 'startup.1=pull -i 1','startup.2=pull -u -i 1','startup.3=pull -u -i 1', 'db.replication=readonly', 'lbr.replication=readonly', 'serviceUser=' + self.SERVICE_USER, 'monitor=2','server=3'): each_setting = self.SLAVE + '_' + str(server_count) + '#' + each_configurable configure_cmd.append(each_setting) try: self.debug("CONFIGURE CMD: {0}".format(configure_cmd)) (setting_out,err) = Popen(configure_cmd, stdin=PIPE, stdout=PIPE).communicate() print("SETTING: {0}".format(setting_out.strip())) self.debug("SETTING: {0}".format(setting_out.strip())) sys.stdout.flush() except Exception as e: self.error(e) finally: configure_cmd.remove(each_setting) def ckp_restore(self, server_count, port): ctr_out = 0 #create a directory if not os.path.exists(self.SLAVE + '_' + str(server_count)): os.mkdir(self.SLAVE + '_' + str(server_count)) ckp_cmd = [self.P4,'-p', str(self.P4PORT), '-u', self.SUPER_USER, 'admin', 'checkpoint'] try: self.debug("Checkpoint CMD: {0}".format(ckp_cmd)) (ckp_out, ckp_err) = Popen(ckp_cmd,stdin=PIPE,stdout=PIPE).communicate() print("{0}".format("Checkpoint created")) self.debug("Checkpoint created") except Exception as e: self.error(e) time.sleep(5) counter_cmd = [self.P4,'-p', str(self.P4PORT), '-u', self.SUPER_USER, 'counter', 'journal'] try: self.debug("Journal Counter CMD: {0} ".format(counter_cmd)) (ctr_out, ctr_err) = Popen(counter_cmd,stdin=PIPE,stdout=PIPE).communicate() print("Journal Counter: {0} ".format(ctr_out)) self.debug("Journal Counter: {0} ".format(ctr_out)) except Exception as e: self.error(e) time.sleep(2) #if os.path.exists(self.SLAVE_ROOT + '_' + str(server_count)): # print self.SLAVE_ROOT + '_' + str(server_count) #else: # print "it is not THERE!" #do not get rid of the strip() for ctr_out. ctr_out has line-ending at the end restore_cmd = [self.SERVER_BIN,'-r', self.SLAVE_ROOT + '_' + str(server_count), '-jr', self.MASTER_ROOT + os.sep + 'checkpoint.' + str(ctr_out).strip()] try: self.debug(restore_cmd) (restore_out, restore_err) = Popen(restore_cmd,stdin=PIPE,stdout=PIPE).communicate() print("{0}".format(restore_out)) self.debug(restore_out) except Exception as e: self.error(e) def restart(self, server_count, port): restart_cmd = [self.P4,'-p', str(port), '-u', self.SUPER_USER, 'admin', 'restart'] try: self.debug("RESTART CMD: {0}".format(restart_cmd)) (restart_out, restart_err) = Popen(restart_cmd,stdin=PIPE,stdout=PIPE).communicate() slave_name = self.SLAVE + '_' + str(server_count) print("{0} at PORT {1} restarted.".format(slave_name, str(port))) self.debug("{0} at PORT {1} restarted.".format(slave_name, str(port))) except Exception as e: self.error(e) def get_avail_local_port(self): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.bind(('',0)) return s.getsockname()[1] def mkdir_p(self, path): try: os.makedirs(path) except OSError as exc: # Python >2.5 if exc.errno == errno.EEXIST and os.path.isdir(path): pass else: raise def print_args(*args): print("Type of input: {0}".format(type(args))) print args print for i, each_arg in enumerate(args[0]): print("sys.argv #{0}: {1}".format(i, each_arg)) print def dcs_log(log_name): logging.basicConfig( level=logging.DEBUG, format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s', datefmt='%m-%d %H:%M', filename=log_name + '.log', filemode='w' ) console = logging.StreamHandler() console.setLevel(logging.INFO) formatter=logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s') console.setFormatter(formatter) logging.getLogger('').addHandler(console) p4debug = logging.getLogger('p4debug') p4error = logging.getLogger('p4error') return (p4debug,p4error) def main(): #print_args(sys.argv) #sys.exit(1) if len(sys.argv) != 3: print("Usage: {0} <number_of_standby> <number_of_ws_server>".format(sys.argv[0])) print("e.g. {0} 3 3".format(sys.argv[0])) sys.exit(1) num_standby = int(sys.argv[1]) num_ws_server = int(sys.argv[2]) (p4debug, p4error) = dcs_log(sys.argv[0].split('.')[0]) srv_obj = Server(num_standby, num_ws_server, p4debug, p4error) #for each_type in ['depot-master','depot-standby','workspace-server','broker']: for each_type in ['depot-master','depot-standby','workspace-server']: srv_obj.p4d_process(each_type) for k, v in sorted(srv_obj.servers.iteritems()): print("SERVER:PORT => {0}, PID => {1}".format(k,v)) if __name__ == '__main__': main()
# | Change | User | Description | Committed | |
---|---|---|---|---|---|
#37 | 10226 | Stephen Moon | Update the dcs setup script to the latest and also add a script to monitor the dcs setup status | ||
#36 | 9658 | Stephen Moon | Removed the entries in the broker config file since it gets added or removed on the fly | ||
#35 | 9497 | Stephen Moon | Closed the opened files properly | ||
#34 | 9169 | Stephen Moon | Change it to the new default value of -1 | ||
#33 | 9159 | Stephen Moon | Removed the comment about the program not working on Python 3.3 | ||
#32 | 9158 | Stephen Moon | The program is compatible with Python 2.7 as well as Python 3.3 | ||
#31 | 9148 | Stephen Moon | Added comments about the script not working with Python 3.x | ||
#30 | 9119 | Stephen Moon | This may be the final version with the new broker configuration | ||
#29 | 9065 | Stephen Moon | Made more formatting changes | ||
#28 | 9063 | Stephen Moon | More formatting changes | ||
#27 | 9062 | Stephen Moon | Some formatting and refactoring changes | ||
#26 | 9061 | Stephen Moon |
First draft of the DCS setup script. Bugs may need to be worked out, but it should work |
||
#25 | 9059 | Stephen Moon |
Worked out bugs. A working version |
||
#24 | 9058 | Stephen Moon | Now router is setup using the script | ||
#23 | 9057 | Stephen Moon | Debugging changes | ||
#22 | 9056 | Stephen Moon | Complete working version withtout router | ||
#21 | 9054 | Stephen Moon | Update how checkpoint is taken and is restored | ||
#20 | 9053 | Stephen Moon | Made further changes to how file is written | ||
#19 | 9052 | Stephen Moon | Added a clean_up function | ||
#18 | 9051 | Stephen Moon | Changing of timing how servers are started | ||
#17 | 9050 | Stephen Moon | Remove the wrong parenthesis | ||
#16 | 9049 | Stephen Moon | Wrong variable name | ||
#15 | 9048 | Stephen Moon | key and value needs to be quoted separately for a dictionary entry | ||
#14 | 9047 | Stephen Moon | Remove equal signs from the dictionary entries | ||
#13 | 9046 | Stephen Moon | Added a method to create a shared depot | ||
#12 | 9045 | Stephen Moon | WIP: made number of bug fixes | ||
#11 | 9044 | Stephen Moon | Implemented checkpoint and restore methods | ||
#10 | 9037 | Stephen Moon | Added a logic creating a configurables list | ||
#9 | 9036 | Stephen Moon | WIP: creating mapping method which sets all the configurables against depotMaster | ||
#8 | 9035 | Stephen Moon | Updating the code to comply with Python 2.7 or higher | ||
#7 | 9034 | Stephen Moon |
Update the code to comply with Python 2.7 or higher. Last change did not incorporate the change |
||
#6 | 9033 | Stephen Moon | Change the code to comply with Python 2.7 or higher | ||
#5 | 9019 | Stephen Moon | WIP: Adding more needed methods to make DCS to work | ||
#4 | 9018 | Stephen Moon | WIP: doing the resubmission since the last submission does not reflect the latest changes | ||
#3 | 9017 | Stephen Moon | WIP: adding more configuration methods for dcs | ||
#2 | 8984 | Stephen Moon | Refactoring the code | ||
#1 | 8983 | Stephen Moon | WIP: dcs setup script |