From d3a694a25d1d6e4df987200d1b177eed5964072d Mon Sep 17 00:00:00 2001 From: Ashraf Fouda Date: Thu, 18 Oct 2018 13:43:24 +0200 Subject: [PATCH 1/5] Adds dev installation Signed-off-by: Ashraf Fouda --- scripts/buildlib/packager.py | 8 + .../cb_master/configure-portal-local.py | 347 ++++++++++++++++++ scripts/dockers/cb_master/dep-manifest.yml | 38 +- scripts/install/installnode | 224 +++++++++-- 4 files changed, 577 insertions(+), 40 deletions(-) create mode 100755 scripts/dockers/cb_master/configure-portal-local.py diff --git a/scripts/buildlib/packager.py b/scripts/buildlib/packager.py index 82ed50d..988a9b7 100644 --- a/scripts/buildlib/packager.py +++ b/scripts/buildlib/packager.py @@ -69,10 +69,18 @@ def check_link(source, destination): elif os.path.isfile(destination): os.remove(destination) return False + if os.path.isdir(src): for filepath in os.listdir(src): srcpath = os.path.join(src, filepath) dstpath = os.path.join(dest, filepath) + if os.path.exists(dstpath): + if os.path.isdir(dstpath): + self._create_link(srcpath, dstpath) + continue + else: + continue + if check_link(srcpath, dstpath): continue os.symlink(srcpath, dstpath.rstrip('/')) diff --git a/scripts/dockers/cb_master/configure-portal-local.py b/scripts/dockers/cb_master/configure-portal-local.py new file mode 100755 index 0000000..1f97f1a --- /dev/null +++ b/scripts/dockers/cb_master/configure-portal-local.py @@ -0,0 +1,347 @@ +from JumpScale import j +import yaml +import os +import requests + +# this is not everything these are only the base components and can be added to + + +class Portal(object): + """ + Class used to define the portal service + """ + + def __init__(self, config): + self.config = config + self.__authheaders = None + self.baseurl = "https://itsyou.online/" + self.scl = j.clients.osis.getNamespace("system") + self.ccl = j.clients.osis.getNamespace("cloudbroker") + self.lcl = j.clients.osis.getNamespace("libvirt") + + @property + def authheaders(self): + if self.__authheaders is None: + accesstokenparams = { + "grant_type": "client_credentials", + "client_id": self.config.client_id, + "client_secret": self.config.secret, + } + accesstoken = requests.post( + os.path.join(self.baseurl, "v1", "oauth", "access_token"), + params=accesstokenparams, + ) + token = accesstoken.json()["access_token"] + self.__authheaders = {"Authorization": "token %s" % token} + return self.__authheaders + + def configure_api_key(self, apikey): + label = apikey["label"] + result = requests.get( + os.path.join( + self.baseurl, "api", "organizations", self.config.client_id, "apikeys", label + ), + headers=self.authheaders, + ) + if result.status_code == 200: + stored_apikey = result.json() + if apikey["callbackURL"] != stored_apikey["callbackURL"]: + print("API key does not match callback url deleting it") + requests.delete( + os.path.join( + self.baseurl, + "api", + "organizations", + self.config.client_id, + "apikeys", + label, + ), + headers=self.authheaders, + ) + apikey = {} + else: + apikey = stored_apikey + + if "secret" not in apikey: + print("Creating API key") + result = requests.post( + os.path.join( + self.baseurl, "api", "organizations", self.config.client_id, "apikeys" + ), + json=apikey, + headers=self.authheaders, + ) + apikey = result.json() + return apikey + + def configure_portal_client(self): + portal_instances = j.core.config.list("portal_client") + for portal_instance in portal_instances: + portal_client = j.core.config.get("portal_client", portal_instance) + portal_client["addr"] = "localhost" + j.core.config.set("portal_client", portal_instance, portal_client) + + def add_user(self): + """ + Add portal user using jsuser. + """ + user = "admin" + passwd = self.config.password + cmd1 = "jsuser list" + res = j.do.execute(cmd1, dieOnNonZeroExitCode=False)[1] + for line in res.splitlines(): + if line.find(user) != -1: + return + + cmd2 = "jsuser add -d %s:%s:admin:fakeemail.test.com:jumpscale" % (user, passwd) + j.do.execute(cmd2, dieOnNonZeroExitCode=False) + + + def configure_user_groups(self, portal): + ovc_environment = self.config.env_subdomain + gid = j.application.whoAmI.gid + portal_links = [ + { + "name": "Cloud Broker", + "url": "/cbgrid", + "scope": "admin", + "theme": "light", + }, + { + "name": "Statistics", + "url": "/grafana/d/1/overall-system-performance", + "scope": "admin", + "theme": "light", + "external": "true", + }, + {"name": "Grid", "url": "/grid", "scope": "admin", "theme": "light"}, + { + "name": "Storage", + "url":self.config.ovs_url, + "scope": "ovs_admin", + "theme": "light", + "external": "true", + }, + {"name": "System", "url": "/system", "scope": "admin", "theme": "light"}, + { + "name": "End User", + "url": "http://{}".format(self.config.fqdn), + "scope": "user", + "theme": "dark", + "external": "true", + }, + ] + + portal["navigationlinks"] = portal_links + portal["url"] = "http://{}".format(self.config.fqdn) + portal["defaultspace"] = "vdc" + portal["force_oauth_instance"] = "itsyouonline" + j.core.config.set("portal", "main", portal) + + # update openvcloud config + openvcloud_config = j.core.config.get("openvcloud", "main") + openvcloud_config["portalurl"] = "https://{}".format(self.config.fqdn) + openvcloud_config["defense_proxy"] = "http://defense-{}".format(self.config.fqdn) + openvcloud_config["supportemail"] = "test@ovclocal.com" + j.core.config.set("openvcloud", "main", openvcloud_config) + + # setup user/groups + for groupname in ( + "user", + "ovs_admin", + "level1", + "level2", + "level3", + "0-access", + ): + if not self.scl.group.search({"id": groupname})[0]: + group = self.scl.group.new() + group.gid = gid + group.id = groupname + group.users = ["admin"] + self.scl.group.set(group) + + # set location + if not self.ccl.location.search({"gid": j.application.whoAmI.gid})[0]: + loc = self.ccl.location.new() + loc.gid = j.application.whoAmI.gid + loc.name = ovc_environment + loc.flag = "black" + loc.locationCode = ovc_environment + self.ccl.location.set(loc) + # set grid + if not self.scl.grid.exists(j.application.whoAmI.gid): + grid = self.scl.grid.new() + grid.id = j.application.whoAmI.gid + grid.name = ovc_environment + else: + grid = self.scl.grid.get(j.application.whoAmI.gid) + + self.scl.grid.set(grid) + # register vnc url + url = "http://novnc-{}/vnc_auto.html?token=".format(self.config.fqdn) + if self.lcl.vnc.count({"url": url, "gid": gid}) == 0: + vnc = self.lcl.vnc.new() + vnc.gid = gid + vnc.url = url + self.lcl.vnc.set(vnc) + # register sizes + sizecbs = [ + ("512 MiB Memory with 1 vcpu", 512, 1), + ("1 GiB Memory with 1 vcpu", 1024, 1), + ("2 GiB Memory with 2 vcpus", 2048, 2), + ("4 GiB Memory with 2 vcpus", 4096, 2), + ("8 GiB Memory with 4 vcpus", 8192, 4), + ("16 GiB Memory with 8 vcpus", 16384, 8), + ] + disksizes = [10, 20, 50, 100, 250, 500, 1000, 2000] + if self.ccl.size.count({}) == 0: + for sizecb in sizecbs: + size = self.ccl.size.new() + size.name = sizecb[0] + size.memory = sizecb[1] + size.vcpus = sizecb[2] + size.disks = disksizes + size.gids = [gid] + self.ccl.size.set(size) + # register network ids + newrange = set(range(int(100), int(1000) + 1)) + if not self.lcl.networkids.exists(gid): + networkids = {"id": gid, "networkids": list(newrange)} + self.lcl.networkids.set(networkids) + # configure Disk-Types + disktypes = [ + ("B", "Boot Disk", "vmstor", 20, True), + ("M", "Meta Data Disk", "vmstor", 20, True), + ("D", "Data Disk", "data", 20, True), + ("C", "CD-ROM Disk", None, None, False), + ("P", "Physical Disk", None, None, False), + ] + for disktype in disktypes: + dtype = self.ccl.disktype.searchOne({"id": disktype[0]}) + if not dtype: + dtype = self.ccl.disktype.new() + dtype.id = disktype[0] + dtype.description = disktype[1] + dtype.vpool = disktype[2] + dtype.cacheratio = disktype[3] + dtype.snapshotable = disktype[4] + self.ccl.disktype.set(dtype) + + def configure_IYO(self): + environment = self.config.env_subdomain + callbackURL = "http://{}/restmachine/system/oauth/authorize".format(self.config.fqdn) + groups = [ + "admin", + "level1", + "level2", + "level3", + "ovs_admin", + "user", + "0-access", + ] + + # register api key + apikeyname = "openvcloud-{}".format(environment) + apikey = { + "callbackURL": callbackURL, + "clientCredentialsGrantType": False, + "label": apikeyname, + } + apikey = self.configure_api_key(apikey) + + # install oauth_client + admin_scopes = ["user:name", "user:email", "user:publickey:ssh"] + user_scopes = ["user:name", "user:email"] + + for group in groups: + admin_scopes.append("user:memberof:{}.{}".format(self.config.client_id, group)) + user_scopes.append("user:memberof:{}.{}".format(self.config.client_id, group)) + + data = { + "id": self.config.client_id, + "logout_url": "", + "redirect_url": callbackURL, + "secret": apikey["secret"], + "url": os.path.join(self.baseurl, "v1/oauth/authorize"), + "url2": os.path.join(self.baseurl, "v1/oauth/access_token"), + "user_info_url": os.path.join(self.baseurl, "api/users/{username}/info"), + } + + admin_data = dict(data) + user_data = dict(data) + + admin_data["scope"] = ",".join(admin_scopes) + user_data["scope"] = ",".join(user_scopes) + + oauthclient = j.core.config.get("oauth_client", "itsyouonline") + + for key, val in admin_data.items(): + oauthclient[key] = val + j.core.config.set("oauth_client", "itsyouonline", oauthclient) + + useroauthclient = j.core.config.get("oauth_client", "itsyouonline_user") + + for key, val in user_data.items(): + useroauthclient[key] = val + j.core.config.set("oauth_client", "itsyouonline_user", useroauthclient) + + # configure groups on itsyouonline + for group in groups: + suborgname = self.config.client_id + "." + group + suborg = {"globalid": suborgname} + print("Check if group %s exists" % suborgname) + result = requests.get( + os.path.join(self.baseurl, "api", "organizations", suborgname), + headers=self.authheaders, + ) + if result.status_code != 200: + print("Creating group {}".format(suborgname)) + result = requests.post( + os.path.join(self.baseurl, "api", "organizations", self.config.client_id), + json=suborg, + headers=self.authheaders, + ) + if result.status_code >= 400: + raise RuntimeError( + "Failed to create suborg {}. Error: {} {}".format( + suborgname, result.status_code, result.text + ) + ) + + # configure portal to use this oauthprovider and restart + portal = j.core.config.get("portal", "main") + portal["force_oauth_instance"] = "itsyouonline" + portal["url"] = self.config.fqdn + j.core.config.set("portal", "main", portal) + return portal + + def patch_mail_client(self): + j.core.config.set("mailclient", "main", self.config["mailclient"]) + + +if __name__ == "__main__": + from argparse import ArgumentParser + + parser = ArgumentParser() + parser.add_argument("-p", "--password", dest="password", help="Password") + parser.add_argument("-fqdn", "--fqdn", dest="fqdn", required=True, help="FQDN") + parser.add_argument( + "-env_subdomain", "--env_subdomain", dest="env_subdomain", help="env_subdomain" + ) + parser.add_argument( + "-client_id", "--client_id", dest="client_id", help="itsyouonline client_id " + ) + parser.add_argument( + "-secret", "--secret", dest="secret", help="itsyouonline Secret" + ) + parser.add_argument( + "-ovs_url", "--ovs_url", dest="ovs_url", help="ovs_url required for local dev installation" + ) + config = parser.parse_args() + portal = Portal(config) + portal.add_user() + portal_data = portal.configure_IYO() + portal.configure_user_groups(portal_data) + portal.configure_portal_client() + j.system.fs.copyDirTree("/opt/jumpscale7/cfg/", "/opt/cfgdir/") diff --git a/scripts/dockers/cb_master/dep-manifest.yml b/scripts/dockers/cb_master/dep-manifest.yml index 14229b3..fca4302 100644 --- a/scripts/dockers/cb_master/dep-manifest.yml +++ b/scripts/dockers/cb_master/dep-manifest.yml @@ -13,25 +13,6 @@ https://git.gig.tech/openvcloud/binary/python: source: web_python/root/jslib/ - dest: /opt/jumpscale7/lib/ source: web_python/root/lib/ -https://github.com/0-complexity/openvcloud: -- dest: /opt/jumpscale7/libext/cloudbrokerlib/ - source: apps/cloudbroker/cloudbrokerlib/ -- dest: /opt/jumpscale7/libext/CloudscalerLibcloud/ - source: libs/CloudscalerLibcloud/CloudscalerLibcloud/ -- dest: /opt/jumpscale7/apps/portals/main/base/ - source: apps/cloudbroker/base/ -- dest: /opt/jumpscale7/apps/portals/main/templates/ - source: apps/cloudbroker/templates/ -- dest: /opt/jumpscale7/apps/portals/main/base/ - source: apps/cbportal/base/ -- dest: /opt/jumpscale7/apps/portals/main/templates/ - source: apps/cbportal/templates/ -- dest: /opt/jumpscale7/apps/osis/logic/ - source: apps/osis/logic/ -- dest: /opt/jumpscale7/apps/agentcontroller/jumpscripts/ - source: libs/agent-scripts/ -- dest: /opt/scripts/versions/ - source: scripts/versions/ https://github.com/0-complexity/selfhealing: - dest: /opt/jumpscale7/apps/agentcontroller/jumpscripts/ source: jumpscripts/ @@ -81,3 +62,22 @@ https://github.com/jumpscale7/jumpscale_portal: source: lib/portal/ - dest: /opt/jumpscale7/apps/portals/main/base/ source: apps/gridportal/base/ +https://github.com/0-complexity/openvcloud: +- dest: /opt/jumpscale7/libext/cloudbrokerlib/ + source: apps/cloudbroker/cloudbrokerlib/ +- dest: /opt/jumpscale7/libext/CloudscalerLibcloud/ + source: libs/CloudscalerLibcloud/CloudscalerLibcloud/ +- dest: /opt/jumpscale7/apps/portals/main/base/ + source: apps/cloudbroker/base/ +- dest: /opt/jumpscale7/apps/portals/main/templates/ + source: apps/cloudbroker/templates/ +- dest: /opt/jumpscale7/apps/portals/main/base/ + source: apps/cbportal/base/ +- dest: /opt/jumpscale7/apps/portals/main/templates/ + source: apps/cbportal/templates/ +- dest: /opt/jumpscale7/apps/osis/logic/ + source: apps/osis/logic/ +- dest: /opt/jumpscale7/apps/agentcontroller/jumpscripts/ + source: libs/agent-scripts/ +- dest: /opt/scripts/versions/ + source: scripts/versions/ diff --git a/scripts/install/installnode b/scripts/install/installnode index bb013f7..48d8f66 100755 --- a/scripts/install/installnode +++ b/scripts/install/installnode @@ -18,9 +18,19 @@ class Node(object): def install_packages(self): j.do.execute("apt-get update") - j.do.execute("apt-get install -y redis-server openvswitch-switch") + j.do.execute("apt-get install -y redis-server openvswitch-switch nginx") def prepare_config(self): + self.grid = { + "config":{ + 'id': self.config.grid_id + }, + "servicename": "system", + "instance": "grid" + } + grid_config = j.core.config.get('system', 'grid') + if not grid_config['id']: + self.update_config(self.grid) self.redis = { "config": { "name": "main", @@ -126,12 +136,22 @@ class Node(object): self.write_config(self.agent_controller_client) def register_service(self, config): - j.system.platform.ubuntu.serviceInstall( - "openvcloud_{}".format(config["servicename"]), - config["daemonpath"], - config["args"], - pwd=config["pwd"], - ) + after=config['after'] if 'after' in config else None + if not after: + j.system.platform.ubuntu.serviceInstall( + "openvcloud_{}".format(config["servicename"]), + config["daemonpath"], + config["args"], + pwd=config["pwd"], + ) + else: + j.system.platform.ubuntu.serviceInstall( + "openvcloud_{}".format(config["servicename"]), + config["daemonpath"], + config["args"], + pwd=config["pwd"], + after=after, + ) j.system.platform.ubuntu.startService( "openvcloud_{}".format(config["servicename"]) ) @@ -141,6 +161,11 @@ class Node(object): def write_config(self, data): j.core.config.set(data["servicename"], data["instance"], data.get("config", {})) + + def update_config(self, data): + conf = j.core.config.get(data['servicename'], data['instance']) + conf.update(data.get("config", {})) + j.core.config.set(data["servicename"], data["instance"], conf) def clean(self): for service in self.services: @@ -190,7 +215,7 @@ class ComputeNode(Node): "rootpasswd": self.config.password, "master.addr": "", "network.gw_mgmt_ip": self.netinfo["gwmgmt_ip"], - "grid.id": self.config.gid, + "grid.id": self.config.grid_id, } self.libvirtlistener = { @@ -203,15 +228,14 @@ class ComputeNode(Node): self.vfwnode = { "config": { "netrange_internal": "10.199.0.0/22", - "passwd": "Dct007", - "newpasswd": "Dct007", + "password": "Dct007", + "newpassword": "Dct007", "login": "vscalers", "ip": "10.199.3.254", }, "servicename": "routeros", "instance": "main", } - self.services.extend([self.vncproxy, self.libvirtlistener]) def install_vfwnode(self): @@ -220,7 +244,7 @@ class ComputeNode(Node): def install_packages(self): super(ComputeNode, self).install_packages() j.do.execute( - "apt-get install -y qemu nginx libvirt-bin python-libvirt virtinst python-pexpect ovmf openvswitch-switch openvswitch-common genisoimage qemu-kvm runc" + "apt-get install -y qemu nginx libmhash2 libvirt-bin python-libvirt virtinst python-pexpect ovmf openvswitch-switch openvswitch-common genisoimage qemu-kvm runc" ) def configure(self): @@ -272,8 +296,8 @@ class ComputeNode(Node): self.install_packages() self.configure() self.install_vncproxy() - self.install_libvirtlistener() self.install_network() + self.install_libvirtlistener() self.install_cb_cpunode_aio() self.install_vfwnode() @@ -285,6 +309,7 @@ class ComputeNode(Node): def install_network(self): import libvirt + self.restart_service("libvirtd") nics = j.system.net.getNics() @@ -347,11 +372,6 @@ class StorageNode(Node): def prepare_config(self): super(StorageNode, self).prepare_config() - # self.data_storage = { - # "param.rootpasswd": self.config.password, - # "param.master.addr": "", - # "param.grid.id": self.config.gid, - # } self.redis["config"]["maxram"] = 200 def install(self): @@ -548,7 +568,166 @@ class ConvergedNode(ComputeNode, StorageNode): def clean(self): super(ConvergedNode, self).clean() +class MasterNode(Node): + def __init__(self, config): + Node.__init__(self, config) + self.prepare_config() + def prepare_config(self): + super(MasterNode, self).prepare_config() + self.ovc_itsyouonline = { + "config":{ + "callbackurl": '{}/restmachine/system/oauth/authorize'.format(self.config.fqdn), + "client_id": self.config.client_id, + "client_secret":self.config.secret, + "environment": self.config.env_subdomain, + } + } + self.portal_client['config'].update({ + "addr": self.config.masterips[0], + "port": 80, + "secret": self.config.password, + } + ) + dbpath = "/opt/jumpscale7/var/mongodb/main/" + j.do.createDir(dbpath) + self.mongodb = { + "config": { + "name": "main", + }, + "servicename": "mongodb", + "instance": "main", + "daemonpath": "/usr/bin/mongod", + "args": "--dbpath {dbpath} --smallfiles --rest --httpinterface".format(dbpath=dbpath), + "pwd": "/opt/jumpscale7", + } + self.mongodb_client = { + "config": { + "addr": self.config.masterips[0], + "replicaset": None, + }, + "servicename": "mongodb_client", + "instance": "main" + } + self.osis = { + "config": { + "name": "main", + }, + "servicename": "osis", + "instance": "main", + "after":"openvcloud_mongodb.target", + "daemonpath": "/usr/local/bin/jspython", + "args": "osisServerStart.py main", + "pwd": "/opt/jumpscale7/apps/osis/", + } + + self.controller = { + "config": { + "name": "main", + }, + "servicename": "agentcontroller", + "after":"openvcloud_jsagent.target", + "instance": "main", + "daemonpath": "/usr/local/bin/jspython", + "args": "controller.py -i main", + "pwd": "/opt/jumpscale7/apps/agentcontroller/", + } + self.portal = { + "config": { + "port": 82, + "url": self.config.fqdn, + "navigationlinks":[ + { + "name": "Cloud Broker", + "url": "/cbgrid", + "scope": "admin", + "theme": "light", + }, + { + "name": "Statistics", + "url": "/grafana/d/1/overall-system-performance", + "scope": "admin", + "theme": "light", + "external": "true", + }, + {"name": "Grid", "url": "/grid", "scope": "admin", "theme": "light"}, + { + "name": "Storage", + "url": self.config.ovs_url, + "scope": "ovs_admin", + "theme": "light", + "external": "true", + }, + {"name": "System", "url": "/system", "scope": "admin", "theme": "light"}, + { + "name": "End User", + "url": self.config.fqdn, + "scope": "user", + "theme": "dark", + "external": "true", + }, + ] + }, + "servicename": "portal", + "instance": "main", + "daemonpath": "/usr/local/bin/jspython", + "args": "portal_start.py main", + "pwd": "/opt/jumpscale7/apps/portals/main/", + } + self.grid = { + "config":{ + 'id': self.config.grid_id + }, + "servicename": "system", + "instance": "grid" + } + self.update_config(self.grid) + self.update_config(self.mongodb_client) + # update openvcloud config + openvcloud_config = j.core.config.get("openvcloud", "main") + openvcloud_config["portalurl"] = self.config.fqdn + openvcloud_config["supportemail"] = j.core.config.get("mailclient", "main")["sender"] + j.core.config.set("openvcloud", "main", openvcloud_config) + + + def install_packages(self): + super(MasterNode, self).install_packages() + j.do.execute( + "apt-get install -y graphviz mongodb libmhash2" + ) + def install_portal(self): + import os + args = "--fqdn={} --password={} --env_subdomain={} --client_id={} --secret={} --ovs_url={}".format( + self.config.fqdn, + self.config.password, + self.config.env_subdomain, + self.config.client_id, + self.config.secret, + self.config.ovs_url + ) + os.system('jspython /opt/code/github/0-complexity/openvcloud_installer/scripts/dockers/cb_master/configure-portal-local.py {}'.format(args)) + self.register_service(self.portal) + self.restart_service('openvcloud_portal') + + def install(self): + self.install_redis() + j.do.execute('systemctl restart nginx') + self.register_service(self.mongodb) + import time + time.sleep(40) + self.register_service(self.osis) + time.sleep(20) + self.install_osis_client() + self.register_service(self.controller) + self.install_agentcontroller_client() + time.sleep(20) + self.install_jsagent() + self.install_portal() + self.install_portal_client() + + + + if __name__ == "__main__": from argparse import ArgumentParser @@ -566,7 +745,6 @@ if __name__ == "__main__": help="master ips", ) parser.add_argument("-p", "--password", dest="password", help="Password") - parser.add_argument("-gid", "--gid", dest="gid", help="GID", required=True) parser.add_argument("-fqdn", "--fqdn", dest="fqdn", required=True, help="FQDN") parser.add_argument( "-vxbackend_vlan", @@ -581,7 +759,7 @@ if __name__ == "__main__": "-gwmgmt_vlan", "--gwmgmt_vlan", dest="gwmgmt_vlan", help="gwmgmt_vlan" ) parser.add_argument("-gwmgmt_ip", "--gwmgmt_ip", dest="gwmgmt_ip", help="gwmgmt_ip") - parser.add_argument("-grid_id", "--grid_id", dest="grid_id", help="grid_id") + parser.add_argument("-grid_id", "--grid_id", dest="grid_id", help="grid_id", type=int) parser.add_argument( "-env_subdomain", "--env_subdomain", dest="env_subdomain", help="env_subdomain" ) @@ -591,8 +769,10 @@ if __name__ == "__main__": parser.add_argument( "-secret", "--secret", dest="secret", help="itsyouonline Secret" ) + parser.add_argument( + "-ovs_url", "--ovs_url", dest="ovs_url", help="ovs_url required for local dev installation" + ) parser.add_argument("--update", help="Update", action="store_true") - config = parser.parse_args() if "cpu" in config.roles and "storage" in config.roles: node = ConvergedNode(config) @@ -600,6 +780,8 @@ if __name__ == "__main__": node = ComputeNode(config) elif "storage" in config.roles: node = StorageNode(config) + elif "master" in config.roles: + node = MasterNode(config) else: node = Node(config) if config.update: From 61dd47f55c37d3c863b9e2013177ce7c5e9586ec Mon Sep 17 00:00:00 2001 From: Ashraf Fouda Date: Thu, 18 Oct 2018 13:43:24 +0200 Subject: [PATCH 2/5] Adds dev installation Signed-off-by: Ashraf Fouda --- scripts/buildlib/packager.py | 8 + .../cb_master/configure-portal-local.py | 347 ++++++++++++++++++ scripts/dockers/cb_master/dep-manifest.yml | 1 + scripts/install/installnode | 222 ++++++++++- 4 files changed, 559 insertions(+), 19 deletions(-) create mode 100755 scripts/dockers/cb_master/configure-portal-local.py diff --git a/scripts/buildlib/packager.py b/scripts/buildlib/packager.py index 70a764f..bd8732c 100644 --- a/scripts/buildlib/packager.py +++ b/scripts/buildlib/packager.py @@ -69,10 +69,18 @@ def check_link(source, destination): elif os.path.isfile(destination): os.remove(destination) return False + if os.path.isdir(src): for filepath in os.listdir(src): srcpath = os.path.join(src, filepath) dstpath = os.path.join(dest, filepath) + if os.path.exists(dstpath): + if os.path.isdir(dstpath): + self._create_link(srcpath, dstpath) + continue + else: + continue + if check_link(srcpath, dstpath): continue os.symlink(srcpath, dstpath.rstrip('/')) diff --git a/scripts/dockers/cb_master/configure-portal-local.py b/scripts/dockers/cb_master/configure-portal-local.py new file mode 100755 index 0000000..1f97f1a --- /dev/null +++ b/scripts/dockers/cb_master/configure-portal-local.py @@ -0,0 +1,347 @@ +from JumpScale import j +import yaml +import os +import requests + +# this is not everything these are only the base components and can be added to + + +class Portal(object): + """ + Class used to define the portal service + """ + + def __init__(self, config): + self.config = config + self.__authheaders = None + self.baseurl = "https://itsyou.online/" + self.scl = j.clients.osis.getNamespace("system") + self.ccl = j.clients.osis.getNamespace("cloudbroker") + self.lcl = j.clients.osis.getNamespace("libvirt") + + @property + def authheaders(self): + if self.__authheaders is None: + accesstokenparams = { + "grant_type": "client_credentials", + "client_id": self.config.client_id, + "client_secret": self.config.secret, + } + accesstoken = requests.post( + os.path.join(self.baseurl, "v1", "oauth", "access_token"), + params=accesstokenparams, + ) + token = accesstoken.json()["access_token"] + self.__authheaders = {"Authorization": "token %s" % token} + return self.__authheaders + + def configure_api_key(self, apikey): + label = apikey["label"] + result = requests.get( + os.path.join( + self.baseurl, "api", "organizations", self.config.client_id, "apikeys", label + ), + headers=self.authheaders, + ) + if result.status_code == 200: + stored_apikey = result.json() + if apikey["callbackURL"] != stored_apikey["callbackURL"]: + print("API key does not match callback url deleting it") + requests.delete( + os.path.join( + self.baseurl, + "api", + "organizations", + self.config.client_id, + "apikeys", + label, + ), + headers=self.authheaders, + ) + apikey = {} + else: + apikey = stored_apikey + + if "secret" not in apikey: + print("Creating API key") + result = requests.post( + os.path.join( + self.baseurl, "api", "organizations", self.config.client_id, "apikeys" + ), + json=apikey, + headers=self.authheaders, + ) + apikey = result.json() + return apikey + + def configure_portal_client(self): + portal_instances = j.core.config.list("portal_client") + for portal_instance in portal_instances: + portal_client = j.core.config.get("portal_client", portal_instance) + portal_client["addr"] = "localhost" + j.core.config.set("portal_client", portal_instance, portal_client) + + def add_user(self): + """ + Add portal user using jsuser. + """ + user = "admin" + passwd = self.config.password + cmd1 = "jsuser list" + res = j.do.execute(cmd1, dieOnNonZeroExitCode=False)[1] + for line in res.splitlines(): + if line.find(user) != -1: + return + + cmd2 = "jsuser add -d %s:%s:admin:fakeemail.test.com:jumpscale" % (user, passwd) + j.do.execute(cmd2, dieOnNonZeroExitCode=False) + + + def configure_user_groups(self, portal): + ovc_environment = self.config.env_subdomain + gid = j.application.whoAmI.gid + portal_links = [ + { + "name": "Cloud Broker", + "url": "/cbgrid", + "scope": "admin", + "theme": "light", + }, + { + "name": "Statistics", + "url": "/grafana/d/1/overall-system-performance", + "scope": "admin", + "theme": "light", + "external": "true", + }, + {"name": "Grid", "url": "/grid", "scope": "admin", "theme": "light"}, + { + "name": "Storage", + "url":self.config.ovs_url, + "scope": "ovs_admin", + "theme": "light", + "external": "true", + }, + {"name": "System", "url": "/system", "scope": "admin", "theme": "light"}, + { + "name": "End User", + "url": "http://{}".format(self.config.fqdn), + "scope": "user", + "theme": "dark", + "external": "true", + }, + ] + + portal["navigationlinks"] = portal_links + portal["url"] = "http://{}".format(self.config.fqdn) + portal["defaultspace"] = "vdc" + portal["force_oauth_instance"] = "itsyouonline" + j.core.config.set("portal", "main", portal) + + # update openvcloud config + openvcloud_config = j.core.config.get("openvcloud", "main") + openvcloud_config["portalurl"] = "https://{}".format(self.config.fqdn) + openvcloud_config["defense_proxy"] = "http://defense-{}".format(self.config.fqdn) + openvcloud_config["supportemail"] = "test@ovclocal.com" + j.core.config.set("openvcloud", "main", openvcloud_config) + + # setup user/groups + for groupname in ( + "user", + "ovs_admin", + "level1", + "level2", + "level3", + "0-access", + ): + if not self.scl.group.search({"id": groupname})[0]: + group = self.scl.group.new() + group.gid = gid + group.id = groupname + group.users = ["admin"] + self.scl.group.set(group) + + # set location + if not self.ccl.location.search({"gid": j.application.whoAmI.gid})[0]: + loc = self.ccl.location.new() + loc.gid = j.application.whoAmI.gid + loc.name = ovc_environment + loc.flag = "black" + loc.locationCode = ovc_environment + self.ccl.location.set(loc) + # set grid + if not self.scl.grid.exists(j.application.whoAmI.gid): + grid = self.scl.grid.new() + grid.id = j.application.whoAmI.gid + grid.name = ovc_environment + else: + grid = self.scl.grid.get(j.application.whoAmI.gid) + + self.scl.grid.set(grid) + # register vnc url + url = "http://novnc-{}/vnc_auto.html?token=".format(self.config.fqdn) + if self.lcl.vnc.count({"url": url, "gid": gid}) == 0: + vnc = self.lcl.vnc.new() + vnc.gid = gid + vnc.url = url + self.lcl.vnc.set(vnc) + # register sizes + sizecbs = [ + ("512 MiB Memory with 1 vcpu", 512, 1), + ("1 GiB Memory with 1 vcpu", 1024, 1), + ("2 GiB Memory with 2 vcpus", 2048, 2), + ("4 GiB Memory with 2 vcpus", 4096, 2), + ("8 GiB Memory with 4 vcpus", 8192, 4), + ("16 GiB Memory with 8 vcpus", 16384, 8), + ] + disksizes = [10, 20, 50, 100, 250, 500, 1000, 2000] + if self.ccl.size.count({}) == 0: + for sizecb in sizecbs: + size = self.ccl.size.new() + size.name = sizecb[0] + size.memory = sizecb[1] + size.vcpus = sizecb[2] + size.disks = disksizes + size.gids = [gid] + self.ccl.size.set(size) + # register network ids + newrange = set(range(int(100), int(1000) + 1)) + if not self.lcl.networkids.exists(gid): + networkids = {"id": gid, "networkids": list(newrange)} + self.lcl.networkids.set(networkids) + # configure Disk-Types + disktypes = [ + ("B", "Boot Disk", "vmstor", 20, True), + ("M", "Meta Data Disk", "vmstor", 20, True), + ("D", "Data Disk", "data", 20, True), + ("C", "CD-ROM Disk", None, None, False), + ("P", "Physical Disk", None, None, False), + ] + for disktype in disktypes: + dtype = self.ccl.disktype.searchOne({"id": disktype[0]}) + if not dtype: + dtype = self.ccl.disktype.new() + dtype.id = disktype[0] + dtype.description = disktype[1] + dtype.vpool = disktype[2] + dtype.cacheratio = disktype[3] + dtype.snapshotable = disktype[4] + self.ccl.disktype.set(dtype) + + def configure_IYO(self): + environment = self.config.env_subdomain + callbackURL = "http://{}/restmachine/system/oauth/authorize".format(self.config.fqdn) + groups = [ + "admin", + "level1", + "level2", + "level3", + "ovs_admin", + "user", + "0-access", + ] + + # register api key + apikeyname = "openvcloud-{}".format(environment) + apikey = { + "callbackURL": callbackURL, + "clientCredentialsGrantType": False, + "label": apikeyname, + } + apikey = self.configure_api_key(apikey) + + # install oauth_client + admin_scopes = ["user:name", "user:email", "user:publickey:ssh"] + user_scopes = ["user:name", "user:email"] + + for group in groups: + admin_scopes.append("user:memberof:{}.{}".format(self.config.client_id, group)) + user_scopes.append("user:memberof:{}.{}".format(self.config.client_id, group)) + + data = { + "id": self.config.client_id, + "logout_url": "", + "redirect_url": callbackURL, + "secret": apikey["secret"], + "url": os.path.join(self.baseurl, "v1/oauth/authorize"), + "url2": os.path.join(self.baseurl, "v1/oauth/access_token"), + "user_info_url": os.path.join(self.baseurl, "api/users/{username}/info"), + } + + admin_data = dict(data) + user_data = dict(data) + + admin_data["scope"] = ",".join(admin_scopes) + user_data["scope"] = ",".join(user_scopes) + + oauthclient = j.core.config.get("oauth_client", "itsyouonline") + + for key, val in admin_data.items(): + oauthclient[key] = val + j.core.config.set("oauth_client", "itsyouonline", oauthclient) + + useroauthclient = j.core.config.get("oauth_client", "itsyouonline_user") + + for key, val in user_data.items(): + useroauthclient[key] = val + j.core.config.set("oauth_client", "itsyouonline_user", useroauthclient) + + # configure groups on itsyouonline + for group in groups: + suborgname = self.config.client_id + "." + group + suborg = {"globalid": suborgname} + print("Check if group %s exists" % suborgname) + result = requests.get( + os.path.join(self.baseurl, "api", "organizations", suborgname), + headers=self.authheaders, + ) + if result.status_code != 200: + print("Creating group {}".format(suborgname)) + result = requests.post( + os.path.join(self.baseurl, "api", "organizations", self.config.client_id), + json=suborg, + headers=self.authheaders, + ) + if result.status_code >= 400: + raise RuntimeError( + "Failed to create suborg {}. Error: {} {}".format( + suborgname, result.status_code, result.text + ) + ) + + # configure portal to use this oauthprovider and restart + portal = j.core.config.get("portal", "main") + portal["force_oauth_instance"] = "itsyouonline" + portal["url"] = self.config.fqdn + j.core.config.set("portal", "main", portal) + return portal + + def patch_mail_client(self): + j.core.config.set("mailclient", "main", self.config["mailclient"]) + + +if __name__ == "__main__": + from argparse import ArgumentParser + + parser = ArgumentParser() + parser.add_argument("-p", "--password", dest="password", help="Password") + parser.add_argument("-fqdn", "--fqdn", dest="fqdn", required=True, help="FQDN") + parser.add_argument( + "-env_subdomain", "--env_subdomain", dest="env_subdomain", help="env_subdomain" + ) + parser.add_argument( + "-client_id", "--client_id", dest="client_id", help="itsyouonline client_id " + ) + parser.add_argument( + "-secret", "--secret", dest="secret", help="itsyouonline Secret" + ) + parser.add_argument( + "-ovs_url", "--ovs_url", dest="ovs_url", help="ovs_url required for local dev installation" + ) + config = parser.parse_args() + portal = Portal(config) + portal.add_user() + portal_data = portal.configure_IYO() + portal.configure_user_groups(portal_data) + portal.configure_portal_client() + j.system.fs.copyDirTree("/opt/jumpscale7/cfg/", "/opt/cfgdir/") diff --git a/scripts/dockers/cb_master/dep-manifest.yml b/scripts/dockers/cb_master/dep-manifest.yml index 3d4e833..22791da 100644 --- a/scripts/dockers/cb_master/dep-manifest.yml +++ b/scripts/dockers/cb_master/dep-manifest.yml @@ -13,6 +13,7 @@ https://git.gig.tech/openvcloud/binary/python: source: web_python/root/jslib/ - dest: /opt/jumpscale7/lib/ source: web_python/root/lib/ + https://github.com/jumpscale7/jumpscale_core7: - dest: /opt/jumpscale7/cfg/system/ source: bootstrap/ diff --git a/scripts/install/installnode b/scripts/install/installnode index c6c5a95..0964916 100755 --- a/scripts/install/installnode +++ b/scripts/install/installnode @@ -19,15 +19,27 @@ class Node(object): def install_packages(self): j.do.execute("apt-get update") - j.do.execute("apt-get install -y redis-server openvswitch-switch libmhash2") + j.do.execute("apt-get install -y redis-server openvswitch-switch libmhash2 nginx") def prepare_config(self): + self.grid = { + "config":{ + 'id': self.config.grid_id + }, + "servicename": "system", + "instance": "grid" + } + grid_config = j.core.config.get('system', 'grid') + if not grid_config['id']: + self.update_config(self.grid) + if self.extraroles: grid = j.core.config.get("system", "grid") for role in self.extraroles: if role not in grid["node"]["roles"]: grid["node"]["roles"].append(role) j.core.config.set("system", "grid", grid) + self.redis = { "config": { "name": "main", @@ -133,13 +145,23 @@ class Node(object): self.write_config(self.agent_controller_client) def register_service(self, config): - j.system.platform.ubuntu.serviceInstall( - "openvcloud_{}".format(config["servicename"]), - config["daemonpath"], - config["args"], - pwd=config["pwd"], - ) - j.system.platform.ubuntu.restartService( + after=config['after'] if 'after' in config else None + if not after: + j.system.platform.ubuntu.serviceInstall( + "openvcloud_{}".format(config["servicename"]), + config["daemonpath"], + config["args"], + pwd=config["pwd"], + ) + else: + j.system.platform.ubuntu.serviceInstall( + "openvcloud_{}".format(config["servicename"]), + config["daemonpath"], + config["args"], + pwd=config["pwd"], + after=after, + ) + j.system.platform.ubuntu.startService( "openvcloud_{}".format(config["servicename"]) ) @@ -148,6 +170,11 @@ class Node(object): def write_config(self, data): j.core.config.set(data["servicename"], data["instance"], data.get("config", {})) + + def update_config(self, data): + conf = j.core.config.get(data['servicename'], data['instance']) + conf.update(data.get("config", {})) + j.core.config.set(data["servicename"], data["instance"], conf) def clean(self): for service in self.services: @@ -198,7 +225,7 @@ class ComputeNode(Node): "rootpasswd": self.config.password, "master.addr": "", "network.gw_mgmt_ip": self.netinfo["gwmgmt_ip"], - "grid.id": self.config.gid, + "grid.id": self.config.grid_id, } self.libvirtlistener = { @@ -217,7 +244,6 @@ class ComputeNode(Node): "servicename": "routeros", "instance": "main", } - self.services.extend([self.vncproxy, self.libvirtlistener]) def install_vfwnode(self): @@ -278,8 +304,8 @@ class ComputeNode(Node): self.install_packages() self.configure() self.install_vncproxy() - self.install_libvirtlistener() self.install_network() + self.install_libvirtlistener() self.install_cb_cpunode_aio() self.install_vfwnode() @@ -291,6 +317,7 @@ class ComputeNode(Node): def install_network(self): import libvirt + self.restart_service("libvirtd") nics = j.system.net.getNics() @@ -342,11 +369,6 @@ class StorageNode(Node): def prepare_config(self): super(StorageNode, self).prepare_config() - # self.data_storage = { - # "param.rootpasswd": self.config.password, - # "param.master.addr": "", - # "param.grid.id": self.config.gid, - # } self.redis["config"]["maxram"] = 200 def install(self): @@ -541,7 +563,166 @@ class ConvergedNode(ComputeNode, StorageNode): def clean(self): super(ConvergedNode, self).clean() +class MasterNode(Node): + def __init__(self, config): + Node.__init__(self, config) + self.prepare_config() + + def prepare_config(self): + super(MasterNode, self).prepare_config() + self.ovc_itsyouonline = { + "config":{ + "callbackurl": '{}/restmachine/system/oauth/authorize'.format(self.config.fqdn), + "client_id": self.config.client_id, + "client_secret":self.config.secret, + "environment": self.config.env_subdomain, + } + } + self.portal_client['config'].update({ + "addr": self.config.masterips[0], + "port": 80, + "secret": self.config.password, + } + ) + dbpath = "/opt/jumpscale7/var/mongodb/main/" + j.do.createDir(dbpath) + self.mongodb = { + "config": { + "name": "main", + }, + "servicename": "mongodb", + "instance": "main", + "daemonpath": "/usr/bin/mongod", + "args": "--dbpath {dbpath} --smallfiles --rest --httpinterface".format(dbpath=dbpath), + "pwd": "/opt/jumpscale7", + } + self.mongodb_client = { + "config": { + "addr": self.config.masterips[0], + "replicaset": None, + }, + "servicename": "mongodb_client", + "instance": "main" + } + self.osis = { + "config": { + "name": "main", + }, + "servicename": "osis", + "instance": "main", + "after":"openvcloud_mongodb.target", + "daemonpath": "/usr/local/bin/jspython", + "args": "osisServerStart.py main", + "pwd": "/opt/jumpscale7/apps/osis/", + } + self.controller = { + "config": { + "name": "main", + }, + "servicename": "agentcontroller", + "after":"openvcloud_jsagent.target", + "instance": "main", + "daemonpath": "/usr/local/bin/jspython", + "args": "controller.py -i main", + "pwd": "/opt/jumpscale7/apps/agentcontroller/", + } + self.portal = { + "config": { + "port": 82, + "url": self.config.fqdn, + "navigationlinks":[ + { + "name": "Cloud Broker", + "url": "/cbgrid", + "scope": "admin", + "theme": "light", + }, + { + "name": "Statistics", + "url": "/grafana/d/1/overall-system-performance", + "scope": "admin", + "theme": "light", + "external": "true", + }, + {"name": "Grid", "url": "/grid", "scope": "admin", "theme": "light"}, + { + "name": "Storage", + "url": self.config.ovs_url, + "scope": "ovs_admin", + "theme": "light", + "external": "true", + }, + {"name": "System", "url": "/system", "scope": "admin", "theme": "light"}, + { + "name": "End User", + "url": self.config.fqdn, + "scope": "user", + "theme": "dark", + "external": "true", + }, + ] + }, + "servicename": "portal", + "instance": "main", + "daemonpath": "/usr/local/bin/jspython", + "args": "portal_start.py main", + "pwd": "/opt/jumpscale7/apps/portals/main/", + } + self.grid = { + "config":{ + 'id': self.config.grid_id + }, + "servicename": "system", + "instance": "grid" + } + self.update_config(self.grid) + self.update_config(self.mongodb_client) + # update openvcloud config + openvcloud_config = j.core.config.get("openvcloud", "main") + openvcloud_config["portalurl"] = self.config.fqdn + openvcloud_config["supportemail"] = j.core.config.get("mailclient", "main")["sender"] + j.core.config.set("openvcloud", "main", openvcloud_config) + + + def install_packages(self): + super(MasterNode, self).install_packages() + j.do.execute( + "apt-get install -y graphviz mongodb libmhash2" + ) + def install_portal(self): + import os + args = "--fqdn={} --password={} --env_subdomain={} --client_id={} --secret={} --ovs_url={}".format( + self.config.fqdn, + self.config.password, + self.config.env_subdomain, + self.config.client_id, + self.config.secret, + self.config.ovs_url + ) + os.system('jspython /opt/code/github/0-complexity/openvcloud_installer/scripts/dockers/cb_master/configure-portal-local.py {}'.format(args)) + self.register_service(self.portal) + self.restart_service('openvcloud_portal') + + def install(self): + self.install_redis() + j.do.execute('systemctl restart nginx') + self.register_service(self.mongodb) + import time + time.sleep(40) + self.register_service(self.osis) + time.sleep(20) + self.install_osis_client() + self.register_service(self.controller) + self.install_agentcontroller_client() + time.sleep(20) + self.install_jsagent() + self.install_portal() + self.install_portal_client() + + + + if __name__ == "__main__": from argparse import ArgumentParser @@ -559,7 +740,6 @@ if __name__ == "__main__": help="master ips", ) parser.add_argument("-p", "--password", dest="password", help="Password") - parser.add_argument("-gid", "--gid", dest="gid", help="GID", required=True) parser.add_argument("-fqdn", "--fqdn", dest="fqdn", required=True, help="FQDN") parser.add_argument( "-vxbackend_vlan", @@ -574,7 +754,7 @@ if __name__ == "__main__": "-gwmgmt_vlan", "--gwmgmt_vlan", dest="gwmgmt_vlan", help="gwmgmt_vlan" ) parser.add_argument("-gwmgmt_ip", "--gwmgmt_ip", dest="gwmgmt_ip", help="gwmgmt_ip") - parser.add_argument("-grid_id", "--grid_id", dest="grid_id", help="grid_id") + parser.add_argument("-grid_id", "--grid_id", dest="grid_id", help="grid_id", type=int) parser.add_argument( "-env_subdomain", "--env_subdomain", dest="env_subdomain", help="env_subdomain" ) @@ -584,8 +764,10 @@ if __name__ == "__main__": parser.add_argument( "-secret", "--secret", dest="secret", help="itsyouonline Secret" ) + parser.add_argument( + "-ovs_url", "--ovs_url", dest="ovs_url", help="ovs_url required for local dev installation" + ) parser.add_argument("--update", help="Update", action="store_true") - config = parser.parse_args() if "cpu" in config.roles and "storage" in config.roles: node = ConvergedNode(config) @@ -593,6 +775,8 @@ if __name__ == "__main__": node = ComputeNode(config) elif "storage" in config.roles: node = StorageNode(config) + elif "master" in config.roles: + node = MasterNode(config) else: extraroles = [] if "controller" in config.roles: From 1944c627e3a3bb84347a2be2b329283424c76272 Mon Sep 17 00:00:00 2001 From: Jo De Boeck Date: Wed, 24 Oct 2018 08:54:37 +0200 Subject: [PATCH 3/5] Pass -gid only once fix dep-manifest link mess Signed-off-by: Jo De Boeck --- scripts/dockers/cb_master/dep-manifest.yml | 4 ++-- scripts/install/installer | 4 ++-- scripts/install/installnode | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/dockers/cb_master/dep-manifest.yml b/scripts/dockers/cb_master/dep-manifest.yml index fca4302..d908997 100644 --- a/scripts/dockers/cb_master/dep-manifest.yml +++ b/scripts/dockers/cb_master/dep-manifest.yml @@ -27,10 +27,10 @@ https://github.com/jumpscale7/jumpscale_core7: source: apps/agentcontroller/processmanager/ - dest: /opt/jumpscale7/apps/agentcontroller/luajumpscripts/jumpscale/ source: apps/agentcontroller/luajumpscripts/jumpscale/ -- dest: /opt/jumpscale7/apps/osis/ - source: apps/osis/ - dest: /opt/jumpscale7/apps/osis/logic/ source: apps/osis/logic/ +- dest: /opt/jumpscale7/apps/osis/osisServerStart.py + source: apps/osis/osisServerStart.py - dest: /opt/jumpscale7/apps/jsagent/ source: apps/jsagent/ - dest: /opt/jumpscale7/bin/ diff --git a/scripts/install/installer b/scripts/install/installer index 8c10bca..e25d7bf 100755 --- a/scripts/install/installer +++ b/scripts/install/installer @@ -688,9 +688,9 @@ class JumpScale7(Node): "envsec": self.config["itsyouonline"]["clientSecret"], } self.remote.run( - """/opt/jumpscale7/bin/installnode {} --roles={} --masterips {} --password={} --gid={} --fqdn={} \ + """/opt/jumpscale7/bin/installnode {} --roles={} --masterips {} --password={} -gid={} --fqdn={} \ --vxbackend_vlan={vxvlan} --vxbackend_ip={vxip} --gwmgmt_vlan={gwvlan} --gwmgmt_ip={gwip} \ - --grid_id={gridid} --env_subdomain={envsub} --client_id={envcli} --secret={envsec}""".format( + --env_subdomain={envsub} --client_id={envcli} --secret={envsec}""".format( extra_flags, " ".join(self.roles), " ".join(self.backplaneips), diff --git a/scripts/install/installnode b/scripts/install/installnode index 48d8f66..3a4f5de 100755 --- a/scripts/install/installnode +++ b/scripts/install/installnode @@ -759,7 +759,7 @@ if __name__ == "__main__": "-gwmgmt_vlan", "--gwmgmt_vlan", dest="gwmgmt_vlan", help="gwmgmt_vlan" ) parser.add_argument("-gwmgmt_ip", "--gwmgmt_ip", dest="gwmgmt_ip", help="gwmgmt_ip") - parser.add_argument("-grid_id", "--grid_id", dest="grid_id", help="grid_id", type=int) + parser.add_argument("-gid", "--grid_id", dest="grid_id", help="grid_id", type=int) parser.add_argument( "-env_subdomain", "--env_subdomain", dest="env_subdomain", help="env_subdomain" ) From 00bfb051af14b2091b92398d3cbacf7245776879 Mon Sep 17 00:00:00 2001 From: Ashraf Fouda Date: Wed, 24 Oct 2018 11:51:38 +0200 Subject: [PATCH 4/5] fix packager Signed-off-by: Ashraf Fouda --- scripts/buildlib/packager.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/scripts/buildlib/packager.py b/scripts/buildlib/packager.py index bd8732c..5e7ac29 100644 --- a/scripts/buildlib/packager.py +++ b/scripts/buildlib/packager.py @@ -74,13 +74,6 @@ def check_link(source, destination): for filepath in os.listdir(src): srcpath = os.path.join(src, filepath) dstpath = os.path.join(dest, filepath) - if os.path.exists(dstpath): - if os.path.isdir(dstpath): - self._create_link(srcpath, dstpath) - continue - else: - continue - if check_link(srcpath, dstpath): continue os.symlink(srcpath, dstpath.rstrip('/')) From 8a63f99d47462313cb4af36ac163aed28ad0eeca Mon Sep 17 00:00:00 2001 From: Ashraf Fouda Date: Thu, 25 Oct 2018 09:04:05 +0200 Subject: [PATCH 5/5] Adds fix gid Signed-off-by: Ashraf Fouda --- scripts/buildlib/packager.py | 1 - scripts/dockers/base/dep-manifest.yml | 2 ++ scripts/install/installnode | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/buildlib/packager.py b/scripts/buildlib/packager.py index 5e7ac29..70a764f 100644 --- a/scripts/buildlib/packager.py +++ b/scripts/buildlib/packager.py @@ -69,7 +69,6 @@ def check_link(source, destination): elif os.path.isfile(destination): os.remove(destination) return False - if os.path.isdir(src): for filepath in os.listdir(src): srcpath = os.path.join(src, filepath) diff --git a/scripts/dockers/base/dep-manifest.yml b/scripts/dockers/base/dep-manifest.yml index 723c2b5..51f1b35 100644 --- a/scripts/dockers/base/dep-manifest.yml +++ b/scripts/dockers/base/dep-manifest.yml @@ -48,6 +48,8 @@ https://github.com/0-complexity/openvcloud: source: apps/libvirtlistener/ - dest: /opt/jumpscale7/apps/vncproxy/utils/ source: apps/vncproxy/utils/ +- dest: /opt/jumpscale7/bin + source: scripts/ovs/ git@git.gig.tech:openvcloud/binary/routeros: - dest: /var/lib/libvirt/images/routeros/template/routeros.qcow2 source: root/routeros-small-NETWORK-ID.qcow2 diff --git a/scripts/install/installnode b/scripts/install/installnode index eb9b4f4..efc5520 100755 --- a/scripts/install/installnode +++ b/scripts/install/installnode @@ -754,7 +754,7 @@ if __name__ == "__main__": "-gwmgmt_vlan", "--gwmgmt_vlan", dest="gwmgmt_vlan", help="gwmgmt_vlan" ) parser.add_argument("-gwmgmt_ip", "--gwmgmt_ip", dest="gwmgmt_ip", help="gwmgmt_ip") - parser.add_argument("-gid", "--grid_id", dest="grid_id", help="grid_id", type=int) + parser.add_argument("-gid", "--gid", dest="grid_id", help="grid_id", type=int) parser.add_argument( "-env_subdomain", "--env_subdomain", dest="env_subdomain", help="env_subdomain" )