Add tasklet and support for taking possession of an existing simulator

src/inworldz/maestro/RegionHost.py

@@ -9,6 +9,7 @@
 import os.path
 import threading
 import shlex
+import shutil
 import subprocess
 import psutil
 import Queue
@@ -19,6 +20,8 @@
 from inworldz.util.filesystem import ConnectUNCShare
 from inworldz.util.rdbhost import GetRdbHost, AssignBestRdbHost
 
+import inworldz.util.general
+
 import inworldz.maestro.uuid as genuuid
 import inworldz.maestro.MaestroStore as store
 
@@ -31,6 +34,7 @@
     GridService
 import inworldz.maestro.environment.ComputeResource as ComputeResource
 from inworldz.maestro.environment.RegionEntry import RegionState, RegionEntry
+from inworldz.maestro.Estate import Estate
 from inworldz.maestro import User
 
 class RegionHost(ServiceBase):
@@ -340,33 +344,39 @@ def ProvisionRegion(self, record):
             if (self.IsSlotFree(slotnum)):
                 record['slot_number'] = slotnum
 
-                try:
-                    region = Region.create(record)
-
-                    #also record this provisioning to the environment
-                    regEntry = RegionEntry(region.sim_uuid, region.sim_name, region.master_avatar_uuid, \
-                                           region.estate_id, region.get_region_product(), \
-                                           region.sim_location_x, region.sim_location_y, \
-                                           self.props.hostingResource.dbid, \
-                                           RegionState.SetupInProgress)
-
-                    self.props.hostingResource.registerNewRegion(regEntry)
-                    region.associateWithRegionEntry()
-
-                    self.region_add(region.get_uuid()) 
-
-                    return region.get_uuid()
-
-                except:
-                    import traceback
-                    exc_type, exc_value, exc_traceback = sys.exc_info()
-                    traceback.print_exception(exc_type, exc_value, exc_traceback)
-                    raise ServiceError(exc_value)     
+                regions = self.RecordSimulatorRegions([record])
+                return regions[0].get_uuid()
 
         """ No slots found available. tell them """
         raise ServiceError("No region slot is available")
 
-
+    def RecordSimulatorRegions(self, region_records):
+        regions = []
+        for record in region_records:
+            try:
+                region = Region.create(record)
+
+                #also record this provisioning to the environment
+                regEntry = RegionEntry(region.sim_uuid, region.sim_name, region.master_avatar_uuid, \
+                                        region.estate_id, region.get_region_product(), \
+                                        region.sim_location_x, region.sim_location_y, \
+                                        self.props.hostingResource.dbid, \
+                                        RegionState.SetupInProgress)
+
+                self.props.hostingResource.registerNewRegion(regEntry)
+                region.associateWithRegionEntry()
+
+                self.region_add(region.get_uuid()) 
+
+                regions.append(region)
+
+            except:
+                import traceback
+                exc_type, exc_value, exc_traceback = sys.exc_info()
+                traceback.print_exception(exc_type, exc_value, exc_traceback)
+                raise ServiceError(exc_value)
+        return regions
+
     def UpdateRegionToRevision(self, region_uuid, revision):
         """ Update the region to the revision specified. """
         region = store.get(region_uuid, Region.getClass())
@@ -596,4 +606,56 @@ def RunCommandAsEx(self, username, password, cwd, cmd, cmdargs):
             return None   
 
     def RunCommandAs(self, username, password, cwd, cmd, cmdargs):
-        self.RunCommandAsEx(username, password, cwd, cmd, cmdargs)  
+        self.RunCommandAsEx(username, password, cwd, cmd, cmdargs)
+
+
+    def ShutdownUncontrolledSimulatorByPath(self, path_to_bin):
+        """
+        Tells the simulator to die, based on where in the filesystem it is run from.
+        DO NOT USE for Maestro-controlled regions.
+        """
+        p = provision._findRegionProcessByBin(path_to_bin)
+        if p == None:
+            return True
+        p.terminate() # This sends CTRL-C, which will cause an immediate clean shutdown, which is perfect.
+
+        if inworldz.util.process.WaitForProcessTermination(p, 30):
+            return True
+        else:
+            return False
+
+    def TakePossessionOfSimulator(self, path_to_bin):
+        """
+        Copies the bin folder to a Maestro slot, getting region data from config files located in the bin folder.
+        Also records each region in the simulator in Maestro.
+        """
+        # Read region configs from bin/Regions/*.xml
+        region_records = provision.ReadRegionConfigs(path_to_bin)
+
+        for record in region_records:
+            estate_id = Estate.FindEstateIDForRegion(record["sim_uuid"])
+            if not estate_id:
+                continue
+            record["estate_id"] = estate_id
+
+            slot_found = False
+
+            for slotnum in range(self.maxRegionSlots):
+                if (self.IsSlotFree(slotnum)):
+                    slotDir = provision.GetSlotDirectory(slotnum)
+                    try:
+                        shutil.copytree(path_to_bin, slotDir)
+                    except:
+                        continue
+                    record['slot_number'] = slotnum
+                    slot_found = True
+                    break
+
+        if not slot_found:
+            return None
+
+        active_region_records = [record for record in region_records if "estate_id" in record and "slot_number" in record]
+
+        return self.RecordSimulatorRegions(active_region_records)
+
+

src/inworldz/util/provision.py

@@ -70,25 +70,39 @@ def GetSlotConfiguration(slotnum):
     """ Return all of the configurations we find in this slot.  Can be more 
         than one.  We use this on startup to build the regions list. """
     slotDirectory  = GetSlotDirectory(slotnum)
-    regionDir = os.path.join(slotDirectory, "bin", "Regions")
+    regionBin = os.path.join(slotDirectory, "bin")
+    return ReadRegionConfigs(regionBin)
+
+def ReadRegionConfigs(path_to_bin):
+    """ Return all of the configurations we find in this bin folder.  Can be more 
+        than one.  We use this on startup to build the regions list. """
+    regionDir = os.path.join(path_to_bin, "Regions")
     configFiles = glob.glob(os.path.join(regionDir,"*.xml"))
     result = {}
     for config in configFiles:
         if (os.access(config, os.O_RDONLY) == False):
             continue
-        tree = ET.parse(config)
-        root = tree.getroot()
-        record = {}
-        for child in root.iter('Config'):
-            for entry in child.attrib:
-                record[str.lower(entry)] = child.attrib[entry]
+        record = ReadRegionConfig(config)
         sim_uuid = record['sim_uuid']
         result[sim_uuid] = record
     return result
 
+def ReadRegionConfig(path_to_xml):
+    if (os.access(path_to_xml, os.O_RDONLY) == False):
+        return None
+    tree = ET.parse(path_to_xml)
+    root = tree.getroot()
+    record = {}
+    for child in root.iter('Config'):
+        for entry in child.attrib:
+            record[str.lower(entry)] = child.attrib[entry]
+    return record
 
 def _findRegionProcess(slotnum):
     bindir = os.path.join(GetSlotDirectory(slotnum), "bin")
+    return _findRegionProcessByBin(bindir)
+
+def _findRegionProcessByBin(bindir):
     for p in psutil.process_iter():
         if (len(p.cmdline()) <= 0):
             continue

taskmanager/tasklets/TakeOverExternallyControlledRegionTasklet.py

@@ -0,0 +1,47 @@
+'''
+Created on Jan 14, 2014
+
+@author: David Daeschler
+'''
+
+from tasklets.TaskletBase import TaskletBase
+from inworldz.maestro.environment.RegionEntry import RegionState
+
+import time
+
+class TakeOverExternallyControlledSimulatorTasklet(TaskletBase):
+    '''
+    Tasklet for taking a simulator away from the control system currently running it, moving it to Maestro, and optionally starting it back up again.
+    Procedure:
+    1. Using your current grid manager send alerts to all regions in the VM about a pending restart.
+    2. Wait for the time you specified in the alert.
+    3. Disable the existing grid manager on that VM.
+    4. Execute this tasklet against every simulator "bin" folder path on that VM.
+
+    If the old simulator process fails to shut down, consequently preventing this tasklet from continuing, simply manually terminate the simulator and this tasklet will continue automatically.
+    '''
+
+    def execute(self):
+        # TakeOverExternallyControlledRegionTasklet
+        # {
+        #    "simulatorPath": "[path to simulator bin folder]",
+        #    "startRegions": true/false,
+        # }
+        host = self.session.api.RegionHost.get_all()[0]
+
+        if self.args['regionsimulatorPathPath'] == None:
+            raise Exception("simulatorPath not specified")
+        if self.args['simulatorPath'] == '':
+            raise Exception("simulatorPath was blank")
+
+        regions = self.session.api.RegionHost.TakePossessionOfRegion(host, self.args['simulatorPath'])
+
+        # Power off the simulator and wait for it to be finished.
+        while not self.session.api.RegionHost.ShutdownUncontrolledSimulatorByPath(self.args['simulatorPath']):
+            time.sleep(0.1)
+
+        if self.args['startRegions']:
+            for region in regions:
+                # Tell the new copy to power up.
+                self.session.api.Region.ChangeState(region, RegionState.DeployedStarting)
+                self.session.api.Region.Start(region)