#!/usr/bin/env python3 import argparse import sys import time import shlex import subprocess # import psutil import logging from os import path from ekos_cli import EkosDbus DESCRIPTION = ("\n" "Code from FromEKOS Sentinel, version 1.0 by Haans.\n" "This variation is intended to be run from the scheduler. It is not concerned with restarting a session,\n" "only with double checking the roof gets closed if the weather is bad.\n" "\n") RUNF = '/tmp/indi_mon_shed_run.tmp' # Exists while scheduler is running # indi_getprop | sort -u | grep WEATHER ####################################################################################################################### # Properties ####################################################################################################################### INDI_DEFAULT_HOST = 'localhost' # Properties in common to Deploy and Simulator definitions INDI_WEATHER_PROPERTY = 'Weather Meta.WEATHER_STATUS.STATION_STATUS' INDI_WEATHER_PROPERTY_OK_SETTING = 'Ok' INDI_MOUNT_PARK_PROPERTY_PARK_SETTING = 'On' INDI_MOUNT_TRACK_STATE_PROPERTY = 'Off' INDI_MOUNT_GEOGRAPHIC_COORD_LAT_PROPERTY = '43' INDI_MOUNT_GEOGRAPHIC_COORD_LAT_MAX_OFFSET = '1' INDI_DOME_PARK_PROPERTY_PARK_SETTING = 'On' INDI_DOME_PARKED_PROPERTY_SETTING = 'Ok' # Deployment INDI_WEATHER_META_STATION_INDEXES = {1, 2} INDI_MOUNT_PARK_PROPERTY = 'AstroPhysics Experimental.TELESCOPE_PARK.PARK' INDI_MOUNT_TRACK_STATE = 'AstroPhysics Experimental.TELESCOPE_TRACK_STATE.TRACK_ON' INDI_MOUNT_GEOGRAPHIC_COORD_LAT = 'AstroPhysics Experimental.GEOGRAPHIC_COORD.LAT' INDI_CAP = None # Correct these two INDI_CAP_PROPERTY = None INDI_DOME_PARK_PROPERTY = 'RollOff ino.DOME_PARK.PARK' INDI_DOME_PARKED_PROPERTY = 'RollOff ino.Roof Status.Closed' INDI_CAMERA_COOLER = 'ZWO CCD ASI1600MM Pro.CCD_COOLER.COOLER_OFF' # Check this one INDI_CAMERA_COOLER_PROPERTY = 'On' """ # Simulator INDI_WEATHER_META_STATION_INDEXES = {1} INDI_MOUNT_PARK_PROPERTY = 'Telescope Simulator.TELESCOPE_PARK.PARK' INDI_MOUNT_TRACK_STATE = 'Telescope Simulator.TELESCOPE_TRACK_STATE.TRACK_ON' INDI_MOUNT_GEOGRAPHIC_COORD_LAT = 'Telescope Simulator.GEOGRAPHIC_COORD.LAT' INDI_CAP = None INDI_CAP_PROPERTY = None INDI_DOME_PARK_PROPERTY = 'RollOff Simulator.DOME_PARK.PARK' INDI_DOME_PARKED_PROPERTY = 'RollOff Simulator.Roof Status.Closed' INDI_CAMERA_COOLER = 0 INDI_CAMERA_COOLER_PROPERTY = 'None' """ MAIN_LOOP_SLEEP_SECONDS = 30 INDI_COMMAND_TIMEOUT = 30 MOUNT_PARK_TIMEOUT = 30 ROOF_CLOSE_TIMEOUT = 30 CAP_CLOSE_TIMEOUT = 15 RET_SUCCESS = 1 # Observatory safe & monitor about to be shutdown RET_WEATHER_UNSAFE = 2 # Roof already closed, unsafe weather RET_FORCED_SHUTDOWN = 3 # Roof closed by this procedure RET_NO_PROPERTY = 4 # Unable to access property values RET_UNSAFE_MOUNT = 5 # Weather bad and mount is unparked RET_UNSAFE_CAP = 6 # Weather bad and cap unparkd RET_UNSAFE_ROOF = 7 # Weather bad and roof is unparked ###################################################################################################################### # Access Properties ###################################################################################################################### class BasicIndi: def __init__(self, host): self.host = host self.logger = logging.getLogger('monitor') self.max_retries = 1 def get_max_retries(self): return self.max_retries def set_max_retries(self, retries): self.max_retries = retries def _run(self, cmd, timeout): try: ws = subprocess.run(shlex.split(cmd), stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, timeout=timeout, check=True ) except subprocess.CalledProcessError as cpe: self.logger.critical( "Command [{}] exited with value [{}] stdout [{}] stderr [{}]".format(cmd, cpe.returncode, cpe.stdout.rstrip(), cpe.stderr.rstrip())) return None except subprocess.TimeoutExpired: self.logger.critical("Command [{}] timed out after {} seconds".format(cmd, timeout)) return None return ws def test_indi_getprop(self, indi_command_timeout): # See if indi_getprop working, use INDI_MOUNT_PARK_PROPERTY to test cmd = "indi_getprop -h {host} -1 '{property}'".format(host=self.host, property=INDI_MOUNT_PARK_PROPERTY) ws = self._run(cmd, indi_command_timeout) if not ws: return False else: return True def get_weather_safety(self, weather_meta_station_indexes, indi_command_timeout): safe = 0 for station in weather_meta_station_indexes: cmd = "indi_getprop -h {host} -1 '{property}_{station}'".format( host=self.host, property=INDI_WEATHER_PROPERTY, station=station) ws = self._run(cmd, indi_command_timeout) if not ws: self.logger.critical("get_weather_safety failed") return False self.logger.debug("{} {} {}".format(__class__, cmd, ws.stdout.rstrip())) if ws.stdout.rstrip() == INDI_WEATHER_PROPERTY_OK_SETTING: safe += 1 if safe == len(weather_meta_station_indexes): return True else: return False def get_roof_safety(self, indi_command_timeout): cmd = "indi_getprop -h {host} -1 '{property}'".format(host=self.host, property=INDI_DOME_PARK_PROPERTY) ws = self._run(cmd, indi_command_timeout) if not ws: self.logger.critical("get_roof_safety failed") return False self.logger.debug("{} {} {}".format(__class__, cmd, ws.stdout.rstrip())) if ws.stdout.rstrip() == INDI_DOME_PARK_PROPERTY_PARK_SETTING: return True else: return False def get_cap_safety(self, indi_command_timeout): if not INDI_CAP: self.logger.debug("get_cap_safety fakes True because INDI_CAP is not set") return True cmd = "indi_getprop -h {host} -1 '{property}'".format(host=self.host, property=INDI_CAP) ws = self._run(cmd, indi_command_timeout) if not ws: self.logger.critical("get_cap_safety failed") return False self.logger.debug("{} {} {}".format(__class__, cmd, ws.stdout.rstrip())) if ws.stdout.rstrip() == INDI_CAP_PROPERTY: return True else: return False def get_mount_safety(self, indi_command_timeout): # multiple steps # 1) INDI_MOUNT_PARK_PROPERTY must be INDI_MOUNT_PARK_PROPERTY_PARK_SETTING cmd = "indi_getprop -h {host} -1 '{property}'".format(host=self.host, property=INDI_MOUNT_PARK_PROPERTY) ws = self._run(cmd, indi_command_timeout) if not ws: self.logger.critical("get_mount_safety failed at step 1") return False self.logger.debug("{} {} {}".format(__class__, cmd, ws.stdout.rstrip())) # self.logger.debug("Comparing {} to {}".format(ws.stdout.rstrip(), INDI_MOUNT_PARK_PROPERTY_PARK_SETTING)) if ws.stdout.rstrip() != INDI_MOUNT_PARK_PROPERTY_PARK_SETTING: self.logger.debug("Mount not parked") return False # 2) INDI_MOUNT_TRACK_STATE must be INDI_MOUNT_TRACK_STATE_PROPERTY cmd = "indi_getprop -h {host} -1 '{property}'".format(host=self.host, property=INDI_MOUNT_TRACK_STATE) ws = self._run(cmd, indi_command_timeout) if not ws: self.logger.critical("get_mount_safety failed at step 2") return False self.logger.debug("{} {} {}".format(__class__, cmd, ws.stdout.rstrip())) if ws.stdout.rstrip() != INDI_MOUNT_TRACK_STATE_PROPERTY: self.logger.debug("Mount still tracking") return False # 3) INDI_MOUNT_GEOGRAPHIC_COORD_LAT needs to be within INDI_MOUNT_GEOGRAPHIC_COORD_LAT_MAX_OFFSET # to INDI_MOUNT_GEOGRAPHIC_COORD_LAT_PROPERTY (park position) cmd = "indi_getprop -h {host} -1 '{property}'".format(host=self.host, property=INDI_MOUNT_GEOGRAPHIC_COORD_LAT) ws = self._run(cmd, indi_command_timeout) if not ws: self.logger.critical("get_mount_safety failed at step 3") return False self.logger.debug("{} {} {}".format(__class__, cmd, ws.stdout.rstrip())) # self.logger.debug("Check {} - {} > {}".format(abs(float(ws.stdout.rstrip())), abs(float(INDI_MOUNT_GEOGRAPHIC_COORD_LAT_PROPERTY)), abs(float(INDI_MOUNT_GEOGRAPHIC_COORD_LAT_MAX_OFFSET)))) if abs(float(ws.stdout.rstrip())) - abs(float(INDI_MOUNT_GEOGRAPHIC_COORD_LAT_PROPERTY)) > float( INDI_MOUNT_GEOGRAPHIC_COORD_LAT_MAX_OFFSET): # self.logger.debug("Mount not in park position") return False # self.logger.debug("Mount is in park position") return True def park_mount(self, indi_command_timeout, mount_park_timeout): tries = 0 mount_parked = False while not mount_parked and tries < self.max_retries: mount_parked = self._park_mount(indi_command_timeout, mount_park_timeout) tries += 1 return mount_parked def _park_mount(self, indi_command_timeout, mount_park_timeout): cmd = "indi_setprop -h {host} '{property}={setting}'".format(host=self.host, property=INDI_MOUNT_PARK_PROPERTY, setting=INDI_MOUNT_PARK_PROPERTY_PARK_SETTING) ws = self._run(cmd, indi_command_timeout) if not ws: self.logger.critical("park_mount failed") return False self.logger.debug("{} {} {}".format(__class__, cmd, ws.stdout.rstrip())) if ws.returncode != 0: return False mount_parked = False mount_park_time = 0 while not mount_parked and mount_park_time < mount_park_timeout: mount_parked = self.get_mount_safety(indi_command_timeout) time.sleep(1) mount_park_time += 1 return mount_parked def park_cap(self, indi_command_timeout, cap_close_timeout): if not INDI_CAP: self.logger.debug("park_cap fakes True because INDI_CAP is not set") return True tries = 0 cap_closed = False while not cap_closed and tries < self.max_retries: cap_closed = self._park_cap(indi_command_timeout, cap_close_timeout) tries += 1 return cap_closed def _park_cap(self, indi_command_timeout, cap_close_timeout): cmd = "indi_setprop -h {host} '{property}={setting}'".format(host=self.host, property=INDI_CAP, setting=INDI_CAP_PROPERTY) ws = self._run(cmd, indi_command_timeout) if not ws: self.logger.critical("park_cap failed") return False self.logger.debug("{} {} {}".format(__class__, cmd, ws.stdout.rstrip())) if ws.returncode != 0: return False cap_closed = False cap_close_time = 0 while not cap_closed and cap_close_time < cap_close_timeout: cap_closed = self.get_cap_safety(indi_command_timeout=indi_command_timeout) time.sleep(1) cap_close_time += 1 return cap_closed def park_roof(self, indi_command_timeout, roof_close_timeout): tries = 0 roof_closed = False while not roof_closed and tries < self.max_retries: roof_closed = self._park_roof(indi_command_timeout, roof_close_timeout) tries += 1 return roof_closed # Park Roof def _park_roof(self, indi_command_timeout, roof_close_timeout): cmd = "indi_setprop -h {host} '{property}={setting}'".format(host=self.host, property=INDI_DOME_PARK_PROPERTY, setting=INDI_DOME_PARK_PROPERTY_PARK_SETTING) self.logger.debug("Command being issued: {}".format(cmd)) ws = self._run(cmd, indi_command_timeout) if not ws: self.logger.critical("park_roof failed") return False self.logger.debug("{} {} {}".format(__class__, cmd, ws.stdout.rstrip())) if ws.returncode != 0: return False roof_closed = False roof_close_time = 0 while not roof_closed and roof_close_time < roof_close_timeout: roof_closed = self.get_roof_safety(indi_command_timeout=indi_command_timeout) time.sleep(1) roof_close_time += 1 return roof_closed def warm_camera(self, indi_command_timeout): if not INDI_CAMERA_COOLER: self.logger.debug("warm_camera fakes True because INDI_CAMERA_COOLER is not set") return True tries = 0 camera_warmed = False while not camera_warmed and tries < self.max_retries: camera_warmed = self._warm_camera(indi_command_timeout) tries += 1 return camera_warmed def _warm_camera(self, indi_command_timeout): cmd = "indi_setprop -h {host} '{property}={setting}'".format(host=self.host, property=INDI_CAMERA_COOLER, setting=INDI_CAMERA_COOLER_PROPERTY) ws = self._run(cmd, indi_command_timeout) if not ws: self.logger.critical("warm_camera failed") return False self.logger.debug("{} {} {}".format(__class__, cmd, ws.stdout.rstrip())) return ws.returncode == 0 def alert_and_abort(reason): logger = logging.getLogger('monitor') logger.critical("TODO wake human stating {}".format(reason)) sys.exit(1) ###################################################################################################################### # Polling ###################################################################################################################### def poll(basic_indi, logger, once): status = RET_SUCCESS ekos_dbus = EkosDbus() prev_weather = False prev_roof = False looping = True while looping: if once: looping = False if not basic_indi.test_indi_getprop(indi_command_timeout=INDI_COMMAND_TIMEOUT): status = RET_NO_PROPERTY break weather_safety_status = basic_indi.get_weather_safety( weather_meta_station_indexes=INDI_WEATHER_META_STATION_INDEXES, indi_command_timeout=INDI_COMMAND_TIMEOUT) roof_safety_status = basic_indi.get_roof_safety(indi_command_timeout=INDI_COMMAND_TIMEOUT) mount_safety_status = basic_indi.get_mount_safety(indi_command_timeout=INDI_COMMAND_TIMEOUT) if weather_safety_status != prev_weather: if weather_safety_status: logger.info('Monitoring, weather is safe') else: logger.info('Monitoring, weather is unsafe') if roof_safety_status != prev_roof: if roof_safety_status: logger.info('Monitoring, roof is closed') else: logger.info('Monitoring, roof is open') if weather_safety_status: status = RET_SUCCESS else: ekos_dbus.stop_scheduler() if roof_safety_status: if weather_safety_status != prev_weather or roof_safety_status != prev_roof: logger.info('weather is unsafe, roof is closed, continuing') status = RET_WEATHER_UNSAFE # break else: # Weather is unsafe and roof is open logger.info('weather is unsafe, roof is open') # there's no way to tell yet if stopping the ekos scheduler succeeded or not # furthermore stopping the scheduler does not park or close anything, so that is done here : if not mount_safety_status: logger.warning('park mount') success = basic_indi.park_mount(indi_command_timeout=INDI_COMMAND_TIMEOUT, mount_park_timeout=MOUNT_PARK_TIMEOUT) if not success: # alert_and_abort("Failed to park the mount, tried {} times".format(self.basic_indi.get_max_retries())) logger.critical("Failed to park the mount, tried {} times".format(basic_indi.get_max_retries())) status = RET_UNSAFE_MOUNT break logger.warning('park cap') success = basic_indi.park_cap(indi_command_timeout=INDI_COMMAND_TIMEOUT, cap_close_timeout=CAP_CLOSE_TIMEOUT) if not success: # alert_and_abort("Failed to close the cap, tried {} times".format(self.basic_indi.get_max_retries())) logger.critical("Failed to close the cap, tried {} times".format(basic_indi.get_max_retries())) status = RET_UNSAFE_CAP break logger.warning('park roof') success = basic_indi.park_roof(indi_command_timeout=INDI_COMMAND_TIMEOUT, roof_close_timeout=ROOF_CLOSE_TIMEOUT) if not success: logger.critical("Failed to close the roof, tried {} times".format(basic_indi.get_max_retries())) status = RET_UNSAFE_ROOF break logger.warning('warm camera') success = basic_indi.warm_camera(indi_command_timeout=INDI_COMMAND_TIMEOUT) if not success: logger.warning('failed to warm the camera, this is not critical to safety') # To get here the weather is unsafe, the mount and roof have been parked status = RET_FORCED_SHUTDOWN break # End of unsafe weather, roof safety check # End of weather safety check # Test to see if the scheduler has called the termination procedure indicating a normal shutdown of the scheduler if not path.exists(RUNF): break if looping: prev_weather = weather_safety_status prev_roof = roof_safety_status logger.debug("sleep {}".format(MAIN_LOOP_SLEEP_SECONDS)) time.sleep(MAIN_LOOP_SLEEP_SECONDS) # Loop has exited return status ###################################################################################################################### # Parse ###################################################################################################################### def main(): # parser = argparse.ArgumentParser(description=DESCRIPTION, # epilog='Only --indi-host is required for normal operation', # formatter_class=argparse.RawDescriptionHelpFormatter) # parser.add_argument('--indi_host', required=True, type=str, help='INDI server address') parser = argparse.ArgumentParser(description=DESCRIPTION, epilog='', formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument('--indi_host', type=str, help='INDI server address') parser.add_argument('--indi_command_retries', type=str, help='try INDI commands this number of times, defaults to 1') parser.add_argument('--debug', action='store_true', help='enable debug level verbosity') parser.add_argument('--once', action='store_true', help='run only once, useful for debugging') parser.add_argument('--get_weather_safety', action='store_true', help='for testing: only call get_weather_safety') parser.add_argument('--get_mount_safety', action='store_true', help='for testing: only call get_mount_safety') parser.add_argument('--get_cap_safety', action='store_true', help='for testing: only call get_cap_safety') parser.add_argument('--get_roof_safety', action='store_true', help='for testing: only call get_roof_safety') parser.add_argument('--park_mount', action='store_true', help='for testing: only call park_mount') parser.add_argument('--park_cap', action='store_true', help='for testing: only call park_cap') parser.add_argument('--park_roof', action='store_true', help='for testing: only call park_roof') parser.add_argument('--warm_camera', action='store_true', help='for testing: only call warm_camera') args = parser.parse_args() ################################################################################################################## # Init ################################################################################################################## logger = logging.getLogger('monitor') if args.debug: logger.setLevel(logging.DEBUG) else: logger.setLevel(logging.INFO) null_handler = logging.NullHandler() logger.addHandler(null_handler) console_handler = logging.StreamHandler(sys.stdout) console_format = logging.Formatter("%(asctime)s %(name)s %(levelname)-8s %(message)s") console_handler.setFormatter(console_format) logger.addHandler(console_handler) # Instantiate the BasicIndi class if args.indi_host: basic_indi = BasicIndi(host=args.indi_host) else: basic_indi = BasicIndi(host=INDI_DEFAULT_HOST) ################################################################################################################## # Debug switches ################################################################################################################## if args.indi_command_retries: basic_indi.set_max_retries(args.indi_command_retries) if args.get_weather_safety: logger.info("get_weather_safety = [{}]".format( basic_indi.get_weather_safety(weather_meta_station_indexes=INDI_WEATHER_META_STATION_INDEXES, indi_command_timeout=INDI_COMMAND_TIMEOUT))) quit(0) if args.get_mount_safety: logger.info( "get_mount_safety = [{}]".format(basic_indi.get_mount_safety(indi_command_timeout=INDI_COMMAND_TIMEOUT))) quit(0) if args.get_roof_safety: logger.info( "get_roof_safety = [{}]".format(basic_indi.get_roof_safety(indi_command_timeout=INDI_COMMAND_TIMEOUT))) quit(0) if args.get_cap_safety: logger.info( "get_cap_safety = [{}]".format(basic_indi.get_cap_safety(indi_command_timeout=INDI_COMMAND_TIMEOUT))) quit(0) if args.park_mount: logger.info( "park_mount = [{}]".format(basic_indi.park_mount(indi_command_timeout=INDI_COMMAND_TIMEOUT, mount_park_timeout=MOUNT_PARK_TIMEOUT))) quit(0) if args.park_cap: logger.info( "park_cap = [{}]".format(basic_indi.park_cap(indi_command_timeout=INDI_COMMAND_TIMEOUT, cap_close_timeout=CAP_CLOSE_TIMEOUT))) quit(0) if args.park_roof: logger.info( "park_roof = [{}]".format(basic_indi.park_roof(indi_command_timeout=INDI_COMMAND_TIMEOUT, roof_close_timeout=ROOF_CLOSE_TIMEOUT))) quit(0) if args.warm_camera: logger.info( "warm_camera = [{}]".format(basic_indi.warm_camera(indi_command_timeout=INDI_COMMAND_TIMEOUT))) quit(0) ################################################################################################################## # Monitor ################################################################################################################## # Normally run from the scheduler's startup procedure, it is called after the cap, telescope and roof have been unparked. # If Ekos/imaging is not being accessed via the scheduler, assume it and the weather is being monitored manually # and rely on the Observatory module for shutdown. # If all is well when the scheduler shutdown procedure runs, the cap, telescope and roof will have been parked. # So while looping Ekos and indiserver should always be available, the cap, telescope and roof should change from # unparked to parked. The scheduler's preprocess and postprocess procedures will create and delete a file to indicate # whether this monitor should continue to run. status = poll(basic_indi, logger, args.once) if status == RET_SUCCESS: logger.info('\nNormal termination with no action taken') quit(0) # Might be due to KStars/Ekos crash/hang, see if can recover if status == RET_NO_PROPERTY: logger.info('\nUnable to access Ekos property values. Ensure Ekos/indiserver are working. Weather conditions not known') quit(1) # Weather unsafe, roof already closed. Need to prevent scheduler from reopening if status == RET_WEATHER_UNSAFE: logger.info('\nWeather conditions unsafe, roof was already closed') quit(0) if status == RET_FORCED_SHUTDOWN: logger.info('\nWeather conditions unsafe, forced shutdown ran, check that roof was closed') quit(0) # The following: Weather unsafe, unable to close roof. Get help. Can roof be forced to shut safely???? if status == RET_UNSAFE_MOUNT: logger.info('\nFailed to park the mount') alert_and_abort("Roof is open and the weather is unsafe") if status == RET_UNSAFE_CAP: logger.info('\nFailed to park the cap') alert_and_abort("Roof is open and the weather is unsafe") if status == RET_UNSAFE_ROOF: logger.info('\nFailed to park the roof') alert_and_abort("Roof is open and the weather is unsafe") if __name__ == "__main__": main()