From a0103b36f0c63058a6868ffb1d676fba434954a3 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 4 Oct 2019 19:07:50 +0200 Subject: [PATCH 1/4] created first version of rest api server based on simple http server --- pype/services/rest_api/__init__.py | 5 + pype/services/rest_api/rest_api.py | 237 +++++++++++++++++++++++++++++ 2 files changed, 242 insertions(+) create mode 100644 pype/services/rest_api/__init__.py create mode 100644 pype/services/rest_api/rest_api.py diff --git a/pype/services/rest_api/__init__.py b/pype/services/rest_api/__init__.py new file mode 100644 index 0000000000..c11ecfd761 --- /dev/null +++ b/pype/services/rest_api/__init__.py @@ -0,0 +1,5 @@ +from .rest_api import RestApiServer + + +def tray_init(tray_widget, main_widget): + return RestApiServer() diff --git a/pype/services/rest_api/rest_api.py b/pype/services/rest_api/rest_api.py new file mode 100644 index 0000000000..894ac8e986 --- /dev/null +++ b/pype/services/rest_api/rest_api.py @@ -0,0 +1,237 @@ +import os +import json +import enum +import collections +import threading +from inspect import signature +import socket +import http.server +from http import HTTPStatus +import socketserver + +from Qt import QtCore + +from pypeapp import config, Logger + +log = Logger().get_logger("RestApiServer") + + +class RestMethods(enum.Enum): + GET = "GET" + POST = "POST" + PUT = "PUT" + PATCH = "PATCH" + DELETE = "DELETE" + + def __repr__(self): + return str(self.value) + + def __eq__(self, other): + if isinstance(other, str): + return self.value == other + return self == other + + def __hash__(self): + return enum.Enum.__hash__(self) + + def __str__(self): + return str(self.value) + + +class Handler(http.server.SimpleHTTPRequestHandler): + + def do_GET(self): + self.process_request(RestMethods.GET) + + def do_POST(self): + """Common code for POST. + + This trigger callbacks on specific paths. + + If request contain data and callback func has arg data are sent to + callback too. + + Send back return values of callbacks. + """ + self.process_request(RestMethods.POST) + + def process_request(self, rest_method): + """Because processing is technically the same for now so it is used + the same way + """ + content_length = int(self.headers["Content-Length"]) + in_data_str = self.rfile.read(content_length) + in_data = None + if in_data_str: + in_data = json.loads(in_data_str) + + registered_callbacks = self.server.registered_callbacks[rest_method] + + path_items = [part.lower() for part in self.path.split("/") if part] + + results = [] + for check_path, callbacks in registered_callbacks.items(): + check_path_items = check_path.split("/") + if check_path_items == path_items: + log.debug( + "Triggering callbacks for path \"{}\"".format(check_path) + ) + for callback in callbacks: + try: + params = signature(callback).parameters + if len(params) > 0 and in_data: + result = callback(in_data) + else: + result = callback() + + if result: + results.append(result) + except Exception: + log.error( + "Callback on path \"{}\" failed".format(check_path), + exc_info=True + ) + + any_result = len(results) > 0 + self.send_response(HTTPStatus.OK) + if any_result: + self.send_header("Content-type", "application/json") + self.end_headers() + + if not any_result: + return + + if len(results) == 1: + json_message = str(results[0]) + else: + index = 1 + messages = {} + for result in results: + if isinstance(result, str): + value = result + else: + value = json.dumps(result) + messages["callback{}".format(str(index))] = value + + json_message = json.dumps(messages) + + self.wfile.write(json_message.encode()) + + +class AdditionalArgsTCPServer(socketserver.TCPServer): + def __init__(self, registered_callbacks, *args, **kwargs): + self.registered_callbacks = registered_callbacks + super(AdditionalArgsTCPServer, self).__init__(*args, **kwargs) + + +class RestApiServer(QtCore.QThread): + """ Listener for REST requests. + + It is possible to register callbacks for url paths. + Be careful about crossreferencing to different QThreads it is not allowed. + """ + + def __init__(self): + super(RestApiServer, self).__init__() + self.registered_callbacks = { + RestMethods.GET: collections.defaultdict(list), + RestMethods.POST: collections.defaultdict(list), + RestMethods.PUT: collections.defaultdict(list), + RestMethods.PATCH: collections.defaultdict(list), + RestMethods.DELETE: collections.defaultdict(list) + } + + self.qaction = None + self.failed_icon = None + self._is_running = False + try: + self.presets = config.get_presets().get( + "services", {}).get( + "rest_api", {} + ) + except Exception: + self.presets = {"default_port": 8011, "exclude_ports": []} + + self.port = self.find_port() + + def set_qaction(self, qaction, failed_icon): + self.qaction = qaction + self.failed_icon = failed_icon + + def register_callback(self, path, callback, rest_method=RestMethods.POST): + if isinstance(path, (list, set)): + path = "/".join([part.lower() for part in path]) + elif isinstance(path, str): + path = "/".join( + [part.lower() for part in str(path).split("/") if part] + ) + + if isinstance(rest_method, str): + rest_method = str(rest_method).upper() + + if path in self.registered_callbacks[rest_method]: + log.warning( + "Path \"{}\" has already registered callback.".format(path) + ) + else: + log.debug( + "Registering callback for path \"{}\"".format(path) + ) + self.registered_callbacks[rest_method][path].append(callback) + + def tray_start(self): + self.start() + + @property + def is_running(self): + return self._is_running + + def stop(self): + self._is_running = False + + def run(self): + self._is_running = True + if not self.registered_callbacks: + log.info("Any registered callbacks for Rest Api server.") + return + + try: + log.debug( + "Running Rest Api server on URL:" + " \"http://localhost:{}\"".format(self.port) + ) + with AdditionalArgsTCPServer( + self.registered_callbacks, + ("", self.port), + Handler + ) as httpd: + while self._is_running: + httpd.handle_request() + except Exception: + log.warning( + "Rest Api Server service has failed", exc_info=True + ) + self._is_running = False + if self.qaction and self.failed_icon: + self.qaction.setIcon(self.failed_icon) + + def find_port(self): + start_port = self.presets["default_port"] + exclude_ports = self.presets["exclude_ports"] + found_port = None + # port check takes time so it's lowered to 100 ports + for port in range(start_port, start_port+100): + if port in exclude_ports: + continue + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + result = sock.connect_ex(("localhost", port)) + if result != 0: + found_port = port + if found_port is not None: + break + if found_port is None: + return None + os.environ["PYPE_REST_API_URL"] = "http://localhost:{}".format( + found_port + ) + return found_port From 726633182fe44ae881e40dd818bcae529ff739e4 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 4 Oct 2019 19:08:10 +0200 Subject: [PATCH 2/4] added register of callback from rest api server to muster module --- pype/muster/muster.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pype/muster/muster.py b/pype/muster/muster.py index 28f1c2ddd1..a4805369aa 100644 --- a/pype/muster/muster.py +++ b/pype/muster/muster.py @@ -36,6 +36,12 @@ class MusterModule: # nothing to do pass + def process_modules(self, modules): + if "RestApiServer" in modules: + modules["RestApiServer"].register_callback( + "muster/show_login", self.show_login, "post" + ) + # Definition of Tray menu def tray_menu(self, parent): """ From 5f46bca934c35de0b69113a86a79d8275f905b97 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 7 Oct 2019 16:59:17 +0200 Subject: [PATCH 3/4] api server count on with invalid Content-Length --- pype/services/rest_api/rest_api.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pype/services/rest_api/rest_api.py b/pype/services/rest_api/rest_api.py index 894ac8e986..ef15238fb5 100644 --- a/pype/services/rest_api/rest_api.py +++ b/pype/services/rest_api/rest_api.py @@ -59,11 +59,13 @@ class Handler(http.server.SimpleHTTPRequestHandler): """Because processing is technically the same for now so it is used the same way """ - content_length = int(self.headers["Content-Length"]) - in_data_str = self.rfile.read(content_length) in_data = None - if in_data_str: - in_data = json.loads(in_data_str) + cont_len = self.headers.get("Content-Length") + if cont_len: + content_length = int(cont_len) + in_data_str = self.rfile.read(content_length) + if in_data_str: + in_data = json.loads(in_data_str) registered_callbacks = self.server.registered_callbacks[rest_method] From 262c186ba84449f08b4da49f579cd399ba838729 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 7 Oct 2019 19:21:16 +0200 Subject: [PATCH 4/4] only one callback can be registered for path and improved response messages --- pype/services/rest_api/rest_api.py | 135 +++++++++++++++++++---------- 1 file changed, 87 insertions(+), 48 deletions(-) diff --git a/pype/services/rest_api/rest_api.py b/pype/services/rest_api/rest_api.py index ef15238fb5..22823a5586 100644 --- a/pype/services/rest_api/rest_api.py +++ b/pype/services/rest_api/rest_api.py @@ -70,54 +70,90 @@ class Handler(http.server.SimpleHTTPRequestHandler): registered_callbacks = self.server.registered_callbacks[rest_method] path_items = [part.lower() for part in self.path.split("/") if part] + request_path = "/".join(path_items) + callback = registered_callbacks.get(request_path) + result = None + if callback: + log.debug( + "Triggering callbacks for path \"{}\"".format(request_path) + ) + try: + params = signature(callback).parameters + if len(params) > 0 and in_data: + result = callback(in_data) + else: + result = callback() - results = [] - for check_path, callbacks in registered_callbacks.items(): - check_path_items = check_path.split("/") - if check_path_items == path_items: - log.debug( - "Triggering callbacks for path \"{}\"".format(check_path) - ) - for callback in callbacks: - try: - params = signature(callback).parameters - if len(params) > 0 and in_data: - result = callback(in_data) - else: - result = callback() + self.send_response(HTTPStatus.OK) - if result: - results.append(result) - except Exception: - log.error( - "Callback on path \"{}\" failed".format(check_path), - exc_info=True + if result in [None, True] or isinstance(result, (str, int)): + message = str(result) + if result in [None, True]: + message = "{} request for \"{}\" passed".format( + rest_method, self.path ) - any_result = len(results) > 0 - self.send_response(HTTPStatus.OK) - if any_result: - self.send_header("Content-type", "application/json") + self.handle_result(rest_method, message=message) + + return + + if isinstance(result, (dict, list)): + message = json.dumps(result).encode() + self.handle_result(rest_method, final_output=message) + + return + + except Exception: + message = "{} request for \"{}\" failed".format( + rest_method, self.path + ) + log.error(message, exc_info=True) + + self.send_response(HTTPStatus.BAD_REQUEST) + self.handle_result(rest_method, message=message, success=False) + + return + + self.handle_result(rest_method) + + else: + message = ( + "{} request for \"{}\" don't have registered callback" + ).format(rest_method, self.path) + log.debug(message) + + self.send_response(HTTPStatus.NOT_FOUND) + self.handle_result(rest_method, message=message, success=False) + + def handle_result( + self, rest_method, final_output=None, message=None, success=True, + content_type="application/json" + ): + self.send_header("Content-type", content_type) + if final_output: + output = final_output + else: + if not message: + output = json.dumps({ + "success": False, + "message": ( + "{} request for \"{}\" has unexpected result" + ).format(rest_method, self.path) + }).encode() + + else: + output = json.dumps({ + "success": success, "message": message + }).encode() + + + if isinstance(output, str): + self.send_header("Content-Length", len(output)) + self.end_headers() - if not any_result: - return - - if len(results) == 1: - json_message = str(results[0]) - else: - index = 1 - messages = {} - for result in results: - if isinstance(result, str): - value = result - else: - value = json.dumps(result) - messages["callback{}".format(str(index))] = value - - json_message = json.dumps(messages) - - self.wfile.write(json_message.encode()) + if output: + self.wfile.write(output) class AdditionalArgsTCPServer(socketserver.TCPServer): @@ -172,14 +208,17 @@ class RestApiServer(QtCore.QThread): rest_method = str(rest_method).upper() if path in self.registered_callbacks[rest_method]: - log.warning( + log.error( "Path \"{}\" has already registered callback.".format(path) ) - else: - log.debug( - "Registering callback for path \"{}\"".format(path) - ) - self.registered_callbacks[rest_method][path].append(callback) + return False + + log.debug( + "Registering callback for path \"{}\"".format(path) + ) + self.registered_callbacks[rest_method][path] = callback + + return True def tray_start(self): self.start()