From 55a12b4abceee9c73834cf819e8bd6b2a2c3cdf6 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 12 Mar 2021 11:32:02 +0100 Subject: [PATCH 01/13] deleted rest api module --- pype/modules/rest_api/__init__.py | 27 -- pype/modules/rest_api/base_class.py | 124 -------- pype/modules/rest_api/lib/__init__.py | 7 - pype/modules/rest_api/lib/exceptions.py | 12 - pype/modules/rest_api/lib/factory.py | 357 ------------------------ pype/modules/rest_api/lib/handler.py | 349 ----------------------- pype/modules/rest_api/lib/lib.py | 197 ------------- pype/modules/rest_api/rest_api.py | 258 ----------------- 8 files changed, 1331 deletions(-) delete mode 100644 pype/modules/rest_api/__init__.py delete mode 100644 pype/modules/rest_api/base_class.py delete mode 100644 pype/modules/rest_api/lib/__init__.py delete mode 100644 pype/modules/rest_api/lib/exceptions.py delete mode 100644 pype/modules/rest_api/lib/factory.py delete mode 100644 pype/modules/rest_api/lib/handler.py delete mode 100644 pype/modules/rest_api/lib/lib.py delete mode 100644 pype/modules/rest_api/rest_api.py diff --git a/pype/modules/rest_api/__init__.py b/pype/modules/rest_api/__init__.py deleted file mode 100644 index b3312e8d31..0000000000 --- a/pype/modules/rest_api/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -from .rest_api import ( - RestApiModule, - IRestApi -) -from .base_class import ( - RestApi, - abort, - route, - register_statics -) -from .lib import ( - RestMethods, - CallbackResult -) - -__all__ = ( - "RestApiModule", - "IRestApi", - - "RestApi", - "abort", - "route", - "register_statics", - - "RestMethods", - "CallbackResult" -) diff --git a/pype/modules/rest_api/base_class.py b/pype/modules/rest_api/base_class.py deleted file mode 100644 index 6f06e01fcf..0000000000 --- a/pype/modules/rest_api/base_class.py +++ /dev/null @@ -1,124 +0,0 @@ -from http import HTTPStatus - -from .lib import ( - RestApiFactory, Splitter, - ObjAlreadyExist, AbortException, -) - - -def route(path, url_prefix="", methods=[], strict_match=False): - """Decorator that register callback and all its attributes. - Callback is registered to Singleton RestApiFactory. - - :param path: Specify url path when callback should be triggered. - :type path: str - :param url_prefix: Specify prefix of path, defaults to "/". - :type url_prefix: str, list, optional - :param methods: Specify request method (GET, POST, PUT, etc.) when - callback will be triggered, defaults to ["GET"] - :type methods: list, str, optional - :param strict_match: Decides if callback can handle both single and - multiple entities (~/projects/ && ~/projects/), - defaults to False. - :type strict_match: bool - - `path` may include dynamic keys that will be stored to object which can - be obtained in callback. - Example: - - registered path: "/projects/" - - url request path: "/projects/S001_test_project" - In this case will be callback triggered and in accessible data will be - stored {"project_name": "S001_test_project"}. - - `url_prefix` is optional but it is better to specify for easier filtering - of requests. - Example: - - url_prefix: `"/avalon"` or `["avalon"]` - - path: `"/projects"` - In this case request path must be "/avalon/projects" to trigger registered - callback. - """ - - def decorator(callback): - RestApiFactory.register_route( - path, callback, url_prefix, methods, strict_match - ) - callback.restapi = True - return callback - return decorator - - -def register_statics(url_prefix, dir_path): - """Decorator that register callback and all its attributes. - Callback is registered to Singleton RestApiFactory. - - :param url_prefix: Specify prefix of path, defaults to "/". - (Example: "/resources") - :type url_prefix: str - :param dir_path: Path to file folder where statics are located. - :type dir_path: str - """ - - RestApiFactory.register_statics((url_prefix, dir_path)) - - -def abort(status_code=HTTPStatus.NOT_FOUND, message=None): - """Should be used to stop registered callback. - `abort` raise AbortException that is handled with request Handler which - returns entered status and may send optional message in body. - - :param status_code: Status that will be send in reply of request, - defaults to 404 - :type status_code: int - :param message: Message to send in body, default messages are based on - statuc_code in Handler, defaults to None - :type message: str, optional - ... - :raises AbortException: This exception is handled in Handler to know - about launched `abort` - """ - - items = [] - items.append(str(status_code)) - if not message: - message = "" - - items.append(message) - - raise AbortException(Splitter.join(items)) - - -class RestApi: - """Base class for RestApi classes. - - Use this class is required when it is necessary to have class for handling - requests and want to use decorators for registering callbacks. - - It is possible to use decorators in another class only when object, - of class where decorators are, is registered to RestApiFactory. - """ - - def route(path, url_prefix="", methods=[], strict_match=False): - return route(path, url_prefix, methods, strict_match) - - @classmethod - def register_route( - cls, callback, path, url_prefix="", methods=[], strict_match=False - ): - return route(path, methods, url_prefix, strict_match)(callback) - - @classmethod - def register_statics(cls, url_prefix, dir_path): - return register_statics(url_prefix, dir_path) - - @classmethod - def abort(cls, status_code=HTTPStatus.NOT_FOUND, message=None): - abort(status_code, message) - - def __new__(cls, *args, **kwargs): - for obj in RestApiFactory.registered_objs: - if type(obj) == cls: - raise ObjAlreadyExist(cls) - instance = super(RestApi, cls).__new__(cls) - RestApiFactory.register_obj(instance) - return instance diff --git a/pype/modules/rest_api/lib/__init__.py b/pype/modules/rest_api/lib/__init__.py deleted file mode 100644 index 1c521de712..0000000000 --- a/pype/modules/rest_api/lib/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from .exceptions import ObjAlreadyExist, AbortException -from .lib import RestMethods, CallbackResult, RequestInfo, Splitter -from .factory import _RestApiFactory - -RestApiFactory = _RestApiFactory() - -from .handler import Handler diff --git a/pype/modules/rest_api/lib/exceptions.py b/pype/modules/rest_api/lib/exceptions.py deleted file mode 100644 index b17e1ce012..0000000000 --- a/pype/modules/rest_api/lib/exceptions.py +++ /dev/null @@ -1,12 +0,0 @@ -class ObjAlreadyExist(Exception): - """Is used when is created multiple objects of same RestApi class.""" - def __init__(self, cls=None, message=None): - if not (cls and message): - message = "RestApi object was created twice." - elif not message: - message = "{} object was created twice.".format(cls.__name__) - super().__init__(message) - - -class AbortException(Exception): - pass diff --git a/pype/modules/rest_api/lib/factory.py b/pype/modules/rest_api/lib/factory.py deleted file mode 100644 index ad884eea1c..0000000000 --- a/pype/modules/rest_api/lib/factory.py +++ /dev/null @@ -1,357 +0,0 @@ -import os -import re -import inspect -import collections -from .lib import RestMethods -from queue import Queue - -from pype.api import Logger - -log = Logger().get_logger("RestApiFactory") - - -def prepare_fullpath(path, prefix): - """Concatenate registered path and prefix with right form. - - :param path: Registered url path for registered callback. - :type path: str, list - :param prefix: Registered and prepared url prefix. - :type prefix: str, None - :return: concatenated prefix and path in right form - :rtype: str - """ - - if isinstance(path, (list, tuple)): - path_items = path - else: - path_items = [part for part in path.split("/") if part] - - fullpath = "/" - if path and prefix: - items = [part for part in prefix.split("/") if part] - items.extend(path_items) - fullpath = "/".join(items) - if path.endswith("/"): - fullpath += "/" - - elif path: - fullpath = "/".join(path_items) - if path.endswith("/"): - fullpath += "/" - - elif prefix: - fullpath = prefix - - if not fullpath.startswith("/"): - fullpath = "/{}".format(fullpath) - - return fullpath - - -def prepare_regex_from_path(full_path, strict_match): - """Prepare regex based on set path. - - When registered path do not contain dynamic keys regex is not set. - Dynamic keys are specified with "<" and ">" ("<{dynamic key}>"). - - :param full_path: Full url path (prefix + path) for registered callback. - :type full_path: str, list, None - :return: regex and keys of all groups in regex - :rtype: tuple(SRE_Pattern, list), tuple(None, None) - """ - get_indexes_regex = "<[^< >]+>" - all_founded_keys = re.findall(get_indexes_regex, full_path) - if not all_founded_keys: - return None, None - - regex_path = full_path - keys = [] - for key in all_founded_keys: - replacement = "(?P{}\w+)".format(key) - keys.append(key.replace("<", "").replace(">", "")) - if not strict_match: - if full_path.endswith(key): - replacement = "?{}?".format(replacement) - regex_path = regex_path.replace(key, replacement) - - regex_path = "^{}$".format(regex_path) - - return re.compile(regex_path), keys - - -def prepare_prefix(url_prefix): - """Check if the url_prefix is set and is in correct form. - - Output is None when prefix is empty or "/". - - :param url_prefix: Registered prefix of registered callback. - :type url_prefix: str, list, None - :return: Url prefix of registered callback - :rtype: str, None - """ - if url_prefix is None or url_prefix.strip() == "/": - return None - elif isinstance(url_prefix, (list, tuple)): - url_prefix = "/".join(url_prefix) - else: - items = [part for part in url_prefix.split("/") if part] - url_prefix = "/".join(items) - - if not url_prefix: - return None - - while url_prefix.endswith("/"): - url_prefix = url_prefix[:-1] - - if not url_prefix.startswith("/"): - url_prefix = "/{}".format(url_prefix) - - return url_prefix - - -def prepare_methods(methods, callback=None): - """Check and convert entered methods. - - String `methods` is converted to list. All values are converted to - `RestMethods` enum object. Invalid methods are ignored and printed out. - - :param methods: Contain rest api methods, when callback is called. - :type methods: str, list - :param callback: Registered callback, helps to identify where is - invalid method. - :type callback: function, method, optional - :return: Valid methods - :rtype: list - """ - invalid_methods = collections.defaultdict(list) - - if not methods: - _methods = [RestMethods.GET] - elif isinstance(methods, str) or isinstance(methods, RestMethods): - _method = RestMethods.get(methods) - _methods = [] - if _method is None: - invalid_methods[methods].append(callback) - else: - _methods.append(_method) - - else: - _methods = [] - for method in methods: - found = False - _method = RestMethods.get(method) - if _method is None: - invalid_methods[methods].append(callback) - continue - - _methods.append(_method) - - for method, callbacks in invalid_methods.items(): - callback_info = "" - - callbacks = [cbk for cbk in callbacks if cbk] - if len(callbacks) > 0: - multiple_ind = "" - if len(callbacks) > 1: - multiple_ind = "s" - - callback_items = [] - for callback in callbacks: - callback_items.append("\"{}<{}>\"".format( - callback.__qualname__, callback.__globals__["__file__"] - )) - - callback_info = " with callback{} {}".format( - multiple_ind, "| ".join(callback_items) - ) - - log.warning( - ("Invalid RestApi method \"{}\"{}").format(method, callback_info) - ) - - return _methods - - -def prepare_callback_info(callback): - """Prepare data for callback handling when should be triggered.""" - callback_info = inspect.getfullargspec(callback) - - callback_args = callback_info.args - callback_args_len = 0 - if callback_args: - callback_args_len = len(callback_args) - if type(callback).__name__ == "method": - callback_args_len -= 1 - - defaults = callback_info.defaults - defaults_len = 0 - if defaults: - defaults_len = len(defaults) - - annotations = callback_info.annotations - - return { - "args": callback_args, - "args_len": callback_args_len, - "defaults": defaults, - "defaults_len": defaults_len, - "hasargs": callback_info.varargs is not None, - "haskwargs": callback_info.varkw is not None, - "annotations": annotations - } - - -class _RestApiFactory: - """Factory is used to store and prepare callbacks for requests. - - Should be created only one object used for all registered callbacks when - it is expected to run only one http server. - """ - registered_objs = [] - unprocessed_routes = [] - unprocessed_statics = Queue() - - prepared_routes = { - method: collections.defaultdict(list) for method in RestMethods - } - prepared_statics = {} - - has_routes = False - - def has_handlers(self): - return (self.has_routes or self.prepared_statics) - - def _process_route(self, route): - return self.unprocessed_routes.pop( - self.unprocessed_routes.index(route) - ) - - def register_route( - self, path, callback, url_prefix, methods, strict_match - ): - log.debug("Registering callback for item \"{}\"".format( - callback.__qualname__ - )) - route = { - "path": path, - "callback": callback, - "url_prefix": url_prefix, - "methods": methods, - "strict_match": strict_match - } - self.unprocessed_routes.append(route) - - def register_obj(self, obj): - """Register object for decorated methods in class definition.""" - self.registered_objs.append(obj) - - def register_statics(self, item): - log.debug("Registering statics path \"{}\"".format(item)) - self.unprocessed_statics.put(item) - - def _prepare_route(self, route): - """Prepare data of registered callbacks for routes. - - Registration info are prepared to easy filter during handling - of requests. - - :param route: Contain all necessary info for filtering and - handling callback for registered route. - :type route: dict - """ - callback = route["callback"] - methods = prepare_methods(route["methods"], callback) - url_prefix = prepare_prefix(route["url_prefix"]) - fullpath = prepare_fullpath(route["path"], url_prefix) - regex, regex_keys = prepare_regex_from_path( - fullpath, route["strict_match"] - ) - callback_info = prepare_callback_info(callback) - - for method in methods: - self.has_routes = True - self.prepared_routes[method][url_prefix].append({ - "regex": regex, - "regex_keys": regex_keys, - "fullpath": fullpath, - "callback": callback, - "callback_info": callback_info - }) - - def prepare_registered(self): - """Iter through all registered callbacks and statics to prepare them. - - First are processed callbacks registered with decorators in classes by - registered objects. Remaining callbacks are filtered, it is checked if - methods has `__self__` or are defined in (it is expeted they - do not requise access to object) - """ - - while not self.unprocessed_statics.empty(): - url_prefix, dir_path = self.unprocessed_statics.get() - dir_path = os.path.normpath(dir_path) - if not os.path.exists(dir_path): - log.warning( - "Directory path \"{}\" was not found".format(dir_path) - ) - continue - url_prefix = prepare_prefix(url_prefix) - self.prepared_statics[url_prefix] = dir_path - - for obj in self.registered_objs: - method_names = [ - attr for attr in dir(obj) - if inspect.ismethod(getattr(obj, attr)) - ] - for method_name in method_names: - method = obj.__getattribute__(method_name) - if not hasattr(method, "restapi"): - continue - - if not method.restapi: - continue - - for route in list(self.unprocessed_routes): - callback = route["callback"] - if not ( - callback.__qualname__ == method.__qualname__ and - callback.__module__ == method.__module__ and - callback.__globals__["__file__"] == ( - method.__globals__["__file__"] - ) - ): - continue - - route["callback"] = method - self._process_route(route) - self._prepare_route(route) - break - - for route in list(self.unprocessed_routes): - callback = route["callback"] - is_class_method = len(callback.__qualname__.split(".")) != 1 - if is_class_method: - missing_self = True - if hasattr(callback, "__self__"): - if callback.__self__ is not None: - missing_self = False - - if "" in callback.__qualname__: - pass - - elif missing_self: - log.warning(( - "Object of callback \"{}\" from \"{}\" is not" - " accessible for api. Register object or" - " register callback with already created object" - "(not with decorator in class).".format( - callback.__qualname__, - callback.__globals__["__file__"] - ) - )) - continue - - self._prepare_route(route) - continue - - self._prepare_route(route) diff --git a/pype/modules/rest_api/lib/handler.py b/pype/modules/rest_api/lib/handler.py deleted file mode 100644 index 93e583882b..0000000000 --- a/pype/modules/rest_api/lib/handler.py +++ /dev/null @@ -1,349 +0,0 @@ -import os -import re -import json -import datetime -import traceback -import http.server -from http import HTTPStatus -from urllib.parse import urlparse - -from .lib import RestMethods, CallbackResult, RequestInfo -from .exceptions import AbortException -from . import RestApiFactory, Splitter - -from pype.api import Logger - -log = Logger().get_logger("RestApiHandler") - - -class Handler(http.server.SimpleHTTPRequestHandler): - # TODO fill will necessary statuses - default_messages = { - HTTPStatus.BAD_REQUEST: "Bad request", - HTTPStatus.NOT_FOUND: "Not found" - } - - statuses = { - "POST": { - "OK": 200, - "CREATED": 201 - }, - "PUT": { - "OK": 200, - "NO_CONTENT": 204 - } - } - - def do_GET(self): - return self._handle_request(RestMethods.GET) - - def do_POST(self): - return self._handle_request(RestMethods.POST) - - def do_PUT(self): - return self._handle_request(RestMethods.PUT) - - def do_DELETE(self): - return self._handle_request(RestMethods.DELETE) - - def do_PATCH(self): - return self._handle_request(RestMethods.PATCH) - - def _handle_request(self, rest_method): - """Handle request by registered callbacks and statics. - - Statics are only for GET method request. `_handle_statics` is called - when path is matching registered statics prefix. Callbacks are filtered - by method and their prefixes. When path is matching `_handle_callback` - is called. - - If any registered callback or statics match requested path 400 status - is responsed. And 500 when unexpected error happens. - """ - parsed_url = urlparse(self.path) - path = parsed_url.path - - if rest_method is RestMethods.GET: - for prefix, dirpath in RestApiFactory.prepared_statics.items(): - if not path.startswith(prefix): - continue - _path = path[len(prefix):] - return self._handle_statics(dirpath, _path) - - matching_item = None - found_prefix = None - url_prefixes = RestApiFactory.prepared_routes[rest_method] - for url_prefix, items in url_prefixes.items(): - if matching_item is not None: - break - - if url_prefix is not None: - if not path.startswith(url_prefix): - continue - - found_prefix = url_prefix - - for item in items: - regex = item["regex"] - item_full_path = item["fullpath"] - if regex is None: - if path == item_full_path: - item["url_data"] = None - matching_item = item - break - - else: - found = re.match(regex, path) - if found: - item["url_data"] = found.groupdict() - matching_item = item - break - - if not matching_item: - if found_prefix is not None: - _path = path.replace(found_prefix, "") - if _path: - request_str = " \"{}\"".format(_path) - else: - request_str = "" - - message = "Invalid path request{} for prefix \"{}\"".format( - request_str, found_prefix - ) - else: - message = "Invalid path request \"{}\"".format(self.path) - log.debug(message) - self.send_error(HTTPStatus.BAD_REQUEST, message) - - return - - try: - log.debug("Triggering callback for path \"{}\"".format(path)) - - result = self._handle_callback( - matching_item, parsed_url, rest_method - ) - - return self._handle_callback_result(result, rest_method) - - except AbortException as exc: - status_code, message = str(exc).split(Splitter) - status_code = int(status_code) - if not message: - message = self.default_messages.get( - status_code, "UnexpectedError" - ) - - self.send_response(status_code) - self.send_header("Content-type", "text/html") - self.send_header("Content-Length", len(message)) - self.end_headers() - - self.wfile.write(message.encode()) - return message - - except Exception as exc: - log_message = "Unexpected Exception was raised (this is bug!)" - log.error(log_message, exc_info=True) - items = [log_message] - items += traceback.extract_tb(exc.__traceback__).format() - message = "\n".join(items) - - self.send_response(HTTPStatus.INTERNAL_SERVER_ERROR) - self.send_header("Content-type", "text/html") - self.send_header("Content-Length", len(message)) - self.end_headers() - - self.wfile.write(message.encode()) - return message - - def _handle_callback_result(self, result, rest_method): - """Send response to request based on result of callback. - :param result: Result returned by callback. - :type result: None, bool, dict, list, CallbackResult - :param rest_method: Rest api method (GET, POST, etc.). - :type rest_method: RestMethods - - Response is based on result type: - - None, True - It is expected everything was OK, status 200. - - False - It is expected callback was not successful, status 400 - - dict, list - Result is send under "data" key of body, status 200 - - CallbackResult - object specify status and data - """ - content_type = "application/json" - status = HTTPStatus.OK - success = True - message = None - data = None - - body = None - # TODO better handling of results - if isinstance(result, CallbackResult): - status = result.status_code - body_dict = {} - for key, value in result.items(): - if value is not None: - body_dict[key] = value - body = json.dumps(body_dict) - - elif result in [None, True]: - status = HTTPStatus.OK - success = True - message = "{} request for \"{}\" passed".format( - rest_method, self.path - ) - - elif result is False: - status = HTTPStatus.BAD_REQUEST - success = False - - elif isinstance(result, (dict, list)): - status = HTTPStatus.OK - data = result - - if status == HTTPStatus.NO_CONTENT: - self.send_response(status) - self.end_headers() - return - - if not body: - body_dict = {"success": success} - if message: - body_dict["message"] = message - - if not data: - data = {} - - body_dict["data"] = data - body = json.dumps(body_dict) - - self.send_response(status) - self.send_header("Content-type", content_type) - self.send_header("Content-Length", len(body)) - self.end_headers() - - self.wfile.write(body.encode()) - return body - - def _handle_callback(self, item, parsed_url, rest_method): - """Prepare data from request and trigger callback. - - Data are loaded from body of request if there are any. - - :param item: Item stored during callback registration with all info. - :type item: dict - :param parsed_url: Url parsed with urllib (separated path, query, etc). - :type parsed_url: ParseResult - :param rest_method: Rest api method (GET, POST, etc.). - :type rest_method: RestMethods - """ - regex_keys = item["regex_keys"] - - url_data = None - if regex_keys: - url_data = {key: None for key in regex_keys} - if item["url_data"]: - for key, value in item["url_data"].items(): - url_data[key] = value - - in_data = None - cont_len = self.headers.get("Content-Length") - if cont_len: - content_length = int(cont_len) - in_data_str = self.rfile.read(content_length) - if in_data_str: - try: - in_data = json.loads(in_data_str) - except Exception as e: - log.error("Invalid JSON recieved: \"{}\"".format( - str(in_data_str) - )) - raise Exception("Invalid JSON recieved") from e - - request_info = RequestInfo( - url_data=url_data, - request_data=in_data, - query=parsed_url.query, - fragment=parsed_url.fragment, - params=parsed_url.params, - method=rest_method, - handler=self - ) - - callback = item["callback"] - callback_info = item["callback_info"] - - _args_len = callback_info["args_len"] - _has_args = callback_info["hasargs"] - _has_kwargs = callback_info["haskwargs"] - - args = [] - kwargs = {} - if _args_len == 0: - if _has_args: - args.append(request_info) - elif _has_kwargs: - kwargs["request_info"] = request_info - else: - args.append(request_info) - - return callback(*args, **kwargs) - - def _handle_statics(self, dirpath, path): - """Return static file in response when file exist in destination.""" - path = os.path.normpath(dirpath + path) - - ctype = self.guess_type(path) - try: - file_obj = open(path, "rb") - except OSError: - self.send_error(HTTPStatus.NOT_FOUND, "File not found") - return None - - try: - file_stat = os.fstat(file_obj.fileno()) - # Use browser cache if possible - if ("If-Modified-Since" in self.headers - and "If-None-Match" not in self.headers): - # compare If-Modified-Since and time of last file modification - try: - ims = http.server.email.utils.parsedate_to_datetime( - self.headers["If-Modified-Since"]) - except (TypeError, IndexError, OverflowError, ValueError): - # ignore ill-formed values - pass - else: - if ims.tzinfo is None: - # obsolete format with no timezone, cf. - # https://tools.ietf.org/html/rfc7231#section-7.1.1.1 - ims = ims.replace(tzinfo=datetime.timezone.utc) - if ims.tzinfo is datetime.timezone.utc: - # compare to UTC datetime of last modification - last_modif = datetime.datetime.fromtimestamp( - file_stat.st_mtime, datetime.timezone.utc) - # remove microseconds, like in If-Modified-Since - last_modif = last_modif.replace(microsecond=0) - - if last_modif <= ims: - self.send_response(HTTPStatus.NOT_MODIFIED) - self.end_headers() - file_obj.close() - return None - - self.send_response(HTTPStatus.OK) - self.send_header("Content-type", ctype) - self.send_header("Content-Length", str(file_stat[6])) - self.send_header( - "Last-Modified", - self.date_time_string(file_stat.st_mtime) - ) - self.end_headers() - self.wfile.write(file_obj.read()) - return file_obj - except Exception: - log.error( - "Failed to read data from file \"{}\"".format(path), - exc_info=True - ) - finally: - file_obj.close() diff --git a/pype/modules/rest_api/lib/lib.py b/pype/modules/rest_api/lib/lib.py deleted file mode 100644 index f5986cce8c..0000000000 --- a/pype/modules/rest_api/lib/lib.py +++ /dev/null @@ -1,197 +0,0 @@ -import enum -from http import HTTPStatus -from urllib.parse import urlencode, parse_qs - -Splitter = "__splitter__" - - -class RestMethods(enum.Enum): - GET = "GET" - POST = "POST" - PUT = "PUT" - PATCH = "PATCH" - DELETE = "DELETE" - - def __eq__(self, other): - if isinstance(other, self.__class__): - return self.value == other.value - - elif isinstance(other, str): - return self.value.lower() == other.lower() - return self.value == other - - def __hash__(self): - return enum.Enum.__hash__(self) - - @classmethod - def get(cls, name, default=None): - for meth in cls: - if meth == name: - return meth - return default - - -class HandlerDict(dict): - def __init__(self, data=None, *args, **kwargs): - if not data: - data = {} - super().__init__(data, *args, **kwargs) - - def __repr__(self): - return "<{}> {}".format(self.__class__.__name__, str(dict(self))) - - -class Params(HandlerDict): - pass - - -class UrlData(HandlerDict): - pass - - -class RequestData(HandlerDict): - pass - - -class Query(HandlerDict): - """Class for url query convert to dict and string.""" - - def __init__(self, query): - if isinstance(query, dict): - pass - else: - query = parse_qs(query) - super().__init__(query) - - def get_string(self): - return urlencode(dict(self), doseq=True) - - -class Fragment(HandlerDict): - """Class for url fragment convert to dict and string.""" - - def __init__(self, fragment): - if isinstance(fragment, dict): - _fragment = fragment - else: - _fragment = {} - for frag in fragment.split("&"): - if not frag: - continue - items = frag.split("=") - - value = None - key = items[0] - if len(items) == 2: - value = items[1] - elif len(items) > 2: - value = "=".join(items[1:]) - - _fragment[key] = value - - super().__init__(_fragment) - - def get_string(self): - items = [] - for parts in dict(self).items(): - items.append( - "=".join([p for p in parts if p]) - ) - return "&".join(items) - - -class RequestInfo: - """Object that can be passed to callback as argument. - - Contain necessary data for handling request. - Object is created to single use and can be used similar to dict. - - :param url_data: Data collected from path when path with dynamic keys - is matching. - :type url_data: dict, None - :param request_data: Data of body from request. - :type request_data: dict, None - :param query: Query from url path of reques. - :type query: str, None - :param fragment: Fragment from url path of reques. - :type fragment: str, None - :param params: Parems from url path of reques. - :type params: str, None - :param method: Method of request (GET, POST, etc.) - :type method: RestMethods - :param handler: Handler handling request from http server. - :type handler: Handler - """ - def __init__( - self, url_data, request_data, query, fragment, params, method, handler - ): - self.url_data = UrlData(url_data) - self.request_data = RequestData(request_data) - self.query = Query(query) - self.fragment = Fragment(fragment) - self.params = Params(params) - self.method = method - self.handler = handler - - def __getitem__(self, key): - return self.__getattribute__(key) - - def __hash__(self): - return { - "url_data": self.url_data, - "request_data": self. request_data, - "query": self.query, - "fragment": self.fragment, - "params": self.params, - "method": self.method, - "handler": self.handler - } - - def items(self): - return dict(self).items() - - -class CallbackResult: - """Can be used as return value of callback. - - It is possible to specify status code, success boolean, message and data - for specify head and body of request response. `abort` should be rather - used when result is error. - - :param status_code: Status code of result. - :type status_code: int - :param success: Success is key in body, may be used for handling response. - :type success: bool - :param message: Similar to success, message is key in body and may - be used for handling response. - :type message: str, None - :param data: Data is also key for body in response. - :type data: dict, None - """ - _data = {} - - def __init__( - self, status_code=HTTPStatus.OK, success=True, message=None, data=None, - **kwargs - ): - self.status_code = status_code - self._data = { - "success": success, - "message": message, - "data": data - } - for k, v in kwargs.items(): - self._data[k] = v - - def __getitem__(self, key): - return self._data[key] - - def __iter__(self): - for key in self._data: - yield key - - def get(self, key, default=None): - return self._data.get(key, default) - - def items(self): - return self._data.items() diff --git a/pype/modules/rest_api/rest_api.py b/pype/modules/rest_api/rest_api.py deleted file mode 100644 index b37f5e13eb..0000000000 --- a/pype/modules/rest_api/rest_api.py +++ /dev/null @@ -1,258 +0,0 @@ -import os -import socket -import threading -from abc import ABCMeta, abstractmethod -from socketserver import ThreadingMixIn -from http.server import HTTPServer - -import six - -from pype.lib import PypeLogger -from pype import resources - -from .lib import RestApiFactory, Handler -from .base_class import route, register_statics -from .. import PypeModule, ITrayService - - -@six.add_metaclass(ABCMeta) -class IRestApi: - """Other modules interface to return paths to ftrack event handlers. - - Expected output is dictionary with "server" and "user" keys. - """ - @abstractmethod - def rest_api_initialization(self, rest_api_module): - pass - - -class RestApiModule(PypeModule, ITrayService): - """Rest Api allows to access statics or callbacks with http requests. - - To register statics use `register_statics`. - - To register callback use `register_callback` method or use `route` decorator. - `route` decorator should be used with not-class functions, it is possible - to use within class when inherits `RestApi` (defined in `base_class.py`) - or created object, with used decorator, is registered with `register_obj`. - - .. code-block:: python - @route("/username", url_prefix="/api", methods=["get"], strict_match=False) - def get_username(): - return {"username": getpass.getuser()} - - In that case response to `localhost:{port}/api/username` will be status - `200` with body including `{"data": {"username": getpass.getuser()}}` - - Callback may expect one argument which will be filled with request - info. Data object has attributes: `request_data`, `query`, - `fragment`, `params`, `method`, `handler`, `url_data`. - request_data - Data from request body if there are any. - query - query from url path (?identificator=name) - fragment - fragment from url path (#reset_credentials) - params - params from url path - method - request method (GET, POST, PUT, etc.) - handler - Handler object of HttpServer Handler - url_data - dynamic url keys from registered path with their values - - Dynamic url keys may be set with path argument. - .. code-block:: python - from rest_api import route - - all_projects = { - "Proj1": {"proj_data": []}, - "Proj2": {"proj_data": []}, - } - - @route("/projects/", url_prefix="/api", methods=["get"], strict_match=False) - def get_projects(request_info): - project_name = request_info.url_data["project_name"] - if not project_name: - return all_projects - return all_projects.get(project_name) - - This example should end with status 404 if project is not found. In that - case is best to use `abort` method. - - .. code-block:: python - from rest_api import abort - - @route("/projects/", url_prefix="/api", methods=["get"], strict_match=False) - def get_projects(request_info): - project_name = request_info.url_data["project_name"] - if not project_name: - return all_projects - - project = all_projects.get(project_name) - if not project: - abort(404, "Project \"{}\".format(project_name) was not found") - return project - - `strict_match` allows to handle not only specific entity but all entity types. - E.g. "/projects/" with set `strict_match` to False will handle also - "/projects" or "/projects/" path. It is necessary to set `strict_match` to - True when should handle only single entity. - - Callback may return many types. For more information read docstring of - `_handle_callback_result` defined in handler. - """ - label = "Rest API Service" - name = "rest_api" - - def initialize(self, modules_settings): - rest_api_settings = modules_settings[self.name] - self.enabled = True - self.default_port = rest_api_settings["default_port"] - self.exclude_ports = rest_api_settings["exclude_ports"] - - self.rest_api_url = None - self.rest_api_thread = None - - def register_callback( - self, path, callback, url_prefix="", methods=[], strict_match=False - ): - RestApiFactory.register_route( - path, callback, url_prefix, methods, strict_match - ) - - def register_statics(self, url_prefix, dir_path): - register_statics(url_prefix, dir_path) - - def register_obj(self, obj): - RestApiFactory.register_obj(obj) - - def connect_with_modules(self, enabled_modules): - # Do not register restapi callbacks out of tray - if self.tray_initialized: - for module in enabled_modules: - if not isinstance(module, IRestApi): - continue - - module.rest_api_initialization(self) - - @staticmethod - def find_free_port(port_from, port_to=None, exclude_ports=None, host=None): - """Find available socket port from entered range. - - It is also possible to only check if entered port is available. - - Args: - port_from (int): Port number which is checked as first. - port_to (int): Last port that is checked in sequence from entered - `port_from`. Only `port_from` is checked if is not entered. - Nothing is processed if is equeal to `port_from`! - exclude_ports (list, tuple, set): List of ports that won't be - checked form entered range. - host (str): Host where will check for free ports. Set to - "localhost" by default. - """ - # Check only entered port if `port_to` is not defined - if port_to is None: - port_to = port_from - - # Excluded ports (e.g. reserved for other servers/clients) - if exclude_ports is None: - exclude_ports = [] - - # Default host is localhost but it is possible to look for other hosts - if host is None: - host = "localhost" - - found_port = None - for port in range(port_from, port_to + 1): - if port in exclude_ports: - continue - - sock = None - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.bind((host, port)) - found_port = port - - except socket.error: - continue - - finally: - if sock: - sock.close() - - if found_port is not None: - break - - return found_port - - def tray_init(self): - port = self.find_free_port( - self.default_port, self.default_port + 100, self.exclude_ports - ) - self.rest_api_url = "http://localhost:{}".format(port) - self.rest_api_thread = RestApiThread(self, port) - self.register_statics("/res", resources.RESOURCES_DIR) - - # Set rest api environments - os.environ["PYPE_REST_API_URL"] = self.rest_api_url - - def tray_start(self): - RestApiFactory.prepare_registered() - if not RestApiFactory.has_handlers(): - self.log.debug("There are not registered any handlers for RestApi") - return - self.rest_api_thread.start() - - @property - def is_running(self): - return self.rest_api_thread.is_running - - def tray_exit(self): - self.stop() - - def stop(self): - self.rest_api_thread.stop() - self.rest_api_thread.join() - - -class ThreadingSimpleServer(ThreadingMixIn, HTTPServer): - pass - - -class RestApiThread(threading.Thread): - """ Listener for REST requests. - - It is possible to register callbacks for url paths. - Be careful about crossreferencing to different Threads it is not allowed. - """ - - def __init__(self, module, port): - super(RestApiThread, self).__init__() - self.is_running = False - self.module = module - self.port = port - self.httpd = None - self.log = PypeLogger().get_logger("RestApiThread") - - def stop(self): - self.is_running = False - if self.httpd: - self.httpd.server_close() - - def run(self): - self.is_running = True - - try: - self.log.debug( - "Running Rest Api server on URL:" - " \"http://localhost:{}\"".format(self.port) - ) - - with ThreadingSimpleServer(("", self.port), Handler) as httpd: - self.httpd = httpd - while self.is_running: - httpd.handle_request() - - except Exception: - self.log.warning( - "Rest Api Server service has failed", exc_info=True - ) - - self.httpd = None - self.is_running = False From 43944663cc58a165de032e0f9efd150a853487ca Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 12 Mar 2021 11:32:30 +0100 Subject: [PATCH 02/13] removed rest_api from settings --- .../defaults/system_settings/modules.json | 4 --- .../schemas/system_schema/schema_modules.json | 27 +------------------ 2 files changed, 1 insertion(+), 30 deletions(-) diff --git a/pype/settings/defaults/system_settings/modules.json b/pype/settings/defaults/system_settings/modules.json index 10a5aa723a..8c4d7fe196 100644 --- a/pype/settings/defaults/system_settings/modules.json +++ b/pype/settings/defaults/system_settings/modules.json @@ -126,10 +126,6 @@ } } }, - "rest_api": { - "default_port": 8021, - "exclude_ports": [] - }, "timers_manager": { "enabled": true, "full_time": 15.0, diff --git a/pype/settings/entities/schemas/system_schema/schema_modules.json b/pype/settings/entities/schemas/system_schema/schema_modules.json index d9f2a23a80..2ee316f29e 100644 --- a/pype/settings/entities/schemas/system_schema/schema_modules.json +++ b/pype/settings/entities/schemas/system_schema/schema_modules.json @@ -30,31 +30,6 @@ "type": "schema", "name": "schema_ftrack" }, - { - "type": "dict", - "key": "rest_api", - "label": "Rest Api", - "collapsible": true, - "children": [ - { - "type": "number", - "key": "default_port", - "label": "Default Port", - "minimum": 1, - "maximum": 65535 - }, - { - "type": "list", - "key": "exclude_ports", - "label": "Exclude ports", - "object_type": { - "type": "number", - "minimum": 1, - "maximum": 65535 - } - } - ] - }, { "type": "dict", "key": "timers_manager", @@ -217,4 +192,4 @@ ] } ] -} \ No newline at end of file +} From fe63883f96ce77a924df15019de78942b7133ffe Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 12 Mar 2021 11:34:17 +0100 Subject: [PATCH 03/13] removed rest api usage from avalon module --- pype/modules/avalon_apps/avalon_app.py | 10 +-- pype/modules/avalon_apps/rest_api.py | 85 -------------------------- 2 files changed, 2 insertions(+), 93 deletions(-) delete mode 100644 pype/modules/avalon_apps/rest_api.py diff --git a/pype/modules/avalon_apps/avalon_app.py b/pype/modules/avalon_apps/avalon_app.py index dcc1a36934..c61704db14 100644 --- a/pype/modules/avalon_apps/avalon_app.py +++ b/pype/modules/avalon_apps/avalon_app.py @@ -3,12 +3,11 @@ import pype from pype import resources from .. import ( PypeModule, - ITrayModule, - IRestApi + ITrayModule ) -class AvalonModule(PypeModule, ITrayModule, IRestApi): +class AvalonModule(PypeModule, ITrayModule): name = "avalon" def initialize(self, modules_settings): @@ -74,11 +73,6 @@ class AvalonModule(PypeModule, ITrayModule, IRestApi): def connect_with_modules(self, _enabled_modules): return - def rest_api_initialization(self, rest_api_module): - if self.tray_initialized: - from .rest_api import AvalonRestApi - self.rest_api_obj = AvalonRestApi() - # Definition of Tray menu def tray_menu(self, tray_menu): from Qt import QtWidgets diff --git a/pype/modules/avalon_apps/rest_api.py b/pype/modules/avalon_apps/rest_api.py deleted file mode 100644 index 2408e56bbc..0000000000 --- a/pype/modules/avalon_apps/rest_api.py +++ /dev/null @@ -1,85 +0,0 @@ -import os -import re -import json -import bson -import bson.json_util -from pype.modules.rest_api import RestApi, abort, CallbackResult -from avalon.api import AvalonMongoDB - - -class AvalonRestApi(RestApi): - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.dbcon = AvalonMongoDB() - self.dbcon.install() - - @RestApi.route("/projects/", url_prefix="/avalon", methods="GET") - def get_project(self, request): - project_name = request.url_data["project_name"] - if not project_name: - output = {} - for project_name in self.dbcon.tables(): - project = self.dbcon[project_name].find_one({ - "type": "project" - }) - output[project_name] = project - - return CallbackResult(data=self.result_to_json(output)) - - project = self.dbcon[project_name].find_one({"type": "project"}) - - if project: - return CallbackResult(data=self.result_to_json(project)) - - abort(404, "Project \"{}\" was not found in database".format( - project_name - )) - - @RestApi.route("/projects//assets/", url_prefix="/avalon", methods="GET") - def get_assets(self, request): - _project_name = request.url_data["project_name"] - _asset = request.url_data["asset"] - - if not self.dbcon.exist_table(_project_name): - abort(404, "Project \"{}\" was not found in database".format( - _project_name - )) - - if not _asset: - assets = self.dbcon[_project_name].find({"type": "asset"}) - output = self.result_to_json(assets) - return CallbackResult(data=output) - - # identificator can be specified with url query (default is `name`) - identificator = request.query.get("identificator", "name") - - asset = self.dbcon[_project_name].find_one({ - "type": "asset", - identificator: _asset - }) - if asset: - id = asset["_id"] - asset["_id"] = str(id) - return asset - - abort(404, "Asset \"{}\" with {} was not found in project {}".format( - _asset, identificator, _project_name - )) - - def result_to_json(self, result): - """ Converts result of MongoDB query to dict without $oid (ObjectId) - keys with help of regex matching. - - ..note: - This will convert object type entries similar to ObjectId. - """ - bson_json = bson.json_util.dumps(result) - # Replace "{$oid: "{entity id}"}" with "{entity id}" - regex1 = '(?P{\"\$oid\": \"[^\"]+\"})' - regex2 = '{\"\$oid\": (?P\"[^\"]+\")}' - for value in re.findall(regex1, bson_json): - for substr in re.findall(regex2, value): - bson_json = bson_json.replace(value, substr) - - return json.loads(bson_json) From 33a0742b3db2a69db3cf6f946af05361bded3133 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 12 Mar 2021 11:35:26 +0100 Subject: [PATCH 04/13] removed rest api usage from muster module --- pype/modules/muster/muster.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/pype/modules/muster/muster.py b/pype/modules/muster/muster.py index 393a10e29a..4f363b923e 100644 --- a/pype/modules/muster/muster.py +++ b/pype/modules/muster/muster.py @@ -2,10 +2,13 @@ import os import json import appdirs import requests -from .. import PypeModule, ITrayModule, IRestApi +from .. import ( + PypeModule, + ITrayModule +) -class MusterModule(PypeModule, ITrayModule, IRestApi): +class MusterModule(PypeModule, ITrayModule): """ Module handling Muster Render credentials. This will display dialog asking for user credentials for Muster if not already specified. @@ -71,14 +74,6 @@ class MusterModule(PypeModule, ITrayModule, IRestApi): parent.addMenu(menu) - def rest_api_initialization(self, rest_api_module): - """Implementation of IRestApi interface.""" - def api_show_login(): - self.action_show_login.trigger() - rest_api_module.register_callback( - "/show_login", api_show_login, "muster", "post" - ) - def load_credentials(self): """ Get credentials from JSON file From 326b3762be70c8a2c8cf429e04e6228c59a5397a Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 12 Mar 2021 11:37:17 +0100 Subject: [PATCH 05/13] removed rest api usage from user module --- pype/modules/user/user_module.py | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/pype/modules/user/user_module.py b/pype/modules/user/user_module.py index 21e687c43d..0340a2e6eb 100644 --- a/pype/modules/user/user_module.py +++ b/pype/modules/user/user_module.py @@ -7,7 +7,10 @@ from abc import ABCMeta, abstractmethod import six import appdirs -from .. import PypeModule, ITrayModule, IRestApi +from .. import ( + PypeModule, + ITrayModule +) @six.add_metaclass(ABCMeta) @@ -20,7 +23,7 @@ class IUserModule: pass -class UserModule(PypeModule, ITrayModule, IRestApi): +class UserModule(PypeModule, ITrayModule): cred_folder_path = os.path.normpath( appdirs.user_data_dir('pype-app', 'pype') ) @@ -68,21 +71,6 @@ class UserModule(PypeModule, ITrayModule, IRestApi): def get_user(self): return self.cred.get("username") or getpass.getuser() - def rest_api_initialization(self, rest_api_module): - def api_get_username(): - return self.cred - - rest_api_module.register_callback( - "user/username", api_get_username, "get" - ) - - def api_show_widget(): - self.action_show_widget.trigger() - - rest_api_module.register_callback( - "user/show_widget", api_show_widget, "post" - ) - def connect_with_modules(self, enabled_modules): for module in enabled_modules: if isinstance(module, IUserModule): From cbd233c93a361db946a82eb6e6779cb8a180fb99 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 12 Mar 2021 11:38:49 +0100 Subject: [PATCH 06/13] added find_port method to webserver module --- pype/modules/webserver/server.py | 13 +++-- pype/modules/webserver/webserver_module.py | 59 +++++++++++++++++++++- 2 files changed, 67 insertions(+), 5 deletions(-) diff --git a/pype/modules/webserver/server.py b/pype/modules/webserver/server.py index 41f8f86a1b..e4b0ec236b 100644 --- a/pype/modules/webserver/server.py +++ b/pype/modules/webserver/server.py @@ -21,7 +21,11 @@ class WebServerManager: # add route with multiple methods for single "external app" - self.webserver_thread = WebServerThread(self, self.module.port) + self.webserver_thread = WebServerThread(self) + + @property + def port(self): + return self.module.port def add_route(self, *args, **kwargs): self.app.router.add_route(*args, **kwargs) @@ -60,17 +64,20 @@ class WebServerManager: class WebServerThread(threading.Thread): """ Listener for requests in thread.""" - def __init__(self, manager, port): + def __init__(self, manager): super(WebServerThread, self).__init__() self.is_running = False - self.port = port self.manager = manager self.loop = None self.runner = None self.site = None self.tasks = [] + @property + def port(self): + return self.manager.port + def run(self): self.is_running = True diff --git a/pype/modules/webserver/webserver_module.py b/pype/modules/webserver/webserver_module.py index 0cdf478d8c..5ebfdbe2d2 100644 --- a/pype/modules/webserver/webserver_module.py +++ b/pype/modules/webserver/webserver_module.py @@ -1,4 +1,5 @@ import os +import socket from pype import resources from .. import PypeModule, ITrayService @@ -11,8 +12,7 @@ class WebServerModule(PypeModule, ITrayService): self.enabled = True self.server_manager = None - # TODO find free port - self.port = 8098 + self.port = self.find_free_port() def connect_with_modules(self, *_a, **_kw): return @@ -53,3 +53,58 @@ class WebServerModule(PypeModule, ITrayService): self.server_manager.on_stop_callbacks.append( self.set_service_failed_icon ) + + @staticmethod + def find_free_port( + port_from=None, port_to=None, exclude_ports=None, host=None + ): + """Find available socket port from entered range. + + It is also possible to only check if entered port is available. + + Args: + port_from (int): Port number which is checked as first. + port_to (int): Last port that is checked in sequence from entered + `port_from`. Only `port_from` is checked if is not entered. + Nothing is processed if is equeal to `port_from`! + exclude_ports (list, tuple, set): List of ports that won't be + checked form entered range. + host (str): Host where will check for free ports. Set to + "localhost" by default. + """ + if port_from is None: + port_from = 8079 + + if port_to is None: + port_to = 65535 + + # Excluded ports (e.g. reserved for other servers/clients) + if exclude_ports is None: + exclude_ports = [] + + # Default host is localhost but it is possible to look for other hosts + if host is None: + host = "localhost" + + found_port = None + for port in range(port_from, port_to + 1): + if port in exclude_ports: + continue + + sock = None + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.bind((host, port)) + found_port = port + + except socket.error: + continue + + finally: + if sock: + sock.close() + + if found_port is not None: + break + + return found_port From 1ac0790f57b2df9a672eb9c655209ee4a8434554 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 12 Mar 2021 11:39:21 +0100 Subject: [PATCH 07/13] implemented interface for webserver module --- pype/modules/__init__.py | 14 ++++++------ pype/modules/webserver/__init__.py | 6 +++++- pype/modules/webserver/webserver_module.py | 25 +++++++++++++++++----- 3 files changed, 31 insertions(+), 14 deletions(-) diff --git a/pype/modules/__init__.py b/pype/modules/__init__.py index 787b9128e8..7b945922ea 100644 --- a/pype/modules/__init__.py +++ b/pype/modules/__init__.py @@ -13,9 +13,9 @@ from .settings_action import ( SettingsAction, LocalSettingsAction ) -from .rest_api import ( - RestApiModule, - IRestApi +from .webserver import ( + WebServerModule, + IWebServerRoutes ) from .user import ( UserModule, @@ -40,7 +40,6 @@ from .log_viewer import LogViewModule from .muster import MusterModule from .deadline import DeadlineModule from .standalonepublish_action import StandAlonePublishAction -from .webserver import WebServerModule from .sync_server import SyncServer @@ -57,6 +56,9 @@ __all__ = ( "SettingsAction", "LocalSettingsAction", + "WebServerModule", + "IWebServerRoutes", + "UserModule", "IUserModule", @@ -66,9 +68,6 @@ __all__ = ( "TimersManager", "ITimersManager", - "RestApiModule", - "IRestApi", - "AvalonModule", "LauncherAction", @@ -82,6 +81,5 @@ __all__ = ( "DeadlineModule", "StandAlonePublishAction", - "WebServerModule", "SyncServer" ) diff --git a/pype/modules/webserver/__init__.py b/pype/modules/webserver/__init__.py index ace27d94ab..defd115e57 100644 --- a/pype/modules/webserver/__init__.py +++ b/pype/modules/webserver/__init__.py @@ -1,6 +1,10 @@ -from .webserver_module import WebServerModule +from .webserver_module import ( + WebServerModule, + IWebServerRoutes +) __all__ = ( "WebServerModule", + "IWebServerRoutes" ) diff --git a/pype/modules/webserver/webserver_module.py b/pype/modules/webserver/webserver_module.py index 5ebfdbe2d2..7280292330 100644 --- a/pype/modules/webserver/webserver_module.py +++ b/pype/modules/webserver/webserver_module.py @@ -4,18 +4,31 @@ from pype import resources from .. import PypeModule, ITrayService +@six.add_metaclass(ABCMeta) +class IWebServerRoutes: + """Other modules interface to register their routes.""" + @abstractmethod + def webserver_initialization(self, server_manager): + pass + + class WebServerModule(PypeModule, ITrayService): name = "webserver" label = "WebServer" - def initialize(self, module_settings): + def initialize(self, _module_settings): self.enabled = True self.server_manager = None self.port = self.find_free_port() - def connect_with_modules(self, *_a, **_kw): - return + def connect_with_modules(self, enabled_modules): + if not self.server_manager: + return + + for module in enabled_modules: + if isinstance(module, IWebServerRoutes): + module.webserver_initialization(self.server_manager) def tray_init(self): self.create_server_manager() @@ -31,8 +44,10 @@ class WebServerModule(PypeModule, ITrayService): static_prefix = "/res" self.server_manager.add_static(static_prefix, resources.RESOURCES_DIR) - os.environ["PYPE_STATICS_SERVER"] = "http://localhost:{}{}".format( - self.port, static_prefix + webserver_url = "http://localhost:{}".format(self.port) + os.environ["PYPE_WEBSERVER_URL"] = webserver_url + os.environ["PYPE_STATICS_SERVER"] = "{}{}".format( + webserver_url, static_prefix ) def start_server(self): From cfd943869c0e8c1663d63417e210cc1876047ce4 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 12 Mar 2021 11:39:49 +0100 Subject: [PATCH 08/13] avalon module has same rest api methods like with rest api module --- pype/modules/avalon_apps/avalon_app.py | 12 +- pype/modules/avalon_apps/rest_api.py | 147 +++++++++++++++++++++++++ 2 files changed, 157 insertions(+), 2 deletions(-) create mode 100644 pype/modules/avalon_apps/rest_api.py diff --git a/pype/modules/avalon_apps/avalon_app.py b/pype/modules/avalon_apps/avalon_app.py index c61704db14..38d6e4394c 100644 --- a/pype/modules/avalon_apps/avalon_app.py +++ b/pype/modules/avalon_apps/avalon_app.py @@ -3,11 +3,12 @@ import pype from pype import resources from .. import ( PypeModule, - ITrayModule + ITrayModule, + IWebServerRoutes ) -class AvalonModule(PypeModule, ITrayModule): +class AvalonModule(PypeModule, ITrayModule, IWebServerRoutes): name = "avalon" def initialize(self, modules_settings): @@ -73,6 +74,13 @@ class AvalonModule(PypeModule, ITrayModule): def connect_with_modules(self, _enabled_modules): return + def webserver_initialization(self, server_manager): + """Implementation of IWebServerRoutes interface.""" + + if self.tray_initialized: + from .rest_api import AvalonRestApiResource + self.rest_api_obj = AvalonRestApiResource(self, server_manager) + # Definition of Tray menu def tray_menu(self, tray_menu): from Qt import QtWidgets diff --git a/pype/modules/avalon_apps/rest_api.py b/pype/modules/avalon_apps/rest_api.py new file mode 100644 index 0000000000..e41801984b --- /dev/null +++ b/pype/modules/avalon_apps/rest_api.py @@ -0,0 +1,147 @@ +import os +import re +import json +import datetime + +import bson +from bson.objectid import ObjectId +import bson.json_util + +from aiohttp.web_request import Request +from aiohttp.web_response import Response + +from avalon.api import AvalonMongoDB +from pype.modules.webserver.base_routes import RestApiEndpoint + + +class _RestApiEndpoint(RestApiEndpoint): + def __init__(self, resource): + self.resource = resource + super(_RestApiEndpoint, self).__init__() + + @property + def dbcon(self): + return self.resource.dbcon + + +class AvalonProjectsEndpoint(_RestApiEndpoint): + async def get(self) -> Response: + output = [] + for project_name in self.dbcon.database.collection_names(): + project_doc = self.dbcon.database[project_name].find_one({ + "type": "project" + }) + output.append(project_doc) + return Response( + status=200, + body=self.resource.encode(output), + content_type="application/json" + ) + + +class AvalonProjectEndpoint(_RestApiEndpoint): + async def get(self, project_name) -> Response: + project_doc = self.dbcon.database[project_name].find_one({ + "type": "project" + }) + if project_doc: + return Response( + status=200, + body=self.resource.encode(project_doc), + content_type="application/json" + ) + return Response( + status=404, + reason="Project name {} not found".format(project_name) + ) + + +class AvalonAssetsEndpoint(_RestApiEndpoint): + async def get(self, project_name) -> Response: + asset_docs = list(self.dbcon.database[project_name].find({ + "type": "asset" + })) + return Response( + status=200, + body=self.resource.encode(asset_docs), + content_type="application/json" + ) + + +class AvalonAssetEndpoint(_RestApiEndpoint): + async def get(self, project_name, asset_name) -> Response: + asset_doc = self.dbcon.database[project_name].find_one({ + "type": "asset", + "name": asset_name + }) + if asset_doc: + return Response( + status=200, + body=self.resource.encode(asset_doc), + content_type="application/json" + ) + return Response( + status=404, + reason="Asset name {} not found in project {}".format( + asset_name, project_name + ) + ) + + +class AvalonRestApiResource: + def __init__(self, avalon_module, server_manager): + self.module = avalon_module + self.server_manager = server_manager + + self.dbcon = AvalonMongoDB() + self.dbcon.install() + + self.prefix = "/avalon" + + self.endpoint_defs = ( + ( + "GET", + "/projects", + AvalonProjectsEndpoint(self) + ), + ( + "GET", + "/projects/{project_name}", + AvalonProjectEndpoint(self) + ), + ( + "GET", + "/projects/{project_name}/assets", + AvalonAssetsEndpoint(self) + ), + ( + "GET", + "/projects/{project_name}/assets/{asset_name}", + AvalonAssetEndpoint(self) + ) + ) + + self.register() + + def register(self): + for methods, url, endpoint in self.endpoint_defs: + final_url = self.prefix + url + self.server_manager.add_route( + methods, final_url, endpoint.dispatch + ) + + @staticmethod + def json_dump_handler(value): + if isinstance(value, datetime.datetime): + return value.isoformat() + if isinstance(value, ObjectId): + return str(value) + raise TypeError(value) + + @classmethod + def encode(cls, data): + return json.dumps( + data, + indent=4, + default=cls.json_dump_handler + ).encode("utf-8") From aea8d1c2ffdae5a0cf6607f20cf9278e8999d512 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 12 Mar 2021 11:41:47 +0100 Subject: [PATCH 09/13] preimplemented helper class for endpoint --- pype/modules/webserver/base_routes.py | 51 ++++++++++++++++++++++ pype/modules/webserver/webserver_module.py | 4 ++ 2 files changed, 55 insertions(+) create mode 100644 pype/modules/webserver/base_routes.py diff --git a/pype/modules/webserver/base_routes.py b/pype/modules/webserver/base_routes.py new file mode 100644 index 0000000000..8730e133c8 --- /dev/null +++ b/pype/modules/webserver/base_routes.py @@ -0,0 +1,51 @@ +"""Helper functions or classes for Webserver module. + +These must not be imported in module itself to not break Python 2 +applications. +""" + +import json +import inspect +from aiohttp.http_exceptions import HttpBadRequest +from aiohttp.web_exceptions import HTTPMethodNotAllowed +from aiohttp.web_request import Request + + +DEFAULT_METHODS = ("GET", "POST", "PUT", "DELETE") + + +class RestApiEndpoint: + """Helper endpoint class for single endpoint. + + Class can define `get`, `post`, `put` or `delete` async methods for the + endpoint. + """ + def __init__(self): + methods = {} + + for method_name in DEFAULT_METHODS: + method = getattr(self, method_name.lower(), None) + if method: + methods[method_name.upper()] = method + + self.methods = methods + + async def dispatch(self, request: Request): + method = self.methods.get(request.method.upper()) + if not method: + raise HTTPMethodNotAllowed("", DEFAULT_METHODS) + + wanted_args = list(inspect.signature(method).parameters.keys()) + + available_args = request.match_info.copy() + available_args["request"] = request + + unsatisfied_args = set(wanted_args) - set(available_args.keys()) + if unsatisfied_args: + # Expected match info that doesn't exist + raise HttpBadRequest("") + + return await method(**{ + arg_name: available_args[arg_name] + for arg_name in wanted_args + }) diff --git a/pype/modules/webserver/webserver_module.py b/pype/modules/webserver/webserver_module.py index 7280292330..3b3f0e7a79 100644 --- a/pype/modules/webserver/webserver_module.py +++ b/pype/modules/webserver/webserver_module.py @@ -1,5 +1,9 @@ import os import socket +from abc import ABCMeta, abstractmethod + +import six + from pype import resources from .. import PypeModule, ITrayService From d6567358de5c4f3da1b6008c3618c4f6ea0fb661 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 12 Mar 2021 11:42:14 +0100 Subject: [PATCH 10/13] user module has same rest api methods like before --- pype/modules/user/rest_api.py | 36 ++++++++++++++++++++++++++++++++ pype/modules/user/user_module.py | 11 +++++++++- 2 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 pype/modules/user/rest_api.py diff --git a/pype/modules/user/rest_api.py b/pype/modules/user/rest_api.py new file mode 100644 index 0000000000..5c872925e0 --- /dev/null +++ b/pype/modules/user/rest_api.py @@ -0,0 +1,36 @@ +import json +from aiohttp.web_request import Request +from aiohttp.web_response import Response + + +class UserModuleRestApi: + def __init__(self, user_module, server_manager): + self.module = user_module + self.server_manager = server_manager + + self.prefix = "/user" + + self.register() + + def register(self): + self.server_manager.add_route( + "GET", + self.prefix + "/username", + self.get_username + ) + self.server_manager.add_route( + "GET", + self.prefix + "/show_widget", + self.show_user_widget + ) + + async def get_username(self, request): + return Response( + status=200, + body=json.dumps(self.module.cred, indent=4), + content_type="application/json" + ) + + async def show_user_widget(self, request): + self.module.action_show_widget.trigger() + return Response(status=200) diff --git a/pype/modules/user/user_module.py b/pype/modules/user/user_module.py index 0340a2e6eb..4a8d11a2f9 100644 --- a/pype/modules/user/user_module.py +++ b/pype/modules/user/user_module.py @@ -9,7 +9,8 @@ import appdirs from .. import ( PypeModule, - ITrayModule + ITrayModule, + IWebServerRoutes ) @@ -46,6 +47,8 @@ class UserModule(PypeModule, ITrayModule): self.widget_login = None self.action_show_widget = None + self.rest_api_obj = None + def tray_init(self): from .widget_user import UserWidget self.widget_login = UserWidget(self) @@ -71,6 +74,12 @@ class UserModule(PypeModule, ITrayModule): def get_user(self): return self.cred.get("username") or getpass.getuser() + def webserver_initialization(self, server_manager): + """Implementation of IWebServerRoutes interface.""" + from .rest_api import UserModuleRestApi + + self.rest_api_obj = UserModuleRestApi(self, server_manager) + def connect_with_modules(self, enabled_modules): for module in enabled_modules: if isinstance(module, IUserModule): From 5c3465ddeb3a4ad133aaa52c50c58e96bd1ac92d Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 12 Mar 2021 11:42:45 +0100 Subject: [PATCH 11/13] modified env variable in premiere --- pype/hosts/premiere/lib.py | 2 +- pype/hosts/premiere/ppro/js/pype_restapi_client.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pype/hosts/premiere/lib.py b/pype/hosts/premiere/lib.py index 5282e2c747..410e159560 100644 --- a/pype/hosts/premiere/lib.py +++ b/pype/hosts/premiere/lib.py @@ -168,7 +168,7 @@ def clearing_caches_ui(): def test_rest_api_server(env): # from pprint import pformat - rest_url = env.get("PYPE_REST_API_URL") + rest_url = env.get("PYPE_WEBSERVER_URL") project_name = "{AVALON_PROJECT}".format(**env) URL = "/".join((rest_url, "avalon/projects", diff --git a/pype/hosts/premiere/ppro/js/pype_restapi_client.js b/pype/hosts/premiere/ppro/js/pype_restapi_client.js index b9a5ec9425..be73a2fb8c 100644 --- a/pype/hosts/premiere/ppro/js/pype_restapi_client.js +++ b/pype/hosts/premiere/ppro/js/pype_restapi_client.js @@ -14,7 +14,7 @@ class PypeRestApiClient { * @return {url string} */ _getApiServerUrl() { - var url = this.env.PYPE_REST_API_URL; + var url = this.env.PYPE_WEBSERVER_URL; return url } From 5252bb006fa4a9ba21a7726a05366094cf249a78 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 12 Mar 2021 11:43:22 +0100 Subject: [PATCH 12/13] muster has same rest api methods as before but changed POST for show widget to GET --- .../maya/plugins/create/create_render.py | 4 ++-- .../maya/plugins/create/create_vrayscene.py | 4 ++-- .../publish/validate_muster_connection.py | 4 ++-- pype/modules/muster/muster.py | 13 +++++++++-- pype/modules/muster/rest_api.py | 22 +++++++++++++++++++ 5 files changed, 39 insertions(+), 8 deletions(-) create mode 100644 pype/modules/muster/rest_api.py diff --git a/pype/hosts/maya/plugins/create/create_render.py b/pype/hosts/maya/plugins/create/create_render.py index 726433de92..1d06cf7080 100644 --- a/pype/hosts/maya/plugins/create/create_render.py +++ b/pype/hosts/maya/plugins/create/create_render.py @@ -274,9 +274,9 @@ class CreateRender(plugin.Creator): # authentication token expired so we need to login to Muster # again to get it. We use Pype API call to show login window. api_url = "{}/muster/show_login".format( - os.environ["PYPE_REST_API_URL"]) + os.environ["PYPE_WEBSERVER_URL"]) self.log.debug(api_url) - login_response = self._requests_post(api_url, timeout=1) + login_response = self._requests_get(api_url, timeout=1) if login_response.status_code != 200: self.log.error("Cannot show login form to Muster") raise Exception("Cannot show login form to Muster") diff --git a/pype/hosts/maya/plugins/create/create_vrayscene.py b/pype/hosts/maya/plugins/create/create_vrayscene.py index 5690fb9f51..f37a6c4e20 100644 --- a/pype/hosts/maya/plugins/create/create_vrayscene.py +++ b/pype/hosts/maya/plugins/create/create_vrayscene.py @@ -191,9 +191,9 @@ class CreateVRayScene(plugin.Creator): # authentication token expired so we need to login to Muster # again to get it. We use Pype API call to show login window. api_url = "{}/muster/show_login".format( - os.environ["PYPE_REST_API_URL"]) + os.environ["PYPE_WEBSERVER_URL"]) self.log.debug(api_url) - login_response = self._requests_post(api_url, timeout=1) + login_response = self._requests_get(api_url, timeout=1) if login_response.status_code != 200: self.log.error("Cannot show login form to Muster") raise Exception("Cannot show login form to Muster") diff --git a/pype/hosts/maya/plugins/publish/validate_muster_connection.py b/pype/hosts/maya/plugins/publish/validate_muster_connection.py index ad1022c71d..868135677e 100644 --- a/pype/hosts/maya/plugins/publish/validate_muster_connection.py +++ b/pype/hosts/maya/plugins/publish/validate_muster_connection.py @@ -85,9 +85,9 @@ class ValidateMusterConnection(pyblish.api.ContextPlugin): Renew authentication token by logging into Muster """ api_url = "{}/muster/show_login".format( - os.environ["PYPE_REST_API_URL"]) + os.environ["PYPE_WEBSERVER_URL"]) cls.log.debug(api_url) - response = cls._requests_post(api_url, timeout=1) + response = cls._requests_get(api_url, timeout=1) if response.status_code != 200: cls.log.error('Cannot show login form to Muster') raise Exception('Cannot show login form to Muster') diff --git a/pype/modules/muster/muster.py b/pype/modules/muster/muster.py index 4f363b923e..5595ccff15 100644 --- a/pype/modules/muster/muster.py +++ b/pype/modules/muster/muster.py @@ -4,11 +4,12 @@ import appdirs import requests from .. import ( PypeModule, - ITrayModule + ITrayModule, + IWebServerRoutes ) -class MusterModule(PypeModule, ITrayModule): +class MusterModule(PypeModule, ITrayModule, IWebServerRoutes): """ Module handling Muster Render credentials. This will display dialog asking for user credentials for Muster if not already specified. @@ -31,6 +32,7 @@ class MusterModule(PypeModule, ITrayModule): # Tray attributes self.widget_login = None self.action_show_login = None + self.rest_api_obj = None def get_global_environments(self): return { @@ -74,6 +76,13 @@ class MusterModule(PypeModule, ITrayModule): parent.addMenu(menu) + def webserver_initialization(self, server_manager): + """Implementation of IWebServerRoutes interface.""" + if self.tray_initialized: + from .rest_api import MusterModuleRestApi + + self.rest_api_obj = MusterModuleRestApi(self, server_manager) + def load_credentials(self): """ Get credentials from JSON file diff --git a/pype/modules/muster/rest_api.py b/pype/modules/muster/rest_api.py new file mode 100644 index 0000000000..bea759919d --- /dev/null +++ b/pype/modules/muster/rest_api.py @@ -0,0 +1,22 @@ +from aiohttp.web_response import Response + + +class MusterModuleRestApi: + def __init__(self, user_module, server_manager): + self.module = user_module + self.server_manager = server_manager + + self.prefix = "/muster" + + self.register() + + def register(self): + self.server_manager.add_route( + "GET", + self.prefix + "/show_login", + self.show_login_widget + ) + + async def show_login_widget(self, request): + self.module.action_show_login.trigger() + return Response(status=200) From 939d17676cac1c5eec68bc3c2672ce43d0ff85f3 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 12 Mar 2021 12:54:17 +0100 Subject: [PATCH 13/13] hound fixes --- pype/modules/avalon_apps/rest_api.py | 1 - pype/modules/user/rest_api.py | 1 - pype/modules/user/user_module.py | 2 +- pype/modules/webserver/base_routes.py | 1 - 4 files changed, 1 insertion(+), 4 deletions(-) diff --git a/pype/modules/avalon_apps/rest_api.py b/pype/modules/avalon_apps/rest_api.py index e41801984b..459103594a 100644 --- a/pype/modules/avalon_apps/rest_api.py +++ b/pype/modules/avalon_apps/rest_api.py @@ -7,7 +7,6 @@ import bson from bson.objectid import ObjectId import bson.json_util -from aiohttp.web_request import Request from aiohttp.web_response import Response from avalon.api import AvalonMongoDB diff --git a/pype/modules/user/rest_api.py b/pype/modules/user/rest_api.py index 5c872925e0..566425a19b 100644 --- a/pype/modules/user/rest_api.py +++ b/pype/modules/user/rest_api.py @@ -1,5 +1,4 @@ import json -from aiohttp.web_request import Request from aiohttp.web_response import Response diff --git a/pype/modules/user/user_module.py b/pype/modules/user/user_module.py index 4a8d11a2f9..71c5fd124e 100644 --- a/pype/modules/user/user_module.py +++ b/pype/modules/user/user_module.py @@ -24,7 +24,7 @@ class IUserModule: pass -class UserModule(PypeModule, ITrayModule): +class UserModule(PypeModule, ITrayModule, IWebServerRoutes): cred_folder_path = os.path.normpath( appdirs.user_data_dir('pype-app', 'pype') ) diff --git a/pype/modules/webserver/base_routes.py b/pype/modules/webserver/base_routes.py index 8730e133c8..f4f1abe16c 100644 --- a/pype/modules/webserver/base_routes.py +++ b/pype/modules/webserver/base_routes.py @@ -4,7 +4,6 @@ These must not be imported in module itself to not break Python 2 applications. """ -import json import inspect from aiohttp.http_exceptions import HttpBadRequest from aiohttp.web_exceptions import HTTPMethodNotAllowed