From 1298bf73bfb1231612507f77c0c884cd4292cb65 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 14 Oct 2019 19:07:52 +0200 Subject: [PATCH 01/15] initial commit of rest api module with partially working rest api handling --- pype/services/rest_api/__init__.py | 11 + pype/services/rest_api/base_class.py | 68 ++++ pype/services/rest_api/lib/__init__.py | 19 ++ pype/services/rest_api/lib/exceptions.py | 10 + pype/services/rest_api/lib/factory.py | 266 +++++++++++++++ pype/services/rest_api/lib/handler.py | 399 +++++++++++++++++++++++ pype/services/rest_api/lib/lib.py | 140 ++++++++ pype/services/rest_api/rest_api.py | 301 ++++------------- 8 files changed, 984 insertions(+), 230 deletions(-) create mode 100644 pype/services/rest_api/base_class.py create mode 100644 pype/services/rest_api/lib/__init__.py create mode 100644 pype/services/rest_api/lib/exceptions.py create mode 100644 pype/services/rest_api/lib/factory.py create mode 100644 pype/services/rest_api/lib/handler.py create mode 100644 pype/services/rest_api/lib/lib.py diff --git a/pype/services/rest_api/__init__.py b/pype/services/rest_api/__init__.py index c11ecfd761..f0fddce0ee 100644 --- a/pype/services/rest_api/__init__.py +++ b/pype/services/rest_api/__init__.py @@ -1,4 +1,15 @@ from .rest_api import RestApiServer +from .base_class import RestApi, abort, route, register_statics +from .lib import ( + RestMethods, + UrlData, + RequestData, + Query, + Fragment, + Params, + Handler, + CallbackResult +) def tray_init(tray_widget, main_widget): diff --git a/pype/services/rest_api/base_class.py b/pype/services/rest_api/base_class.py new file mode 100644 index 0000000000..94c7eb0bbd --- /dev/null +++ b/pype/services/rest_api/base_class.py @@ -0,0 +1,68 @@ +from functools import wraps +from http import HTTPStatus + +from .lib import ( + RestApiFactory, Splitter, + ObjAlreadyExist, AbortException, + Params, UrlData, RequestData, Query, Fragment, Handler +) + + +def route(path, url_prefix="", methods=[]): + def decorator(callback): + @wraps(callback) + def wrapper(*args, **kwargs): + return callback(*args, **kwargs) + + func = wrapper + func.restapi = True + func.path = path + func.methods = methods + func.url_prefix = url_prefix + if hasattr(callback, "__self__"): + func.__self__ = callback.__self__ + func.callback = callback + + RestApiFactory.register_route(func) + return func + return decorator + + +def register_statics(url_prefix, dir_path): + RestApiFactory.register_statics((url_prefix, dir_path)) + + +def abort(status_code=HTTPStatus.NOT_FOUND, message=None): + items = [] + items.append(str(status_code)) + if not message: + message = "" + + items.append(message) + + raise AbortException(Splitter.join(items)) + + +class RestApi: + def route(path, url_prefix="", methods=[]): + return route(path, url_prefix, methods) + + @classmethod + def register_route(cls, callback, path, url_prefix="", methods=[]): + return route(path, methods, url_prefix)(callback) + + @classmethod + def register_statics(cls, url_prefix, dir_path): + return register_statics(url_prefix, dir_path) + + @classmethod + def abort(cls, status_code=HTTPStatus.NOT_FOUND, message=None): + abort(status_code, message) + + def __new__(cls, *args, **kwargs): + for obj in RestApiFactory.registered_objs: + if type(obj) == cls: + raise ObjAlreadyExist(cls) + instance = super(RestApi, cls).__new__(cls) + RestApiFactory.register_obj(instance) + return instance diff --git a/pype/services/rest_api/lib/__init__.py b/pype/services/rest_api/lib/__init__.py new file mode 100644 index 0000000000..38d476050a --- /dev/null +++ b/pype/services/rest_api/lib/__init__.py @@ -0,0 +1,19 @@ +Splitter = "__splitter__" + +from .exceptions import ObjAlreadyExist, AbortException +from .lib import ( + RestMethods, + CustomNone, + UrlData, + RequestData, + Query, + Fragment, + Params, + CallbackResult +) + +from .factory import _RestApiFactory + +RestApiFactory = _RestApiFactory() + +from .handler import Handler diff --git a/pype/services/rest_api/lib/exceptions.py b/pype/services/rest_api/lib/exceptions.py new file mode 100644 index 0000000000..849b522ddd --- /dev/null +++ b/pype/services/rest_api/lib/exceptions.py @@ -0,0 +1,10 @@ +class ObjAlreadyExist(Exception): + def __init__(self, cls=None, message=None): + if not (cls and message): + message = "RestApi object was created twice." + elif not message: + message = "{} object was created twice.".format(cls.__name__) + super().__init__(message) + + +class AbortException(Exception): pass diff --git a/pype/services/rest_api/lib/factory.py b/pype/services/rest_api/lib/factory.py new file mode 100644 index 0000000000..dbbd6ec5af --- /dev/null +++ b/pype/services/rest_api/lib/factory.py @@ -0,0 +1,266 @@ +import os +import re +import inspect +import collections +from .lib import RestMethods + +from pypeapp import Logger + +log = Logger().get_logger("RestApiFactory") + + +def prepare_fullpath(path, prefix): + if path and prefix: + fullpath = "{}/{}".format(prefix, path).replace("//", "/") + elif path: + fullpath = path + elif prefix: + fullpath = prefix + else: + fullpath = "/" + + if not fullpath.startswith("/"): + fullpath = "/{}".format(fullpath) + + return fullpath + + +def prepare_regex_from_path(full_path): + get_indexes_regex = "<[^< >]+>" + all_founded_keys = re.findall(get_indexes_regex, full_path) + if not all_founded_keys: + return None, None + + regex_path = full_path + keys = [] + for key in all_founded_keys: + replacement = "(?P{}\w+)".format(key) + keys.append(key.replace("<", "").replace(">", "")) + if full_path.endswith(key): + replacement = "?{}?".format(replacement) + regex_path = regex_path.replace(key, replacement) + + regex_path = "^{}$".format(regex_path) + + return re.compile(regex_path), keys + + +def prepare_prefix(url_prefix): + if url_prefix is None: + url_prefix = "" + elif isinstance(url_prefix, (list, tuple)): + url_prefix = "/".join(url_prefix) + else: + items = [part for part in url_prefix.split("/") if part] + url_prefix = "/".join(items) + + if not url_prefix: + return None + + if not url_prefix.startswith("/"): + url_prefix = "/{}".format(url_prefix) + + return url_prefix + + +def prepare_methods(methods, callback=None): + invalid_methods = collections.defaultdict(list) + + if not methods: + _methods = [RestMethods.GET] + elif isinstance(methods, str) or isinstance(methods, RestMethods): + _method = RestMethods.get(methods) + _methods = [] + if _method is None: + invalid_methods[methods].append(callback) + else: + _methods.append(_method) + + else: + _methods = [] + for method in methods: + found = False + _method = RestMethods.get(method) + if _method == None: + invalid_methods[methods].append(callback) + continue + + _methods.append(_method) + + for method, callbacks in invalid_methods.items(): + callback_info = "" + + callbacks = [cbk for cbk in callbacks if cbk] + if len(callbacks) > 0: + multiple_ind = "" + if len(callbacks) > 1: + multiple_ind = "s" + + callback_items = [] + for callback in callbacks: + callback_items.append("\"{}<{}>\"".format( + callback.__qualname__, callback.__globals__["__file__"] + )) + + callback_info = " with callback{} {}".format( + multiple_ind, "| ".join(callback_items) + ) + + log.warning( + ("Invalid RestApi method \"{}\"{}").format(method, callback_info) + ) + + return _methods + +def prepare_callback_info(_callback): + callback = _callback.callback + callback_info = inspect.getfullargspec(callback) + + callback_args = callback_info.args + callback_args_len = 0 + if callback_args: + callback_args_len = len(callback_args) + if ( + type(_callback).__name__ == "method" + ): + callback_args_len -= 1 + + defaults = callback_info.defaults + defaults_len = 0 + if defaults: + defaults_len = len(defaults) + + annotations = callback_info.annotations + + return { + "args": callback_args, + "args_len": callback_args_len, + "defaults": defaults, + "defaults_len": defaults_len, + "hasargs": callback_info.varargs is not None, + "haskwargs": callback_info.varkw is not None, + "annotations": annotations + } + + +class _RestApiFactory: + registered_objs = [] + unprocessed_routes = [] + unprocessed_statics = [] + + prepared_routes = { + method: collections.defaultdict(list) for method in RestMethods + } + prepared_statics = {} + + has_routes = False + + def has_handlers(self): + return (self.has_routes or self.prepared_statics) + + def _process_route(self, callback): + return self.unprocessed_routes.pop( + self.unprocessed_routes.index(callback) + ) + + def _process_statics(self, item): + return self.unprocessed_statics.pop( + self.unprocessed_statics.index(item) + ) + + def register_route(self, item): + log.debug("Registering callback for item \"{}\"".format( + item.__qualname__ + )) + self.unprocessed_routes.append(item) + + def register_obj(self, obj): + self.registered_objs.append(obj) + + def register_statics(self, item): + log.debug("Registering statics path \"{}\"".format(item)) + self.unprocessed_statics.append(item) + + def _prepare_route(self, callback): + methods = prepare_methods(callback.methods, callback) + url_prefix = prepare_prefix(callback.url_prefix) + fullpath = prepare_fullpath(callback.path, url_prefix) + regex, regex_keys = prepare_regex_from_path(fullpath) + callback_info = prepare_callback_info(callback) + + for method in methods: + self.has_routes = True + self.prepared_routes[method][url_prefix].append({ + "regex": regex, + "regex_keys": regex_keys, + "fullpath": fullpath, + "callback": callback, + "callback_info": callback_info + }) + + def prepare_registered(self): + for url_prefix, dir_path in self.unprocessed_statics: + self._process_statics((url_prefix, dir_path)) + dir_path = os.path.normpath(dir_path) + if not os.path.exists(dir_path): + log.warning( + "Directory path \"{}\" was not found".format(dir_path) + ) + continue + url_prefix = prepare_prefix(url_prefix) + self.prepared_statics[url_prefix] = dir_path + + for obj in self.registered_objs: + method_names = [ + attr for attr in dir(obj) + if inspect.ismethod(getattr(obj, attr)) + ] + for method_name in method_names: + method = obj.__getattribute__(method_name) + + for callback in self.unprocessed_routes: + if not ( + callback.__qualname__ == method.__qualname__ and + callback.__module__ == method.__module__ and + callback.__globals__["__file__"] == method.__globals__["__file__"] + ): + continue + + self._process_route(callback) + + if not hasattr(method, "restapi"): + continue + + if not method.restapi: + continue + + self._prepare_route(method) + break + + for callback in self.unprocessed_routes: + is_class_method = len(callback.__qualname__.split(".")) != 1 + if is_class_method: + missing_self = True + if hasattr(callback, "__self__"): + if callback.__self__ is not None: + missing_self = False + + if "" in callback.__qualname__: + pass + + elif missing_self: + log.warning(( + "Object of callback \"{}\" from \"{}\" is not" + " accessible for api. Register object or" + " register callback with already created object" + "(not with decorator in class).".format( + callback.__qualname__, + callback.__globals__["__file__"] + ) + )) + continue + + self._prepare_route(callback) + continue + + self._prepare_route(callback) diff --git a/pype/services/rest_api/lib/handler.py b/pype/services/rest_api/lib/handler.py new file mode 100644 index 0000000000..71215c1f2c --- /dev/null +++ b/pype/services/rest_api/lib/handler.py @@ -0,0 +1,399 @@ +import os +import re +import json +import traceback +import http.server +from http import HTTPStatus +from urllib.parse import urlparse + +from .lib import ( + RestMethods, CallbackResult, + UrlData, RequestData, Query, Fragment, Params +) +from .exceptions import AbortException +from . import RestApiFactory, CustomNone, Splitter + +from pypeapp import Logger + +log = Logger().get_logger("RestApiHandler") +NotSet = CustomNone("NotSet") + + +class Handler(http.server.SimpleHTTPRequestHandler): + # TODO fill will necessary statuses + default_messages = { + HTTPStatus.BAD_REQUEST: "Bad request", + HTTPStatus.NOT_FOUND: "Not found" + } + + statuses = { + "POST": { + "OK": 200, + "CREATED": 201 + }, + "PUT": { + "OK": 200, + "NO_CONTENT": 204 + } + } + def do_GET(self): + return self._handle_request(RestMethods.GET) + + def do_POST(self): + return self._handle_request(RestMethods.POST) + + def do_PUT(self): + return self._handle_request(RestMethods.PUT) + + def do_DELETE(self): + return self._handle_request(RestMethods.DELETE) + + def do_PATCH(self): + return self._handle_request(RestMethods.PATCH) + + def _handle_request(self, rest_method): + """Because processing is technically the same for now so it is used + the same way + """ + parsed_url = urlparse(self.path) + path = parsed_url.path + + if rest_method is RestMethods.GET: + for prefix, dirpath in RestApiFactory.prepared_statics.items(): + if not path.startswith(prefix): + continue + _path = path[len(prefix):] + return self._handle_statics(dirpath, _path) + + matching_item = None + found_prefix = None + url_prefixes = RestApiFactory.prepared_routes[rest_method] + for url_prefix, items in url_prefixes.items(): + if matching_item is not None: + break + + if url_prefix is not None: + if not path.startswith(url_prefix): + continue + + found_prefix = url_prefix + + for item in items: + regex = item["regex"] + item_full_path = item["fullpath"] + if regex is None: + if path == item_full_path: + item["url_data"] = None + matching_item = item + break + + else: + found = re.match(regex, path) + if found: + item["url_data"] = found.groupdict() + matching_item = item + break + + if not matching_item: + if found_prefix is not None: + _path = path.replace(found_prefix, "") + if _path: + request_str = " \"{}\"".format(_path) + else: + request_str = "" + + message = "Invalid path request{} for prefix \"{}\"".format( + request_str, found_prefix + ) + else: + message = "Invalid path request \"{}\"".format(self.path) + log.debug(message) + self.send_error(HTTPStatus.BAD_REQUEST, message) + + return + + try: + log.debug("Triggering callback for path \"{}\"".format(path)) + + result = self._handle_callback(matching_item, parsed_url) + + return self._handle_callback_result(result, rest_method) + + except AbortException as exc: + status_code, message = str(exc).split(Splitter) + status_code = int(status_code) + if not message: + message = self.default_messages.get( + status_code, "UnexpectedError" + ) + + self.send_response(status_code) + self.send_header("Content-type", "text/html") + self.send_header("Content-Length", len(message)) + self.end_headers() + + self.wfile.write(message.encode()) + return message + + except Exception as exc: + log_message = "Unexpected Exception was raised (this is bug!)" + log.error(log_message, exc_info=True) + replace_helper = 0 + items = [log_message] + items += traceback.extract_tb(exc.__traceback__).format() + message = "\n".join(items) + + self.send_response(HTTPStatus.INTERNAL_SERVER_ERROR) + self.send_header("Content-type", "text/html") + self.send_header("Content-Length", len(message)) + self.end_headers() + + self.wfile.write(message.encode()) + return message + + + def _handle_callback_result(self, result, rest_method): + content_type = "application/json" + status = HTTPStatus.OK + success = True + message = None + data = None + + body = None + # TODO better handling of results + if isinstance(result, CallbackResult): + status = result.status_code + body_dict = {} + for key, value in result.items(): + if value is not None: + body_dict[key] = value + body = json.dumps(body_dict) + + elif result in [None, True]: + status = HTTPStatus.OK + success = True + message = "{} request for \"{}\" passed".format( + rest_method, self.path + ) + + elif result is False: + status = HTTPStatus.BAD_REQUEST + success = False + + elif isinstance(result, (dict, list)): + status = HTTPStatus.OK + data = result + + if status == HTTPStatus.NO_CONTENT: + self.send_response(status) + self.end_headers() + return + + if not body: + body_dict = {"success": success} + if message: + body_dict["message"] = message + + if not data: + data = {} + + body_dict["data"] = data + body = json.dumps(body_dict) + + self.send_response(status) + self.send_header("Content-type", content_type) + self.send_header("Content-Length", len(body)) + self.end_headers() + + self.wfile.write(body.encode()) + return body + + def _handle_callback(self, item, parsed_url): + + regex = item["regex"] + regex_keys = item["regex_keys"] + + url_data = None + if regex_keys: + url_data = {key: None for key in regex_keys} + if item["url_data"]: + for key, value in item["url_data"].items(): + url_data[key] = value + + in_data = None + cont_len = self.headers.get("Content-Length") + if cont_len: + content_length = int(cont_len) + in_data_str = self.rfile.read(content_length) + if in_data_str: + in_data = json.loads(in_data_str) + + url_data = UrlData(url_data) + request_data = RequestData(in_data) + query = Query(parsed_url.query) + params = Params(parsed_url.params) + fragment = Fragment(parsed_url.fragment) + + callback = item["callback"] + callback_info = item["callback_info"] + + _args = callback_info["args"] + _args_len = callback_info["args_len"] + _defaults = callback_info["defaults"] + _defaults_len = callback_info["defaults_len"] + _has_args = callback_info["hasargs"] + _has_kwargs = callback_info["haskwargs"] + + _annotations = callback_info["annotations"] + + arg_index = 0 + if type(callback).__name__ == "method": + arg_index = 1 + + _kwargs = {arg: NotSet for arg in _args[arg_index:]} + _available_kwargs = { + "url_data": url_data, + "request_data": request_data, + "query": query, + "params": params, + "fragment": fragment, + "handler": self + } + if not regex: + _available_kwargs.pop("url_data") + + available_len = len(_available_kwargs) + if available_len < (_args_len - _defaults_len): + raise Exception(( + "Callback expects {} required positional arguments but {} are" + " available {}<{}>" + ).format( + _args_len, available_len, callback.__qualname__, + callback.__globals__.get("__file__", "unknown file") + )) + elif available_len < _args_len: + log.warning(( + "Handler \"{}\" will never fill all args of callback {}<{}>" + ).format( + self.__class__.__name__, + callback.__qualname__, + callback.__globals__.get("__file__", "unknown file") + )) + + if _args_len == 0: + if _has_args: + return callback(*_available_kwargs.values()) + if _has_kwargs: + return callback(**_available_kwargs) + else: + return callback() + + if _annotations: + for arg, argtype in _annotations.items(): + if argtype == Query: + key = "query" + elif argtype == Params: + key = "params" + elif argtype == Fragment: + key = "fragment" + elif argtype == Handler: + key = "handler" + elif argtype == UrlData: + key = "url_data" + elif argtype == RequestData: + key = "request_data" + else: + continue + + _kwargs[arg] = _available_kwargs[key] + _available_kwargs[key] = NotSet + + for key1, value1 in _kwargs.items(): + if value1 is not NotSet: + continue + has_values = False + for key2, value2 in _available_kwargs.items(): + if value2 is NotSet: + continue + has_values = True + _kwargs[key1] = value2 + _available_kwargs[key2] = NotSet + break + + if not has_values: + break + + _args = [] + kw_keys = [key for key in _kwargs.keys()] + for key in kw_keys: + value = _kwargs.pop(key) + if value is not NotSet: + _args.append(value) + continue + + if _has_args: + for key, value in _available_kwargs.items(): + if value is NotSet: + continue + _args.append(value) + + elif _has_kwargs: + for key, value in _available_kwargs.items(): + if value is NotSet: + continue + _kwargs[key] = value + + return callback(*_args, **_kwargs) + + def _handle_statics(self, dirpath, path): + path = os.path.normpath(dirpath + path) + + ctype = self.guess_type(path) + try: + file_obj = open(path, "rb") + except OSError: + self.send_error(HTTPStatus.NOT_FOUND, "File not found") + return None + + try: + file_stat = os.fstat(file_obj.fileno()) + # Use browser cache if possible + if ("If-Modified-Since" in self.headers + and "If-None-Match" not in self.headers): + # compare If-Modified-Since and time of last file modification + try: + ims = http.server.email.utils.parsedate_to_datetime( + self.headers["If-Modified-Since"]) + except (TypeError, IndexError, OverflowError, ValueError): + # ignore ill-formed values + pass + else: + if ims.tzinfo is None: + # obsolete format with no timezone, cf. + # https://tools.ietf.org/html/rfc7231#section-7.1.1.1 + ims = ims.replace(tzinfo=datetime.timezone.utc) + if ims.tzinfo is datetime.timezone.utc: + # compare to UTC datetime of last modification + last_modif = datetime.datetime.fromtimestamp( + file_stat.st_mtime, datetime.timezone.utc) + # remove microseconds, like in If-Modified-Since + last_modif = last_modif.replace(microsecond=0) + + if last_modif <= ims: + self.send_response(HTTPStatus.NOT_MODIFIED) + self.end_headers() + file_obj.close() + return None + + self.send_response(HTTPStatus.OK) + self.send_header("Content-type", ctype) + self.send_header("Content-Length", str(file_stat[6])) + self.send_header("Last-Modified", + self.date_time_string(file_stat.st_mtime)) + self.end_headers() + self.wfile.write(file_obj.read()) + return file_obj + except: + self.log.error("Failed to read data from file \"{}\"".format(path)) + finally: + file_obj.close() diff --git a/pype/services/rest_api/lib/lib.py b/pype/services/rest_api/lib/lib.py new file mode 100644 index 0000000000..4a9e41ce26 --- /dev/null +++ b/pype/services/rest_api/lib/lib.py @@ -0,0 +1,140 @@ +import os +import re +import enum +from http import HTTPStatus +from urllib.parse import urlencode, parse_qs + +from pypeapp import Logger + +log = Logger().get_logger("RestApiServer") + + +class RestMethods(enum.Enum): + GET = "GET" + POST = "POST" + PUT = "PUT" + PATCH = "PATCH" + DELETE = "DELETE" + + def __eq__(self, other): + if isinstance(other, self.__class__): + return self.value == other.value + + elif isinstance(other, str): + return self.value.lower() == other.lower() + return self.value == other + + def __hash__(self): + return enum.Enum.__hash__(self) + + @classmethod + def get(cls, name, default=None): + for meth in cls: + if meth == name: + return meth + return default + + +class CustomNone: + def __init__(self, name): + self._name = name + + def __bool__(self): + return False + + def __eq__(self, other): + if type(other) == type(self): + if other._name == self._name: + return True + return False + + def __str__(self): + return self._name + + def __repr__(self): + return self._name + + +class HandlerDict(dict): + def __init__(self, data=None, *args, **kwargs): + if not data: + data = {} + super().__init__(data, *args, **kwargs) + + def __repr__(self): + return "<{}> {}".format(self.__class__.__name__, str(dict(self))) + +class Params(HandlerDict): pass +class UrlData(HandlerDict): pass +class RequestData(HandlerDict): pass + +class Query(HandlerDict): + def __init__(self, query): + if isinstance(query, dict): + pass + else: + query = parse_qs(query) + super().__init__(query) + + def get_string(self): + return urlencode(dict(self), doseq=True) + +class Fragment(HandlerDict): + def __init__(self, fragment): + if isinstance(fragment, dict): + _fragment = fragment + else: + _fragment = {} + for frag in fragment.split("&"): + if not frag: + continue + items = frag.split("=") + + value = None + key = items[0] + if len(items) == 2: + value = items[1] + elif len(items) > 2: + value = "=".join(items[1:]) + + _fragment[key] = value + + super().__init__(_fragment) + + def get_string(self): + items = [] + for parts in dict(self).items(): + items.append( + "=".join([p for p in parts if p]) + ) + return "&".join(items) + + +class CallbackResult: + _data = {} + + def __init__( + self, status_code=HTTPStatus.OK, success=True, message=None, data=None, + **kwargs + ): + self.status_code = status_code + self._data = { + "success": success, + "message": message, + "data": data + } + for k, v in kwargs.items(): + self._data[k] = v + + def __getitem__(self, key): + return self._data[key] + + def __iter__(self): + for key in self._data: + yield key + + def get(self, key, default=None): + return self._data.get(key, default) + + def items(self): + return self._data.items() diff --git a/pype/services/rest_api/rest_api.py b/pype/services/rest_api/rest_api.py index 22823a5586..ed644f3e8b 100644 --- a/pype/services/rest_api/rest_api.py +++ b/pype/services/rest_api/rest_api.py @@ -1,260 +1,51 @@ import os -import json -import enum +import re import collections import threading -from inspect import signature import socket -import http.server -from http import HTTPStatus import socketserver - from Qt import QtCore +from .lib import RestApiFactory, Handler +from .base_class import route, register_statics from pypeapp import config, Logger log = Logger().get_logger("RestApiServer") -class RestMethods(enum.Enum): - GET = "GET" - POST = "POST" - PUT = "PUT" - PATCH = "PATCH" - DELETE = "DELETE" - - def __repr__(self): - return str(self.value) - - def __eq__(self, other): - if isinstance(other, str): - return self.value == other - return self == other - - def __hash__(self): - return enum.Enum.__hash__(self) - - def __str__(self): - return str(self.value) - - -class Handler(http.server.SimpleHTTPRequestHandler): - - def do_GET(self): - self.process_request(RestMethods.GET) - - def do_POST(self): - """Common code for POST. - - This trigger callbacks on specific paths. - - If request contain data and callback func has arg data are sent to - callback too. - - Send back return values of callbacks. - """ - self.process_request(RestMethods.POST) - - def process_request(self, rest_method): - """Because processing is technically the same for now so it is used - the same way - """ - in_data = None - cont_len = self.headers.get("Content-Length") - if cont_len: - content_length = int(cont_len) - in_data_str = self.rfile.read(content_length) - if in_data_str: - in_data = json.loads(in_data_str) - - registered_callbacks = self.server.registered_callbacks[rest_method] - - path_items = [part.lower() for part in self.path.split("/") if part] - request_path = "/".join(path_items) - callback = registered_callbacks.get(request_path) - result = None - if callback: - log.debug( - "Triggering callbacks for path \"{}\"".format(request_path) - ) - try: - params = signature(callback).parameters - if len(params) > 0 and in_data: - result = callback(in_data) - else: - result = callback() - - self.send_response(HTTPStatus.OK) - - if result in [None, True] or isinstance(result, (str, int)): - message = str(result) - if result in [None, True]: - message = "{} request for \"{}\" passed".format( - rest_method, self.path - ) - - self.handle_result(rest_method, message=message) - - return - - if isinstance(result, (dict, list)): - message = json.dumps(result).encode() - self.handle_result(rest_method, final_output=message) - - return - - except Exception: - message = "{} request for \"{}\" failed".format( - rest_method, self.path - ) - log.error(message, exc_info=True) - - self.send_response(HTTPStatus.BAD_REQUEST) - self.handle_result(rest_method, message=message, success=False) - - return - - self.handle_result(rest_method) - - else: - message = ( - "{} request for \"{}\" don't have registered callback" - ).format(rest_method, self.path) - log.debug(message) - - self.send_response(HTTPStatus.NOT_FOUND) - self.handle_result(rest_method, message=message, success=False) - - def handle_result( - self, rest_method, final_output=None, message=None, success=True, - content_type="application/json" - ): - self.send_header("Content-type", content_type) - if final_output: - output = final_output - else: - if not message: - output = json.dumps({ - "success": False, - "message": ( - "{} request for \"{}\" has unexpected result" - ).format(rest_method, self.path) - }).encode() - - else: - output = json.dumps({ - "success": success, "message": message - }).encode() - - - if isinstance(output, str): - self.send_header("Content-Length", len(output)) - - self.end_headers() - - if output: - self.wfile.write(output) - - -class AdditionalArgsTCPServer(socketserver.TCPServer): - def __init__(self, registered_callbacks, *args, **kwargs): - self.registered_callbacks = registered_callbacks - super(AdditionalArgsTCPServer, self).__init__(*args, **kwargs) - - -class RestApiServer(QtCore.QThread): - """ Listener for REST requests. - - It is possible to register callbacks for url paths. - Be careful about crossreferencing to different QThreads it is not allowed. - """ - +class RestApiServer: def __init__(self): - super(RestApiServer, self).__init__() - self.registered_callbacks = { - RestMethods.GET: collections.defaultdict(list), - RestMethods.POST: collections.defaultdict(list), - RestMethods.PUT: collections.defaultdict(list), - RestMethods.PATCH: collections.defaultdict(list), - RestMethods.DELETE: collections.defaultdict(list) - } - self.qaction = None self.failed_icon = None self._is_running = False + try: - self.presets = config.get_presets().get( - "services", {}).get( - "rest_api", {} - ) + self.presets = config.get_presets()["services"]["rest_api"] except Exception: self.presets = {"default_port": 8011, "exclude_ports": []} + log.debug(( + "There are not set presets for RestApiModule." + " Using defaults \"{}\"" + ).format(str(self.presets))) - self.port = self.find_port() + port = self.find_port() + self.rest_api_thread = RestApiThread(self, port) + + statics_dir = os.path.sep.join([os.environ["PYPE_MODULE_ROOT"], "res"]) + self.register_statics("/res", statics_dir) def set_qaction(self, qaction, failed_icon): self.qaction = qaction self.failed_icon = failed_icon - def register_callback(self, path, callback, rest_method=RestMethods.POST): - if isinstance(path, (list, set)): - path = "/".join([part.lower() for part in path]) - elif isinstance(path, str): - path = "/".join( - [part.lower() for part in str(path).split("/") if part] - ) + def register_callback(self, path, callback, url_prefix="", methods=[]): + route(path, url_prefix, methods)(callback) - if isinstance(rest_method, str): - rest_method = str(rest_method).upper() + def register_statics(self, url_prefix, dir_path): + register_statics(url_prefix, dir_path) - if path in self.registered_callbacks[rest_method]: - log.error( - "Path \"{}\" has already registered callback.".format(path) - ) - return False - - log.debug( - "Registering callback for path \"{}\"".format(path) - ) - self.registered_callbacks[rest_method][path] = callback - - return True - - def tray_start(self): - self.start() - - @property - def is_running(self): - return self._is_running - - def stop(self): - self._is_running = False - - def run(self): - self._is_running = True - if not self.registered_callbacks: - log.info("Any registered callbacks for Rest Api server.") - return - - try: - log.debug( - "Running Rest Api server on URL:" - " \"http://localhost:{}\"".format(self.port) - ) - with AdditionalArgsTCPServer( - self.registered_callbacks, - ("", self.port), - Handler - ) as httpd: - while self._is_running: - httpd.handle_request() - except Exception: - log.warning( - "Rest Api Server service has failed", exc_info=True - ) - self._is_running = False - if self.qaction and self.failed_icon: - self.qaction.setIcon(self.failed_icon) + def register_obj(self, obj): + RestApiFactory.register_obj(obj) def find_port(self): start_port = self.presets["default_port"] @@ -276,3 +67,53 @@ class RestApiServer(QtCore.QThread): found_port ) return found_port + + def tray_start(self): + RestApiFactory.prepare_registered() + if not RestApiFactory.has_handlers(): + log.debug("There are not registered any handlers for RestApi") + return + self.rest_api_thread.start() + + @property + def is_running(self): + return self.rest_api_thread.is_running + + def stop(self): + self.rest_api_thread.is_running = False + + def thread_stopped(self): + self._is_running = False + + +class RestApiThread(QtCore.QThread): + """ Listener for REST requests. + + It is possible to register callbacks for url paths. + Be careful about crossreferencing to different QThreads it is not allowed. + """ + + def __init__(self, module, port): + super(RestApiThread, self).__init__() + self.is_running = False + self.module = module + self.port = port + + def run(self): + self.is_running = True + + try: + log.debug( + "Running Rest Api server on URL:" + " \"http://localhost:{}\"".format(self.port) + ) + with socketserver.TCPServer(("", self.port), Handler) as httpd: + while self.is_running: + httpd.handle_request() + except Exception: + log.warning( + "Rest Api Server service has failed", exc_info=True + ) + + self.is_running = False + self.module.thread_stopped() From 0ff704a733466cc153375c06613f85f5b600f83e Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 14 Oct 2019 19:08:20 +0200 Subject: [PATCH 02/15] enchancements of custom db connector --- pype/ftrack/lib/custom_db_connector.py | 47 ++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 6 deletions(-) diff --git a/pype/ftrack/lib/custom_db_connector.py b/pype/ftrack/lib/custom_db_connector.py index 505ac96610..e32e62948a 100644 --- a/pype/ftrack/lib/custom_db_connector.py +++ b/pype/ftrack/lib/custom_db_connector.py @@ -20,6 +20,8 @@ import requests import pymongo from pymongo.client_session import ClientSession +class NotActiveTable(Exception): pass + def auto_reconnect(func): """Handling auto reconnect in 3 retry times""" @functools.wraps(func) @@ -37,12 +39,23 @@ def auto_reconnect(func): return decorated +def check_active_table(func): + """Handling auto reconnect in 3 retry times""" + @functools.wraps(func) + def decorated(obj, *args, **kwargs): + if not obj.active_table: + raise NotActiveTable("Active table is not set. (This is bug)") + return func(obj, *args, **kwargs) + + return decorated + + class DbConnector: log = logging.getLogger(__name__) timeout = 1000 - def __init__(self, mongo_url, database_name, table_name): + def __init__(self, mongo_url, database_name, table_name=None): self._mongo_client = None self._sentry_client = None self._sentry_logging_handler = None @@ -53,6 +66,9 @@ class DbConnector: self.active_table = table_name + def __getitem__(self, key): + return self._database[key] + def install(self): """Establish a persistent connection to the database""" if self._is_installed: @@ -100,6 +116,15 @@ class DbConnector: self._database = None self._is_installed = False + def create_table(self, name, **options): + if self.exist_table(name): + return + + return self._database.create_collection(name, **options) + + def exist_table(self, table_name): + return table_name in self.tables() + def tables(self): """List available tables Returns: @@ -115,6 +140,7 @@ class DbConnector: def collections(self): return self._database.collection_names() + @check_active_table @auto_reconnect def insert_one(self, item, session=None): assert isinstance(item, dict), "item must be of type " @@ -123,6 +149,7 @@ class DbConnector: session=session ) + @check_active_table @auto_reconnect def insert_many(self, items, ordered=True, session=None): # check if all items are valid @@ -136,6 +163,7 @@ class DbConnector: session=session ) + @check_active_table @auto_reconnect def find(self, filter, projection=None, sort=None, session=None): return self._database[self.active_table].find( @@ -145,6 +173,7 @@ class DbConnector: session=session ) + @check_active_table @auto_reconnect def find_one(self, filter, projection=None, sort=None, session=None): assert isinstance(filter, dict), "filter must be " @@ -156,13 +185,14 @@ class DbConnector: session=session ) + @check_active_table @auto_reconnect - def replace_one(self, filter, replacement, session=None): + def replace_one(self, filter, replacement, **kw): return self._database[self.active_table].replace_one( - filter, replacement, - session=session + filter, replacement, **kw ) + @check_active_table @auto_reconnect def update_one(self, filter, update, session=None): return self._database[self.active_table].update_one( @@ -170,6 +200,7 @@ class DbConnector: session=session ) + @check_active_table @auto_reconnect def update_many(self, filter, update, session=None): return self._database[self.active_table].update_many( @@ -177,12 +208,14 @@ class DbConnector: session=session ) + @check_active_table @auto_reconnect def distinct(self, *args, **kwargs): return self._database[self.active_table].distinct( *args, **kwargs ) + @check_active_table @auto_reconnect def drop_collection(self, name_or_collection, session=None): return self._database[self.active_table].drop( @@ -190,16 +223,18 @@ class DbConnector: session=session ) + @check_active_table @auto_reconnect - def delete_one(filter, collation=None, session=None): + def delete_one(self, filter, collation=None, session=None): return self._database[self.active_table].delete_one( filter, collation=collation, session=session ) + @check_active_table @auto_reconnect - def delete_many(filter, collation=None, session=None): + def delete_many(self, filter, collation=None, session=None): return self._database[self.active_table].delete_many( filter, collation=collation, From cdd6bd9c5230e1b87cd2c23a1cc49544901e66d8 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 14 Oct 2019 19:09:25 +0200 Subject: [PATCH 03/15] created first rest api for avalon module --- pype/avalon_apps/avalon_app.py | 5 +++ pype/avalon_apps/rest_api.py | 79 ++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 pype/avalon_apps/rest_api.py diff --git a/pype/avalon_apps/avalon_app.py b/pype/avalon_apps/avalon_app.py index 547ecd2299..35ab4c1eb7 100644 --- a/pype/avalon_apps/avalon_app.py +++ b/pype/avalon_apps/avalon_app.py @@ -14,6 +14,11 @@ class AvalonApps: self.parent = parent self.app_launcher = None + def process_modules(self, modules): + if "RestApiServer" in modules: + from .rest_api import AvalonRestApi + self.rest_api_obj = AvalonRestApi() + # Definition of Tray menu def tray_menu(self, parent_menu=None): # Actions diff --git a/pype/avalon_apps/rest_api.py b/pype/avalon_apps/rest_api.py new file mode 100644 index 0000000000..dc74f17bd1 --- /dev/null +++ b/pype/avalon_apps/rest_api.py @@ -0,0 +1,79 @@ +import os +import re +import json +import bson +import bson.json_util +from pype.services.rest_api import RestApi, abort, CallbackResult, Query +from pype.ftrack.lib.custom_db_connector import DbConnector + + +class AvalonRestApi(RestApi): + dbcon = DbConnector( + os.environ["AVALON_MONGO"], + os.environ["AVALON_DB"] + ) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.dbcon.install() + + @RestApi.route("/projects/", url_prefix="/avalon", methods="GET") + def get_project(self, url_data): + project_name = url_data["project_name"] + if not project_name: + output = {} + for project_name in self.dbcon.tables(): + project = self.dbcon[project_name].find_one({"type": "project"}) + output[project_name] = project + + return CallbackResult(data=self.result_to_json(output)) + + project = self.dbcon[project_name].find_one({"type": "project"}) + + if project: + return CallbackResult(data=self.result_to_json(project)) + + abort(404, "Project \"{}\" was not found in database".format( + project_name + )) + + @RestApi.route("/projects//assets/", url_prefix="/avalon", methods="GET") + def get_assets(self, url_data, query:Query): + _project_name = url_data["project_name"] + _asset = url_data["asset"] + + if not self.dbcon.exist_table(_project_name): + abort(404, "Project \"{}\" was not found in database".format( + project_name + )) + + if not _asset: + assets = self.dbcon[_project_name].find({"type": "asset"}) + output = self.result_to_json(assets) + return CallbackResult(data=output) + + identificator = query.get("identificator", "name") + + asset = self.dbcon[_project_name].find_one({ + "type": "asset", + identificator: _asset + }) + if asset: + id = asset["_id"] + asset["_id"] = str(id) + return asset + + abort(404, "Asset \"{}\" with {} was not found in project {}".format( + _asset, identificator, project_name + )) + + def result_to_json(self, result): + bson_json = bson.json_util.dumps(result) + # Replace "{$oid: "{entity id}"}" with "{entity id}" + regex1 = '(?P{\"\$oid\": \"[^\"]+\"})' + regex2 = '{\"\$oid\": (?P\"[^\"]+\")}' + for value in re.findall(regex1, bson_json): + for substr in re.findall(regex2, value): + bson_json = bson_json.replace(value, substr) + + return json.loads(bson_json) From cc9dd16d98a06c7ec6ad7945daf40160a3aa081c Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 17 Oct 2019 11:20:52 +0200 Subject: [PATCH 04/15] changed registering troute info are not stored to callback --- pype/services/rest_api/base_class.py | 18 ++------- pype/services/rest_api/lib/factory.py | 57 +++++++++++++++------------ pype/services/rest_api/rest_api.py | 4 +- 3 files changed, 38 insertions(+), 41 deletions(-) diff --git a/pype/services/rest_api/base_class.py b/pype/services/rest_api/base_class.py index 94c7eb0bbd..a389ba517b 100644 --- a/pype/services/rest_api/base_class.py +++ b/pype/services/rest_api/base_class.py @@ -10,21 +10,9 @@ from .lib import ( def route(path, url_prefix="", methods=[]): def decorator(callback): - @wraps(callback) - def wrapper(*args, **kwargs): - return callback(*args, **kwargs) - - func = wrapper - func.restapi = True - func.path = path - func.methods = methods - func.url_prefix = url_prefix - if hasattr(callback, "__self__"): - func.__self__ = callback.__self__ - func.callback = callback - - RestApiFactory.register_route(func) - return func + RestApiFactory.register_route(path, callback, url_prefix, methods) + callback.restapi = True + return callback return decorator diff --git a/pype/services/rest_api/lib/factory.py b/pype/services/rest_api/lib/factory.py index dbbd6ec5af..58f7fa8f8a 100644 --- a/pype/services/rest_api/lib/factory.py +++ b/pype/services/rest_api/lib/factory.py @@ -112,8 +112,7 @@ def prepare_methods(methods, callback=None): return _methods -def prepare_callback_info(_callback): - callback = _callback.callback +def prepare_callback_info(callback): callback_info = inspect.getfullargspec(callback) callback_args = callback_info.args @@ -121,7 +120,7 @@ def prepare_callback_info(_callback): if callback_args: callback_args_len = len(callback_args) if ( - type(_callback).__name__ == "method" + type(callback).__name__ == "method" ): callback_args_len -= 1 @@ -158,9 +157,9 @@ class _RestApiFactory: def has_handlers(self): return (self.has_routes or self.prepared_statics) - def _process_route(self, callback): + def _process_route(self, route): return self.unprocessed_routes.pop( - self.unprocessed_routes.index(callback) + self.unprocessed_routes.index(route) ) def _process_statics(self, item): @@ -168,11 +167,17 @@ class _RestApiFactory: self.unprocessed_statics.index(item) ) - def register_route(self, item): + def register_route(self, path, callback, url_prefix, methods): log.debug("Registering callback for item \"{}\"".format( - item.__qualname__ + callback.__qualname__ )) - self.unprocessed_routes.append(item) + route = { + "path": path, + "callback": callback, + "url_prefix": url_prefix, + "methods": methods + } + self.unprocessed_routes.append(route) def register_obj(self, obj): self.registered_objs.append(obj) @@ -181,10 +186,11 @@ class _RestApiFactory: log.debug("Registering statics path \"{}\"".format(item)) self.unprocessed_statics.append(item) - def _prepare_route(self, callback): - methods = prepare_methods(callback.methods, callback) - url_prefix = prepare_prefix(callback.url_prefix) - fullpath = prepare_fullpath(callback.path, url_prefix) + def _prepare_route(self, route): + callback = route["callback"] + methods = prepare_methods(route["methods"], callback) + url_prefix = prepare_prefix(route["url_prefix"]) + fullpath = prepare_fullpath(route["path"], url_prefix) regex, regex_keys = prepare_regex_from_path(fullpath) callback_info = prepare_callback_info(callback) @@ -217,8 +223,14 @@ class _RestApiFactory: ] for method_name in method_names: method = obj.__getattribute__(method_name) + if not hasattr(method, "restapi"): + continue - for callback in self.unprocessed_routes: + if not method.restapi: + continue + + for route in self.unprocessed_routes: + callback = route["callback"] if not ( callback.__qualname__ == method.__qualname__ and callback.__module__ == method.__module__ and @@ -226,18 +238,13 @@ class _RestApiFactory: ): continue - self._process_route(callback) - - if not hasattr(method, "restapi"): - continue - - if not method.restapi: - continue - - self._prepare_route(method) + route["callback"] = method + self._process_route(route) + self._prepare_route(route) break - for callback in self.unprocessed_routes: + for route in self.unprocessed_routes: + callback = route["callback"] is_class_method = len(callback.__qualname__.split(".")) != 1 if is_class_method: missing_self = True @@ -260,7 +267,7 @@ class _RestApiFactory: )) continue - self._prepare_route(callback) + self._prepare_route(route) continue - self._prepare_route(callback) + self._prepare_route(route) diff --git a/pype/services/rest_api/rest_api.py b/pype/services/rest_api/rest_api.py index ed644f3e8b..00e38b8102 100644 --- a/pype/services/rest_api/rest_api.py +++ b/pype/services/rest_api/rest_api.py @@ -39,7 +39,9 @@ class RestApiServer: self.failed_icon = failed_icon def register_callback(self, path, callback, url_prefix="", methods=[]): - route(path, url_prefix, methods)(callback) + callback.restapi = True + RestApiFactory.register_route(path, callback, url_prefix, methods) + # route(path, url_prefix, methods)(callback) def register_statics(self, url_prefix, dir_path): register_statics(url_prefix, dir_path) From dc14235e97c4bb811fdb5ed147e5293e0dfd17e9 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 17 Oct 2019 12:03:03 +0200 Subject: [PATCH 05/15] rest api is not sending to callback specific data by args but send request info obj that contain all of that --- pype/avalon_apps/rest_api.py | 14 +-- pype/muster/muster.py | 4 +- pype/services/rest_api/__init__.py | 11 +- pype/services/rest_api/base_class.py | 1 - pype/services/rest_api/lib/__init__.py | 12 +-- pype/services/rest_api/lib/handler.py | 134 +++++-------------------- pype/services/rest_api/lib/lib.py | 26 +++++ pype/services/rest_api/rest_api.py | 1 - 8 files changed, 61 insertions(+), 142 deletions(-) diff --git a/pype/avalon_apps/rest_api.py b/pype/avalon_apps/rest_api.py index dc74f17bd1..d58d080339 100644 --- a/pype/avalon_apps/rest_api.py +++ b/pype/avalon_apps/rest_api.py @@ -3,7 +3,7 @@ import re import json import bson import bson.json_util -from pype.services.rest_api import RestApi, abort, CallbackResult, Query +from pype.services.rest_api import RestApi, abort, CallbackResult from pype.ftrack.lib.custom_db_connector import DbConnector @@ -18,8 +18,8 @@ class AvalonRestApi(RestApi): self.dbcon.install() @RestApi.route("/projects/", url_prefix="/avalon", methods="GET") - def get_project(self, url_data): - project_name = url_data["project_name"] + def get_project(self, request): + project_name = request.url_data["project_name"] if not project_name: output = {} for project_name in self.dbcon.tables(): @@ -38,9 +38,9 @@ class AvalonRestApi(RestApi): )) @RestApi.route("/projects//assets/", url_prefix="/avalon", methods="GET") - def get_assets(self, url_data, query:Query): - _project_name = url_data["project_name"] - _asset = url_data["asset"] + def get_assets(self, request): + _project_name = request.url_data["project_name"] + _asset = request.url_data["asset"] if not self.dbcon.exist_table(_project_name): abort(404, "Project \"{}\" was not found in database".format( @@ -52,7 +52,7 @@ class AvalonRestApi(RestApi): output = self.result_to_json(assets) return CallbackResult(data=output) - identificator = query.get("identificator", "name") + identificator = request.query.get("identificator", "name") asset = self.dbcon[_project_name].find_one({ "type": "asset", diff --git a/pype/muster/muster.py b/pype/muster/muster.py index a4805369aa..e33caf40ca 100644 --- a/pype/muster/muster.py +++ b/pype/muster/muster.py @@ -38,8 +38,10 @@ class MusterModule: def process_modules(self, modules): if "RestApiServer" in modules: + def api_show_login(): + self.aShowLogin.trigger() modules["RestApiServer"].register_callback( - "muster/show_login", self.show_login, "post" + "/show_login", api_show_login, "muster", "post" ) # Definition of Tray menu diff --git a/pype/services/rest_api/__init__.py b/pype/services/rest_api/__init__.py index f0fddce0ee..fbeec00c88 100644 --- a/pype/services/rest_api/__init__.py +++ b/pype/services/rest_api/__init__.py @@ -1,15 +1,6 @@ from .rest_api import RestApiServer from .base_class import RestApi, abort, route, register_statics -from .lib import ( - RestMethods, - UrlData, - RequestData, - Query, - Fragment, - Params, - Handler, - CallbackResult -) +from .lib import RestMethods, CallbackResult def tray_init(tray_widget, main_widget): diff --git a/pype/services/rest_api/base_class.py b/pype/services/rest_api/base_class.py index a389ba517b..8d4f7cbaa5 100644 --- a/pype/services/rest_api/base_class.py +++ b/pype/services/rest_api/base_class.py @@ -4,7 +4,6 @@ from http import HTTPStatus from .lib import ( RestApiFactory, Splitter, ObjAlreadyExist, AbortException, - Params, UrlData, RequestData, Query, Fragment, Handler ) diff --git a/pype/services/rest_api/lib/__init__.py b/pype/services/rest_api/lib/__init__.py index 38d476050a..97fb232409 100644 --- a/pype/services/rest_api/lib/__init__.py +++ b/pype/services/rest_api/lib/__init__.py @@ -1,17 +1,7 @@ Splitter = "__splitter__" from .exceptions import ObjAlreadyExist, AbortException -from .lib import ( - RestMethods, - CustomNone, - UrlData, - RequestData, - Query, - Fragment, - Params, - CallbackResult -) - +from .lib import RestMethods, CustomNone, CallbackResult, RequestInfo from .factory import _RestApiFactory RestApiFactory = _RestApiFactory() diff --git a/pype/services/rest_api/lib/handler.py b/pype/services/rest_api/lib/handler.py index 71215c1f2c..de6b5a9a44 100644 --- a/pype/services/rest_api/lib/handler.py +++ b/pype/services/rest_api/lib/handler.py @@ -6,17 +6,13 @@ import http.server from http import HTTPStatus from urllib.parse import urlparse -from .lib import ( - RestMethods, CallbackResult, - UrlData, RequestData, Query, Fragment, Params -) +from .lib import RestMethods, CallbackResult, RequestInfo from .exceptions import AbortException -from . import RestApiFactory, CustomNone, Splitter +from . import RestApiFactory, Splitter from pypeapp import Logger log = Logger().get_logger("RestApiHandler") -NotSet = CustomNone("NotSet") class Handler(http.server.SimpleHTTPRequestHandler): @@ -115,7 +111,9 @@ class Handler(http.server.SimpleHTTPRequestHandler): try: log.debug("Triggering callback for path \"{}\"".format(path)) - result = self._handle_callback(matching_item, parsed_url) + result = self._handle_callback( + matching_item, parsed_url, rest_method + ) return self._handle_callback_result(result, rest_method) @@ -208,7 +206,7 @@ class Handler(http.server.SimpleHTTPRequestHandler): self.wfile.write(body.encode()) return body - def _handle_callback(self, item, parsed_url): + def _handle_callback(self, item, parsed_url, rest_method): regex = item["regex"] regex_keys = item["regex_keys"] @@ -228,11 +226,15 @@ class Handler(http.server.SimpleHTTPRequestHandler): if in_data_str: in_data = json.loads(in_data_str) - url_data = UrlData(url_data) - request_data = RequestData(in_data) - query = Query(parsed_url.query) - params = Params(parsed_url.params) - fragment = Fragment(parsed_url.fragment) + request_info = RequestInfo( + url_data=url_data, + request_data=in_data, + query=parsed_url.query, + fragment=parsed_url.fragment, + params=parsed_url.params, + method=rest_method, + handler=self + ) callback = item["callback"] callback_info = item["callback_info"] @@ -240,110 +242,20 @@ class Handler(http.server.SimpleHTTPRequestHandler): _args = callback_info["args"] _args_len = callback_info["args_len"] _defaults = callback_info["defaults"] - _defaults_len = callback_info["defaults_len"] _has_args = callback_info["hasargs"] _has_kwargs = callback_info["haskwargs"] - _annotations = callback_info["annotations"] - - arg_index = 0 - if type(callback).__name__ == "method": - arg_index = 1 - - _kwargs = {arg: NotSet for arg in _args[arg_index:]} - _available_kwargs = { - "url_data": url_data, - "request_data": request_data, - "query": query, - "params": params, - "fragment": fragment, - "handler": self - } - if not regex: - _available_kwargs.pop("url_data") - - available_len = len(_available_kwargs) - if available_len < (_args_len - _defaults_len): - raise Exception(( - "Callback expects {} required positional arguments but {} are" - " available {}<{}>" - ).format( - _args_len, available_len, callback.__qualname__, - callback.__globals__.get("__file__", "unknown file") - )) - elif available_len < _args_len: - log.warning(( - "Handler \"{}\" will never fill all args of callback {}<{}>" - ).format( - self.__class__.__name__, - callback.__qualname__, - callback.__globals__.get("__file__", "unknown file") - )) - + args = [] + kwargs = {} if _args_len == 0: if _has_args: - return callback(*_available_kwargs.values()) - if _has_kwargs: - return callback(**_available_kwargs) - else: - return callback() + args.append(request_info) + elif _has_kwargs: + kwargs["request_info"] = request_info + else: + args.append(request_info) - if _annotations: - for arg, argtype in _annotations.items(): - if argtype == Query: - key = "query" - elif argtype == Params: - key = "params" - elif argtype == Fragment: - key = "fragment" - elif argtype == Handler: - key = "handler" - elif argtype == UrlData: - key = "url_data" - elif argtype == RequestData: - key = "request_data" - else: - continue - - _kwargs[arg] = _available_kwargs[key] - _available_kwargs[key] = NotSet - - for key1, value1 in _kwargs.items(): - if value1 is not NotSet: - continue - has_values = False - for key2, value2 in _available_kwargs.items(): - if value2 is NotSet: - continue - has_values = True - _kwargs[key1] = value2 - _available_kwargs[key2] = NotSet - break - - if not has_values: - break - - _args = [] - kw_keys = [key for key in _kwargs.keys()] - for key in kw_keys: - value = _kwargs.pop(key) - if value is not NotSet: - _args.append(value) - continue - - if _has_args: - for key, value in _available_kwargs.items(): - if value is NotSet: - continue - _args.append(value) - - elif _has_kwargs: - for key, value in _available_kwargs.items(): - if value is NotSet: - continue - _kwargs[key] = value - - return callback(*_args, **_kwargs) + return callback(*args, **kwargs) def _handle_statics(self, dirpath, path): path = os.path.normpath(dirpath + path) diff --git a/pype/services/rest_api/lib/lib.py b/pype/services/rest_api/lib/lib.py index 4a9e41ce26..f8508ceea0 100644 --- a/pype/services/rest_api/lib/lib.py +++ b/pype/services/rest_api/lib/lib.py @@ -109,6 +109,32 @@ class Fragment(HandlerDict): ) return "&".join(items) +class RequestInfo: + def __init__( + self, url_data, request_data, query, fragment, params, method, handler + ): + self.url_data = UrlData(url_data) + self.request_data = RequestData(request_data) + self.query = Query(query) + self.fragment = Fragment(fragment) + self.params = Params(params) + self.method = method + self.handler = handler + + def __getitem__(self, key): + return self.__getattribute__(key) + + def __hash__(self): + return { + "url_data": self.url_data, + "request_data": self. request_data, + "query": self.query, + "fragment": self.fragment, + "params": self.params, + "method": self.method, + "handler": self.handler + } + class CallbackResult: _data = {} diff --git a/pype/services/rest_api/rest_api.py b/pype/services/rest_api/rest_api.py index 00e38b8102..994632a1e9 100644 --- a/pype/services/rest_api/rest_api.py +++ b/pype/services/rest_api/rest_api.py @@ -39,7 +39,6 @@ class RestApiServer: self.failed_icon = failed_icon def register_callback(self, path, callback, url_prefix="", methods=[]): - callback.restapi = True RestApiFactory.register_route(path, callback, url_prefix, methods) # route(path, url_prefix, methods)(callback) From 7d623a2bb5d9f404be81f0d973131c1d7356a582 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 17 Oct 2019 12:17:46 +0200 Subject: [PATCH 06/15] minor change in request class --- pype/services/rest_api/lib/lib.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pype/services/rest_api/lib/lib.py b/pype/services/rest_api/lib/lib.py index f8508ceea0..73cf13683a 100644 --- a/pype/services/rest_api/lib/lib.py +++ b/pype/services/rest_api/lib/lib.py @@ -135,6 +135,9 @@ class RequestInfo: "handler": self.handler } + def items(self): + return dict(self).items() + class CallbackResult: _data = {} From 518885a39f8a141c311c1f18cb59efea5f583c88 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Fri, 18 Oct 2019 00:26:10 +0200 Subject: [PATCH 07/15] added docstrings --- pype/services/rest_api/base_class.py | 53 +++++++++++++++++++ pype/services/rest_api/lib/exceptions.py | 1 + pype/services/rest_api/lib/factory.py | 67 +++++++++++++++++++++++- pype/services/rest_api/lib/handler.py | 35 +++++++++++-- pype/services/rest_api/lib/lib.py | 38 ++++++++++++++ pype/services/rest_api/rest_api.py | 66 ++++++++++++++++++++++- 6 files changed, 254 insertions(+), 6 deletions(-) diff --git a/pype/services/rest_api/base_class.py b/pype/services/rest_api/base_class.py index 8d4f7cbaa5..5355c5886c 100644 --- a/pype/services/rest_api/base_class.py +++ b/pype/services/rest_api/base_class.py @@ -8,6 +8,32 @@ from .lib import ( def route(path, url_prefix="", methods=[]): + """Decorator that register callback and all its attributes. + Callback is registered to Singleton RestApiFactory. + + :param path: Specify url path when callback should be triggered. + :type path: str + :param url_prefix: Specify prefix of path, defaults to "/". + :type url_prefix: str, list, optional + :param methods: Specify request method (GET, POST, PUT, UPDATE, DELETE) when callback will be triggered, defaults to ["GET"] + :type methods: list, str, optional + + `path` may include dynamic keys that will be stored to object which can + be obtained in callback. + Example: + - registered path: "/projects/" + - url request path: "/projects/S001_test_project" + In this case will be callback triggered and in accessible data will be + stored {"project_name": "S001_test_project"}. + + `url_prefix` is optional but it is better to specify for easier filtering + of requests. + Example: + - url_prefix: `"/avalon"` or `["avalon"]` + - path: `"/projects"` + In this case request path must be "/avalon/projects" to trigger registered + callback. + """ def decorator(callback): RestApiFactory.register_route(path, callback, url_prefix, methods) callback.restapi = True @@ -16,10 +42,29 @@ def route(path, url_prefix="", methods=[]): def register_statics(url_prefix, dir_path): + """Decorator that register callback and all its attributes. + Callback is registered to Singleton RestApiFactory. + + :param url_prefix: Specify prefix of path, defaults to "/". (Example: "/resources") + :type url_prefix: str + :param dir_path: Path to file folder where statics are located. + :type dir_path: str + """ RestApiFactory.register_statics((url_prefix, dir_path)) def abort(status_code=HTTPStatus.NOT_FOUND, message=None): + """Should be used to stop registered callback + `abort` raise AbortException that is handled with request Handler which + returns entered status and may send optional message in body. + + :param status_code: Status that will be send in reply of request, defaults to 404 + :type status_code: int + :param message: Message to send in body, default messages are based on statuc_code in Handler, defaults to None + :type message: str, optional + ... + :raises AbortException: This exception is handled in Handler to know about launched `abort` + """ items = [] items.append(str(status_code)) if not message: @@ -31,6 +76,14 @@ def abort(status_code=HTTPStatus.NOT_FOUND, message=None): class RestApi: + """Base class for RestApi classes. + + Use this class is required when it is necessary to have class for handling + requests and want to use decorators for registering callbacks. + + It is possible to use decorators in another class only when object, of class + where decorators are, is registered to RestApiFactory. + """ def route(path, url_prefix="", methods=[]): return route(path, url_prefix, methods) diff --git a/pype/services/rest_api/lib/exceptions.py b/pype/services/rest_api/lib/exceptions.py index 849b522ddd..118fa4ffc8 100644 --- a/pype/services/rest_api/lib/exceptions.py +++ b/pype/services/rest_api/lib/exceptions.py @@ -1,4 +1,5 @@ class ObjAlreadyExist(Exception): + """Is used when is created multiple objects of same RestApi class.""" def __init__(self, cls=None, message=None): if not (cls and message): message = "RestApi object was created twice." diff --git a/pype/services/rest_api/lib/factory.py b/pype/services/rest_api/lib/factory.py index 58f7fa8f8a..3a24fe5ae8 100644 --- a/pype/services/rest_api/lib/factory.py +++ b/pype/services/rest_api/lib/factory.py @@ -10,6 +10,16 @@ log = Logger().get_logger("RestApiFactory") def prepare_fullpath(path, prefix): + """Concatenate registered path and prefix with right form. + + :param path: Registered url path for registered callback. + :type path: str, list + :param prefix: Registered and prepared url prefix. + :type prefix: str, None + :return: concatenated prefix and path in right form + :rtype: str + """ + if path and prefix: fullpath = "{}/{}".format(prefix, path).replace("//", "/") elif path: @@ -26,6 +36,16 @@ def prepare_fullpath(path, prefix): def prepare_regex_from_path(full_path): + """Prepare regex based on set path. + + When registered path do not contain dynamic keys regex is not set. + Dynamic keys are specified with "<" and ">" ("<{dynamic key}>"). + + :param full_path: Full url path (prefix + path) for registered callback. + :type full_path: str, list, None + :return: regex and keys of all groups in regex + :rtype: tuple(SRE_Pattern, list), tuple(None, None) + """ get_indexes_regex = "<[^< >]+>" all_founded_keys = re.findall(get_indexes_regex, full_path) if not all_founded_keys: @@ -46,8 +66,17 @@ def prepare_regex_from_path(full_path): def prepare_prefix(url_prefix): - if url_prefix is None: - url_prefix = "" + """Check if the url_prefix is set and is in correct form. + + Output is None when prefix is empty or "/". + + :param url_prefix: Registered prefix of registered callback. + :type url_prefix: str, list, None + :return: Url prefix of registered callback + :rtype: str, None + """ + if url_prefix is None or url_prefix.strip() == "/": + return None elif isinstance(url_prefix, (list, tuple)): url_prefix = "/".join(url_prefix) else: @@ -64,6 +93,18 @@ def prepare_prefix(url_prefix): def prepare_methods(methods, callback=None): + """Check and convert entered methods. + + String `methods` is converted to list. All values are converted to + `RestMethods` enum object. Invalid methods are ignored and printed out. + + :param methods: Contain rest api methods, when callback is called. + :type methods: str, list + :param callback: Registered callback, helps to identify where is invalid method. + :type callback: function, method, optional + :return: Valid methods + :rtype: list + """ invalid_methods = collections.defaultdict(list) if not methods: @@ -113,6 +154,7 @@ def prepare_methods(methods, callback=None): return _methods def prepare_callback_info(callback): + """Prepare data for callback handling when should be triggered.""" callback_info = inspect.getfullargspec(callback) callback_args = callback_info.args @@ -143,6 +185,11 @@ def prepare_callback_info(callback): class _RestApiFactory: + """Factory is used to store and prepare callbacks for requests. + + Should be created only one object used for all registered callbacks when + it is expected to run only one http server. + """ registered_objs = [] unprocessed_routes = [] unprocessed_statics = [] @@ -180,6 +227,7 @@ class _RestApiFactory: self.unprocessed_routes.append(route) def register_obj(self, obj): + """Register object for decorated methods in class definition""" self.registered_objs.append(obj) def register_statics(self, item): @@ -187,6 +235,14 @@ class _RestApiFactory: self.unprocessed_statics.append(item) def _prepare_route(self, route): + """Prepare data of registered callbacks for routes. + + Registration info are prepared to easy filter during handling + of requests. + + :param route: Contain all necessary info for filtering and handling callback for registered route. + :type route: dict + """ callback = route["callback"] methods = prepare_methods(route["methods"], callback) url_prefix = prepare_prefix(route["url_prefix"]) @@ -205,6 +261,13 @@ class _RestApiFactory: }) def prepare_registered(self): + """Iterate through all registered callbacks and statics and prepare them + + First are processed callbacks registered with decorators in classes by + registered objects. Remaining callbacks are filtered, it is checked if + methods has `__self__` or are defined in (it is expeted they + do not requise access to object) + """ for url_prefix, dir_path in self.unprocessed_statics: self._process_statics((url_prefix, dir_path)) dir_path = os.path.normpath(dir_path) diff --git a/pype/services/rest_api/lib/handler.py b/pype/services/rest_api/lib/handler.py index de6b5a9a44..cd5c0abe55 100644 --- a/pype/services/rest_api/lib/handler.py +++ b/pype/services/rest_api/lib/handler.py @@ -48,8 +48,15 @@ class Handler(http.server.SimpleHTTPRequestHandler): return self._handle_request(RestMethods.PATCH) def _handle_request(self, rest_method): - """Because processing is technically the same for now so it is used - the same way + """Handle request by registered callbacks and statics. + + Statics are only for GET method request. `_handle_statics` is called + when path is matching registered statics prefix. Callbacks are filtered + by method and their prefixes. When path is matching `_handle_callback` + is called. + + If any registered callback or statics match requested path 400 status + is responsed. And 500 when unexpected error happens. """ parsed_url = urlparse(self.path) path = parsed_url.path @@ -151,6 +158,18 @@ class Handler(http.server.SimpleHTTPRequestHandler): def _handle_callback_result(self, result, rest_method): + """Send response to request based on result of callback. + :param result: Result returned by callback. + :type result: None, bool, dict, list, CallbackResult + :param rest_method: Rest api method (GET, POST, etc.). + :type rest_method: RestMethods + + Response is based on result type: + - None, True - It is expected everything was OK, status 200. + - False - It is expected callback was not successful, status 400 + - dict, list - Result is send under "data" key of body, status 200 + - CallbackResult - object specify status and data + """ content_type = "application/json" status = HTTPStatus.OK success = True @@ -207,8 +226,17 @@ class Handler(http.server.SimpleHTTPRequestHandler): return body def _handle_callback(self, item, parsed_url, rest_method): + """Prepare data from request and trigger callback. - regex = item["regex"] + Data are loaded from body of request if there are any. + + :param item: Item stored during callback registration with all info. + :type item: dict + :param parsed_url: Url parsed with urllib (separated path, query, etc.). + :type parsed_url: ParseResult + :param rest_method: Rest api method (GET, POST, etc.). + :type rest_method: RestMethods + """ regex_keys = item["regex_keys"] url_data = None @@ -258,6 +286,7 @@ class Handler(http.server.SimpleHTTPRequestHandler): return callback(*args, **kwargs) def _handle_statics(self, dirpath, path): + """Return static file in response when file exist in registered destination.""" path = os.path.normpath(dirpath + path) ctype = self.guess_type(path) diff --git a/pype/services/rest_api/lib/lib.py b/pype/services/rest_api/lib/lib.py index 73cf13683a..4fab11469e 100644 --- a/pype/services/rest_api/lib/lib.py +++ b/pype/services/rest_api/lib/lib.py @@ -36,6 +36,7 @@ class RestMethods(enum.Enum): class CustomNone: + """Created object can be used as custom None (not equal to None)""" def __init__(self, name): self._name = name @@ -69,6 +70,7 @@ class UrlData(HandlerDict): pass class RequestData(HandlerDict): pass class Query(HandlerDict): + """Class for url query convert to dict and string""" def __init__(self, query): if isinstance(query, dict): pass @@ -80,6 +82,7 @@ class Query(HandlerDict): return urlencode(dict(self), doseq=True) class Fragment(HandlerDict): + """Class for url fragment convert to dict and string""" def __init__(self, fragment): if isinstance(fragment, dict): _fragment = fragment @@ -110,6 +113,26 @@ class Fragment(HandlerDict): return "&".join(items) class RequestInfo: + """Object that can be passed to callback as argument. + + Contain necessary data for handling request. + Object is created to single use and can be used similar to dict. + + :param url_data: Data collected from path when path with dynamic keys is matching. + :type url_data: dict, None + :param request_data: Data of body from request. + :type request_data: dict, None + :param query: Query from url path of reques. + :type query: str, None + :param fragment: Fragment from url path of reques. + :type fragment: str, None + :param params: Parems from url path of reques. + :type params: str, None + :param method: Method of request (GET, POST, etc.) + :type method: RestMethods + :param handler: Handler handling request from http server. + :type handler: Handler + """ def __init__( self, url_data, request_data, query, fragment, params, method, handler ): @@ -140,6 +163,21 @@ class RequestInfo: class CallbackResult: + """Can be used as return value of callback. + + It is possible to specify status code, success boolean, message and data + for specify head and body of request response. `abort` should be rather used + when result is error. + + :param status_code: Status code of result. + :type status_code: int + :param success: Success is key in body, may be used for handling response. + :type success: bool + :param message: Similar to success, message is key in body and may be used for handling response. + :type message: str, None + :param data: Data is also key for body in response. + :type data: dict, None + """ _data = {} def __init__( diff --git a/pype/services/rest_api/rest_api.py b/pype/services/rest_api/rest_api.py index 994632a1e9..2df666b45a 100644 --- a/pype/services/rest_api/rest_api.py +++ b/pype/services/rest_api/rest_api.py @@ -14,6 +14,70 @@ log = Logger().get_logger("RestApiServer") class RestApiServer: + """Rest Api allows to access statics or callbacks with http requests. + + To register statics use `register_statics`. + + To register callback use `register_callback` method or use `route` decorator. + `route` decorator should be used with not-class functions, it is possible + to use within class when inherits `RestApi` (defined in `base_class.py`) + or created object, with used decorator, is registered with `register_obj`. + + .. code-block:: python + @route("/username", url_prefix="/api", methods=["get"]) + def get_username(): + return {"username": getpass.getuser()} + + In that case response to `localhost:{port}/api/username` will be status + `200` with body including `{"data": {"username": getpass.getuser()}}` + + Callback may expect one argument which will be filled with request + info. Data object has attributes: `request_data`, `query`, + `fragment`, `params`, `method`, `handler`, `url_data`. + request_data - Data from request body if there are any. + query - query from url path (?identificator=name) + fragment - fragment from url path (#reset_credentials) + params - params from url path + method - request method (GET, POST, PUT, etc.) + handler - Handler object of HttpServer Handler + url_data - dynamic url keys from registered path with their values + + Dynamic url keys may be set with path argument. + .. code-block:: python + from rest_api import route + + all_projects = { + "Proj1": {"proj_data": []}, + "Proj2": {"proj_data": []}, + } + + @route("/projects/", url_prefix="/api", methods=["get"]) + def get_projects(request_info): + project_name = request_info.url_data["project_name"] + if not project_name: + return all_projects + return all_projects.get(project_name) + + This example should end with status 404 if project is not found. In that + case is best to use `abort` method. + + .. code-block:: python + from rest_api import abort + + @route("/projects/", url_prefix="/api", methods=["get"]) + def get_projects(request_info): + project_name = request_info.url_data["project_name"] + if not project_name: + return all_projects + + project = all_projects.get(project_name) + if not project: + abort(404, "Project \"{}\".format(project_name) was not found") + return project + + Callback may return many types. For more information read docstring of + `_handle_callback_result` defined in handler. + """ def __init__(self): self.qaction = None self.failed_icon = None @@ -91,7 +155,7 @@ class RestApiThread(QtCore.QThread): """ Listener for REST requests. It is possible to register callbacks for url paths. - Be careful about crossreferencing to different QThreads it is not allowed. + Be careful about crossreferencing to different Threads it is not allowed. """ def __init__(self, module, port): From 70853e25e35ec16681de4520600c56ce3c7aea7d Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Fri, 18 Oct 2019 00:26:23 +0200 Subject: [PATCH 08/15] removed unused variables --- pype/services/rest_api/lib/handler.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pype/services/rest_api/lib/handler.py b/pype/services/rest_api/lib/handler.py index cd5c0abe55..3f4ec8937d 100644 --- a/pype/services/rest_api/lib/handler.py +++ b/pype/services/rest_api/lib/handler.py @@ -267,9 +267,7 @@ class Handler(http.server.SimpleHTTPRequestHandler): callback = item["callback"] callback_info = item["callback_info"] - _args = callback_info["args"] _args_len = callback_info["args_len"] - _defaults = callback_info["defaults"] _has_args = callback_info["hasargs"] _has_kwargs = callback_info["haskwargs"] From 339b623e89ebdbdd4a715884757b6a1496ccabae Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Fri, 18 Oct 2019 00:27:01 +0200 Subject: [PATCH 09/15] minor changes in prefix and fullpath preparing --- pype/services/rest_api/lib/factory.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/pype/services/rest_api/lib/factory.py b/pype/services/rest_api/lib/factory.py index 3a24fe5ae8..3c9f84cc85 100644 --- a/pype/services/rest_api/lib/factory.py +++ b/pype/services/rest_api/lib/factory.py @@ -20,14 +20,26 @@ def prepare_fullpath(path, prefix): :rtype: str """ + if isinstance(path, (list, tuple)): + path_items = path + else: + path_items = [part for part in path.split("/") if part] + + fullpath = "/" if path and prefix: - fullpath = "{}/{}".format(prefix, path).replace("//", "/") + items = [part for part in prefix.split("/") if part] + items.extend(path_items) + fullpath = "/".join(items) + if path.endswith("/"): + fullpath += "/" + elif path: - fullpath = path + fullpath = "/".join(path_items) + if path.endswith("/"): + fullpath += "/" + elif prefix: fullpath = prefix - else: - fullpath = "/" if not fullpath.startswith("/"): fullpath = "/{}".format(fullpath) @@ -86,6 +98,9 @@ def prepare_prefix(url_prefix): if not url_prefix: return None + while url_prefix.endswith("/"): + url_prefix = url_prefix[:-1] + if not url_prefix.startswith("/"): url_prefix = "/{}".format(url_prefix) @@ -161,9 +176,7 @@ def prepare_callback_info(callback): callback_args_len = 0 if callback_args: callback_args_len = len(callback_args) - if ( - type(callback).__name__ == "method" - ): + if type(callback).__name__ == "method": callback_args_len -= 1 defaults = callback_info.defaults From 9233530df5cb2fc5306c729b51ac8a27865c6193 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 18 Oct 2019 11:19:48 +0200 Subject: [PATCH 10/15] added default message to NotActiveTable --- pype/ftrack/lib/custom_db_connector.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pype/ftrack/lib/custom_db_connector.py b/pype/ftrack/lib/custom_db_connector.py index e32e62948a..931b46fc00 100644 --- a/pype/ftrack/lib/custom_db_connector.py +++ b/pype/ftrack/lib/custom_db_connector.py @@ -20,7 +20,13 @@ import requests import pymongo from pymongo.client_session import ClientSession -class NotActiveTable(Exception): pass +class NotActiveTable(Exception): + def __init__(self, *args, **kwargs): + msg = "Active table is not set. (This is bug)" + if not (args or kwargs): + args = (default_message,) + super().__init__(*args, **kwargs) + def auto_reconnect(func): """Handling auto reconnect in 3 retry times""" @@ -35,12 +41,11 @@ def auto_reconnect(func): time.sleep(0.1) else: raise - return decorated def check_active_table(func): - """Handling auto reconnect in 3 retry times""" + """Check if DbConnector has active table before db method is called""" @functools.wraps(func) def decorated(obj, *args, **kwargs): if not obj.active_table: From e0827d0597ebba4660ed144a89443314321e3929 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 18 Oct 2019 11:20:32 +0200 Subject: [PATCH 11/15] added __getattribute__ to have acces to not implemented methods of pymongo db --- pype/ftrack/lib/custom_db_connector.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/pype/ftrack/lib/custom_db_connector.py b/pype/ftrack/lib/custom_db_connector.py index 931b46fc00..102cbe2526 100644 --- a/pype/ftrack/lib/custom_db_connector.py +++ b/pype/ftrack/lib/custom_db_connector.py @@ -49,14 +49,12 @@ def check_active_table(func): @functools.wraps(func) def decorated(obj, *args, **kwargs): if not obj.active_table: - raise NotActiveTable("Active table is not set. (This is bug)") + raise NotActiveTable() return func(obj, *args, **kwargs) - return decorated class DbConnector: - log = logging.getLogger(__name__) timeout = 1000 @@ -72,8 +70,19 @@ class DbConnector: self.active_table = table_name def __getitem__(self, key): + # gives direct access to collection withou setting `active_table` return self._database[key] + def __getattribute__(self, attr): + # not all methods of PyMongo database are implemented with this it is + # possible to use them too + try: + return super(DbConnector, self).__getattribute__(attr) + except AttributeError: + if self.active_table is None: + raise NotActiveTable() + return self._database[self.active_table].__getattribute__(attr) + def install(self): """Establish a persistent connection to the database""" if self._is_installed: From d1b294e4a20b1af10848d703bce6f18f6bbc9524 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 18 Oct 2019 11:21:36 +0200 Subject: [PATCH 12/15] implemented options to methods args to have access to use all optional args --- pype/ftrack/lib/custom_db_connector.py | 69 +++++++++++--------------- 1 file changed, 28 insertions(+), 41 deletions(-) diff --git a/pype/ftrack/lib/custom_db_connector.py b/pype/ftrack/lib/custom_db_connector.py index 102cbe2526..9ffcc1f6c8 100644 --- a/pype/ftrack/lib/custom_db_connector.py +++ b/pype/ftrack/lib/custom_db_connector.py @@ -156,7 +156,7 @@ class DbConnector: @check_active_table @auto_reconnect - def insert_one(self, item, session=None): + def insert_one(self, item, **options): assert isinstance(item, dict), "item must be of type " return self._database[self.active_table].insert_one( item, @@ -165,92 +165,79 @@ class DbConnector: @check_active_table @auto_reconnect - def insert_many(self, items, ordered=True, session=None): + def insert_many(self, items, ordered=True, **options): # check if all items are valid assert isinstance(items, list), "`items` must be of type " for item in items: assert isinstance(item, dict), "`item` must be of type " - return self._database[self.active_table].insert_many( - items, - ordered=ordered, - session=session - ) + options["ordered"] = ordered + return self._database[self.active_table].insert_many(items, **options) @check_active_table @auto_reconnect - def find(self, filter, projection=None, sort=None, session=None): + def find(self, filter, projection=None, sort=None, **options): + options["sort"] = sort return self._database[self.active_table].find( - filter=filter, - projection=projection, - sort=sort, - session=session + filter, projection, **options ) @check_active_table @auto_reconnect - def find_one(self, filter, projection=None, sort=None, session=None): + def find_one(self, filter, projection=None, sort=None, **options): assert isinstance(filter, dict), "filter must be " - + options["sort"] = sort return self._database[self.active_table].find_one( - filter=filter, - projection=projection, - sort=sort, - session=session + filter, + projection, + **options ) @check_active_table @auto_reconnect - def replace_one(self, filter, replacement, **kw): + def replace_one(self, filter, replacement, **options): return self._database[self.active_table].replace_one( - filter, replacement, **kw + filter, replacement, **options ) @check_active_table @auto_reconnect - def update_one(self, filter, update, session=None): + def update_one(self, filter, update, **options): return self._database[self.active_table].update_one( - filter, update, - session=session + filter, update, **options ) @check_active_table @auto_reconnect - def update_many(self, filter, update, session=None): + def update_many(self, filter, update, **options): return self._database[self.active_table].update_many( - filter, update, - session=session + filter, update, **options ) @check_active_table @auto_reconnect - def distinct(self, *args, **kwargs): - return self._database[self.active_table].distinct( - *args, **kwargs - ) + def distinct(self, **options): + return self._database[self.active_table].distinct(**options) @check_active_table @auto_reconnect - def drop_collection(self, name_or_collection, session=None): + def drop_collection(self, name_or_collection, **options): return self._database[self.active_table].drop( - name_or_collection, - session=session + name_or_collection, **options ) @check_active_table @auto_reconnect - def delete_one(self, filter, collation=None, session=None): + def delete_one(self, filter, collation=None, **options): + options["collation"] = collation return self._database[self.active_table].delete_one( - filter, - collation=collation, - session=session + filter, **options ) @check_active_table @auto_reconnect - def delete_many(self, filter, collation=None, session=None): + def delete_many(self, filter, collation=None, **options): + options["collation"] = collation return self._database[self.active_table].delete_many( - filter, - collation=collation, - session=session + filter, **options ) From ee143aaee095237ad863d25ef74c3e524bc00180 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 21 Oct 2019 13:37:09 +0200 Subject: [PATCH 13/15] added possibility to turn off auto regex without entity specificment --- pype/services/rest_api/base_class.py | 20 +++++++++++++------- pype/services/rest_api/lib/factory.py | 16 ++++++++++------ pype/services/rest_api/rest_api.py | 19 ++++++++++++++----- 3 files changed, 37 insertions(+), 18 deletions(-) diff --git a/pype/services/rest_api/base_class.py b/pype/services/rest_api/base_class.py index 5355c5886c..8a3c9d3704 100644 --- a/pype/services/rest_api/base_class.py +++ b/pype/services/rest_api/base_class.py @@ -7,7 +7,7 @@ from .lib import ( ) -def route(path, url_prefix="", methods=[]): +def route(path, url_prefix="", methods=[], strict_match=False): """Decorator that register callback and all its attributes. Callback is registered to Singleton RestApiFactory. @@ -15,8 +15,10 @@ def route(path, url_prefix="", methods=[]): :type path: str :param url_prefix: Specify prefix of path, defaults to "/". :type url_prefix: str, list, optional - :param methods: Specify request method (GET, POST, PUT, UPDATE, DELETE) when callback will be triggered, defaults to ["GET"] + :param methods: Specify request method (GET, POST, PUT, etc.) when callback will be triggered, defaults to ["GET"] :type methods: list, str, optional + :param strict_match: Decides if callback can handle both single and multiple entities (~/projects/ && ~/projects/), defaults to False. + :type strict_match: bool `path` may include dynamic keys that will be stored to object which can be obtained in callback. @@ -35,7 +37,9 @@ def route(path, url_prefix="", methods=[]): callback. """ def decorator(callback): - RestApiFactory.register_route(path, callback, url_prefix, methods) + RestApiFactory.register_route( + path, callback, url_prefix, methods, strict_match + ) callback.restapi = True return callback return decorator @@ -84,12 +88,14 @@ class RestApi: It is possible to use decorators in another class only when object, of class where decorators are, is registered to RestApiFactory. """ - def route(path, url_prefix="", methods=[]): - return route(path, url_prefix, methods) + def route(path, url_prefix="", methods=[], strict_match=False): + return route(path, url_prefix, methods, strict_match) @classmethod - def register_route(cls, callback, path, url_prefix="", methods=[]): - return route(path, methods, url_prefix)(callback) + def register_route( + cls, callback, path, url_prefix="", methods=[], strict_match=False + ): + return route(path, methods, url_prefix, strict_match)(callback) @classmethod def register_statics(cls, url_prefix, dir_path): diff --git a/pype/services/rest_api/lib/factory.py b/pype/services/rest_api/lib/factory.py index 3c9f84cc85..a07a8262c1 100644 --- a/pype/services/rest_api/lib/factory.py +++ b/pype/services/rest_api/lib/factory.py @@ -47,7 +47,7 @@ def prepare_fullpath(path, prefix): return fullpath -def prepare_regex_from_path(full_path): +def prepare_regex_from_path(full_path, strict_match): """Prepare regex based on set path. When registered path do not contain dynamic keys regex is not set. @@ -68,8 +68,9 @@ def prepare_regex_from_path(full_path): for key in all_founded_keys: replacement = "(?P{}\w+)".format(key) keys.append(key.replace("<", "").replace(">", "")) - if full_path.endswith(key): - replacement = "?{}?".format(replacement) + if not strict_match: + if full_path.endswith(key): + replacement = "?{}?".format(replacement) regex_path = regex_path.replace(key, replacement) regex_path = "^{}$".format(regex_path) @@ -227,7 +228,7 @@ class _RestApiFactory: self.unprocessed_statics.index(item) ) - def register_route(self, path, callback, url_prefix, methods): + def register_route(self, path, callback, url_prefix, methods, strict_match): log.debug("Registering callback for item \"{}\"".format( callback.__qualname__ )) @@ -235,7 +236,8 @@ class _RestApiFactory: "path": path, "callback": callback, "url_prefix": url_prefix, - "methods": methods + "methods": methods, + "strict_match": strict_match } self.unprocessed_routes.append(route) @@ -260,7 +262,9 @@ class _RestApiFactory: methods = prepare_methods(route["methods"], callback) url_prefix = prepare_prefix(route["url_prefix"]) fullpath = prepare_fullpath(route["path"], url_prefix) - regex, regex_keys = prepare_regex_from_path(fullpath) + regex, regex_keys = prepare_regex_from_path( + fullpath, route["strict_match"] + ) callback_info = prepare_callback_info(callback) for method in methods: diff --git a/pype/services/rest_api/rest_api.py b/pype/services/rest_api/rest_api.py index 2df666b45a..8dadbd524a 100644 --- a/pype/services/rest_api/rest_api.py +++ b/pype/services/rest_api/rest_api.py @@ -24,7 +24,7 @@ class RestApiServer: or created object, with used decorator, is registered with `register_obj`. .. code-block:: python - @route("/username", url_prefix="/api", methods=["get"]) + @route("/username", url_prefix="/api", methods=["get"], strict_match=False) def get_username(): return {"username": getpass.getuser()} @@ -51,7 +51,7 @@ class RestApiServer: "Proj2": {"proj_data": []}, } - @route("/projects/", url_prefix="/api", methods=["get"]) + @route("/projects/", url_prefix="/api", methods=["get"], strict_match=False) def get_projects(request_info): project_name = request_info.url_data["project_name"] if not project_name: @@ -64,7 +64,7 @@ class RestApiServer: .. code-block:: python from rest_api import abort - @route("/projects/", url_prefix="/api", methods=["get"]) + @route("/projects/", url_prefix="/api", methods=["get"], strict_match=False) def get_projects(request_info): project_name = request_info.url_data["project_name"] if not project_name: @@ -75,6 +75,11 @@ class RestApiServer: abort(404, "Project \"{}\".format(project_name) was not found") return project + `strict_match` allows to handle not only specific entity but all entity types. + E.g. "/projects/" with set `strict_match` to False will handle also + "/projects" or "/projects/" path. It is necessary to set `strict_match` to + True when should handle only single entity. + Callback may return many types. For more information read docstring of `_handle_callback_result` defined in handler. """ @@ -102,8 +107,12 @@ class RestApiServer: self.qaction = qaction self.failed_icon = failed_icon - def register_callback(self, path, callback, url_prefix="", methods=[]): - RestApiFactory.register_route(path, callback, url_prefix, methods) + def register_callback( + self, path, callback, url_prefix="", methods=[], strict_match=False + ): + RestApiFactory.register_route( + path, callback, url_prefix, methods, strict_match + ) # route(path, url_prefix, methods)(callback) def register_statics(self, url_prefix, dir_path): From a2ae970b650600f4dc3db4fa26c0a8cafe62136e Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 21 Oct 2019 13:42:14 +0200 Subject: [PATCH 14/15] added docstrings --- pype/avalon_apps/rest_api.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pype/avalon_apps/rest_api.py b/pype/avalon_apps/rest_api.py index d58d080339..ae027383a1 100644 --- a/pype/avalon_apps/rest_api.py +++ b/pype/avalon_apps/rest_api.py @@ -52,6 +52,7 @@ class AvalonRestApi(RestApi): output = self.result_to_json(assets) return CallbackResult(data=output) + # identificator can be specified with url query (default is `name`) identificator = request.query.get("identificator", "name") asset = self.dbcon[_project_name].find_one({ @@ -68,6 +69,12 @@ class AvalonRestApi(RestApi): )) def result_to_json(self, result): + """ Converts result of MongoDB query to dict without $oid (ObjectId) + keys with help of regex matching. + + ..note: + This will convert object type entries similar to ObjectId. + """ bson_json = bson.json_util.dumps(result) # Replace "{$oid: "{entity id}"}" with "{entity id}" regex1 = '(?P{\"\$oid\": \"[^\"]+\"})' From 588199821a138dc2e2857b675edd15a2c423334a Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 24 Oct 2019 15:25:45 +0200 Subject: [PATCH 15/15] removed statics server since rest api server can handle statics too --- pype/services/rest_api/rest_api.py | 4 +- pype/services/statics_server/__init__.py | 5 - .../services/statics_server/statics_server.py | 202 ------------------ 3 files changed, 3 insertions(+), 208 deletions(-) delete mode 100644 pype/services/statics_server/__init__.py delete mode 100644 pype/services/statics_server/statics_server.py diff --git a/pype/services/rest_api/rest_api.py b/pype/services/rest_api/rest_api.py index 8dadbd524a..d70bcfc7c5 100644 --- a/pype/services/rest_api/rest_api.py +++ b/pype/services/rest_api/rest_api.py @@ -102,6 +102,9 @@ class RestApiServer: statics_dir = os.path.sep.join([os.environ["PYPE_MODULE_ROOT"], "res"]) self.register_statics("/res", statics_dir) + os.environ["PYPE_STATICS_SERVER"] = "{}/res".format( + os.environ["PYPE_REST_API_URL"] + ) def set_qaction(self, qaction, failed_icon): self.qaction = qaction @@ -113,7 +116,6 @@ class RestApiServer: RestApiFactory.register_route( path, callback, url_prefix, methods, strict_match ) - # route(path, url_prefix, methods)(callback) def register_statics(self, url_prefix, dir_path): register_statics(url_prefix, dir_path) diff --git a/pype/services/statics_server/__init__.py b/pype/services/statics_server/__init__.py deleted file mode 100644 index 4b2721b18b..0000000000 --- a/pype/services/statics_server/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from .statics_server import StaticsServer - - -def tray_init(tray_widget, main_widget): - return StaticsServer() diff --git a/pype/services/statics_server/statics_server.py b/pype/services/statics_server/statics_server.py deleted file mode 100644 index 8655cd9df9..0000000000 --- a/pype/services/statics_server/statics_server.py +++ /dev/null @@ -1,202 +0,0 @@ -import os -import sys -import datetime -import socket -import http.server -from http import HTTPStatus -import urllib -import posixpath -import socketserver - -from Qt import QtCore -from pypeapp import config, Logger - - -DIRECTORY = os.path.sep.join([os.environ['PYPE_MODULE_ROOT'], 'res']) - - -class Handler(http.server.SimpleHTTPRequestHandler): - def __init__(self, *args, **kwargs): - py_version = sys.version.split('.') - # If python version is 3.7 or higher - if int(py_version[0]) >= 3 and int(py_version[1]) >= 7: - super().__init__(*args, directory=DIRECTORY, **kwargs) - else: - self.directory = DIRECTORY - super().__init__(*args, **kwargs) - - def send_head(self): - """Common code for GET and HEAD commands. - - This sends the response code and MIME headers. - - Return value is either a file object (which has to be copied - to the outputfile by the caller unless the command was HEAD, - and must be closed by the caller under all circumstances), or - None, in which case the caller has nothing further to do. - - """ - path = self.translate_path(self.path) - f = None - if os.path.isdir(path): - parts = urllib.parse.urlsplit(self.path) - if not parts.path.endswith('/'): - # redirect browser - doing basically what apache does - self.send_response(HTTPStatus.MOVED_PERMANENTLY) - new_parts = (parts[0], parts[1], parts[2] + '/', - parts[3], parts[4]) - new_url = urllib.parse.urlunsplit(new_parts) - self.send_header("Location", new_url) - self.end_headers() - return None - for index in "index.html", "index.htm": - index = os.path.join(path, index) - if os.path.exists(index): - path = index - break - else: - return self.list_directory(path) - ctype = self.guess_type(path) - try: - f = open(path, 'rb') - except OSError: - self.send_error(HTTPStatus.NOT_FOUND, "File not found") - return None - - try: - fs = os.fstat(f.fileno()) - # Use browser cache if possible - if ("If-Modified-Since" in self.headers - and "If-None-Match" not in self.headers): - # compare If-Modified-Since and time of last file modification - try: - ims = http.server.email.utils.parsedate_to_datetime( - self.headers["If-Modified-Since"]) - except (TypeError, IndexError, OverflowError, ValueError): - # ignore ill-formed values - pass - else: - if ims.tzinfo is None: - # obsolete format with no timezone, cf. - # https://tools.ietf.org/html/rfc7231#section-7.1.1.1 - ims = ims.replace(tzinfo=datetime.timezone.utc) - if ims.tzinfo is datetime.timezone.utc: - # compare to UTC datetime of last modification - last_modif = datetime.datetime.fromtimestamp( - fs.st_mtime, datetime.timezone.utc) - # remove microseconds, like in If-Modified-Since - last_modif = last_modif.replace(microsecond=0) - - if last_modif <= ims: - self.send_response(HTTPStatus.NOT_MODIFIED) - self.end_headers() - f.close() - return None - - self.send_response(HTTPStatus.OK) - self.send_header("Content-type", ctype) - self.send_header("Content-Length", str(fs[6])) - self.send_header("Last-Modified", - self.date_time_string(fs.st_mtime)) - self.end_headers() - return f - except: - f.close() - raise - - def translate_path(self, path): - """Translate a /-separated PATH to the local filename syntax. - - Components that mean special things to the local file system - (e.g. drive or directory names) are ignored. (XXX They should - probably be diagnosed.) - - """ - # abandon query parameters - path = path.split('?',1)[0] - path = path.split('#',1)[0] - # Don't forget explicit trailing slash when normalizing. Issue17324 - trailing_slash = path.rstrip().endswith('/') - try: - path = urllib.parse.unquote(path, errors='surrogatepass') - except UnicodeDecodeError: - path = urllib.parse.unquote(path) - path = posixpath.normpath(path) - words = path.split('/') - words = filter(None, words) - path = self.directory - for word in words: - if os.path.dirname(word) or word in (os.curdir, os.pardir): - # Ignore components that are not a simple file/directory name - continue - path = os.path.join(path, word) - if trailing_slash: - path += '/' - return path - - -class StaticsServer(QtCore.QThread): - """ Measure user's idle time in seconds. - Idle time resets on keyboard/mouse input. - Is able to emit signals at specific time idle. - """ - def __init__(self): - super(StaticsServer, self).__init__() - self.qaction = None - self.failed_icon = None - self._is_running = False - self.log = Logger().get_logger(self.__class__.__name__) - try: - self.presets = config.get_presets().get( - 'services', {}).get('statics_server') - except Exception: - self.presets = {'default_port': 8010, 'exclude_ports': []} - - self.port = self.find_port() - - def set_qaction(self, qaction, failed_icon): - self.qaction = qaction - self.failed_icon = failed_icon - - def tray_start(self): - self.start() - - @property - def is_running(self): - return self._is_running - - def stop(self): - self._is_running = False - - def run(self): - self._is_running = True - try: - with socketserver.TCPServer(("", self.port), Handler) as httpd: - while self._is_running: - httpd.handle_request() - except Exception: - self.log.warning( - 'Statics Server service has failed', exc_info=True - ) - self._is_running = False - if self.qaction and self.failed_icon: - self.qaction.setIcon(self.failed_icon) - - def find_port(self): - start_port = self.presets['default_port'] - exclude_ports = self.presets['exclude_ports'] - found_port = None - # port check takes time so it's lowered to 100 ports - for port in range(start_port, start_port+100): - if port in exclude_ports: - continue - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: - result = sock.connect_ex(('localhost', port)) - if result != 0: - found_port = port - if found_port is not None: - break - if found_port is None: - return None - os.environ['PYPE_STATICS_SERVER'] = 'http://localhost:{}'.format(found_port) - return found_port