Frame13/ayon-core
1
0
Fork 1
mirror of https://github.com/ynput/ayon-core.git synced 2025-12-24 21:04:40 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merged in feature/advanced_rest_api (pull request #334)

Browse source 
Feature/advanced rest api

Approved-by: Milan Kolar <milan@orbi.tools>
This commit is contained in:
Jakub Trllo 2019-11-19 14:52:30 +00:00 • committed by Milan Kolar
commit d15c57b57f
14 changed files with 1315 additions and 441 deletions
Download patch file Download diff file Expand all files Collapse all files

5
pype/avalon_apps/avalon_app.py
View file

@ -14,6 +14,11 @@ class AvalonApps:
self.parent = parent
self.app_launcher = None
def process_modules(self, modules):
if "RestApiServer" in modules:
from .rest_api import AvalonRestApi
self.rest_api_obj = AvalonRestApi()
# Definition of Tray menu
def tray_menu(self, parent_menu=None):
# Actions

86
pype/avalon_apps/rest_api.py Normal file
View file

@ -0,0 +1,86 @@
import os
import re
import json
import bson
import bson.json_util
from pype.services.rest_api import RestApi, abort, CallbackResult
from pype.ftrack.lib.custom_db_connector import DbConnector
class AvalonRestApi(RestApi):
dbcon = DbConnector(
os.environ["AVALON_MONGO"],
os.environ["AVALON_DB"]
)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.dbcon.install()
@RestApi.route("/projects/<project_name>", url_prefix="/avalon", methods="GET")
def get_project(self, request):
project_name = request.url_data["project_name"]
if not project_name:
output = {}
for project_name in self.dbcon.tables():
project = self.dbcon[project_name].find_one({"type": "project"})
output[project_name] = project
return CallbackResult(data=self.result_to_json(output))
project = self.dbcon[project_name].find_one({"type": "project"})
if project:
return CallbackResult(data=self.result_to_json(project))
abort(404, "Project \"{}\" was not found in database".format(
project_name
))
@RestApi.route("/projects/<project_name>/assets/<asset>", url_prefix="/avalon", methods="GET")
def get_assets(self, request):
_project_name = request.url_data["project_name"]
_asset = request.url_data["asset"]
if not self.dbcon.exist_table(_project_name):
abort(404, "Project \"{}\" was not found in database".format(
project_name
))
if not _asset:
assets = self.dbcon[_project_name].find({"type": "asset"})
output = self.result_to_json(assets)
return CallbackResult(data=output)
# identificator can be specified with url query (default is `name`)
identificator = request.query.get("identificator", "name")
asset = self.dbcon[_project_name].find_one({
"type": "asset",
identificator: _asset
})
if asset:
id = asset["_id"]
asset["_id"] = str(id)
return asset
abort(404, "Asset \"{}\" with {} was not found in project {}".format(
_asset, identificator, project_name
))
def result_to_json(self, result):
""" Converts result of MongoDB query to dict without $oid (ObjectId)
keys with help of regex matching.
..note:
This will convert object type entries similar to ObjectId.
"""
bson_json = bson.json_util.dumps(result)
# Replace "{$oid: "{entity id}"}" with "{entity id}"
regex1 = '(?P<id>{\"\$oid\": \"[^\"]+\"})'
regex2 = '{\"\$oid\": (?P<id>\"[^\"]+\")}'
for value in re.findall(regex1, bson_json):
for substr in re.findall(regex2, value):
bson_json = bson_json.replace(value, substr)
return json.loads(bson_json)

61
pype/ftrack/lib/custom_db_connector.py
View file

@ -22,7 +22,12 @@ import pymongo
from pymongo.client_session import ClientSession
class NotActiveTable(Exception):
pass
def __init__(self, *args, **kwargs):
msg = "Active table is not set. (This is bug)"
if not (args or kwargs):
args = (default_message,)
super().__init__(*args, **kwargs)
def auto_reconnect(func):
"""Handling auto reconnect in 3 retry times"""
@ -37,7 +42,16 @@ def auto_reconnect(func):
time.sleep(0.1)
else:
raise
return decorated
def check_active_table(func):
"""Check if DbConnector has active table before db method is called"""
@functools.wraps(func)
def decorated(obj, *args, **kwargs):
if not obj.active_table:
raise NotActiveTable()
return func(obj, *args, **kwargs)
return decorated
@ -53,7 +67,6 @@ def check_active_table(func):
class DbConnector:
log = logging.getLogger(__name__)
timeout = 1000
@ -68,10 +81,18 @@ class DbConnector:
self.active_table = table_name
def __getitem__(self, key):
# gives direct access to collection withou setting `active_table`
return self._database[key]
def __getattribute__(self, attr):
# not all methods of PyMongo database are implemented with this it is
# possible to use them too
try:
return super().__getattribute__(attr)
return super(DbConnector, self).__getattribute__(attr)
except AttributeError:
if self.active_table is None:
raise NotActiveTable()
return self._database[self.active_table].__getattribute__(attr)
def install(self):
@ -131,6 +152,15 @@ class DbConnector:
def exist_table(self, table_name):
return table_name in self.tables()
def create_table(self, name, **options):
if self.exist_table(name):
return
return self._database.create_collection(name, **options)
def exist_table(self, table_name):
return table_name in self.tables()
def tables(self):
"""List available tables
Returns:
@ -166,18 +196,21 @@ class DbConnector:
@check_active_table
@auto_reconnect
def find(self, filter, projection=None, sort=None, **options):
options["projection"] = projection
options["sort"] = sort
return self._database[self.active_table].find(filter, **options)
return self._database[self.active_table].find(
filter, projection, **options
)
@check_active_table
@auto_reconnect
def find_one(self, filter, projection=None, sort=None, **options):
assert isinstance(filter, dict), "filter must be <dict>"
options["projection"] = projection
options["sort"] = sort
return self._database[self.active_table].find_one(filter, **options)
return self._database[self.active_table].find_one(
filter,
projection,
**options
)
@check_active_table
@auto_reconnect
@ -202,8 +235,8 @@ class DbConnector:
@check_active_table
@auto_reconnect
def distinct(self, *args, **kwargs):
return self._database[self.active_table].distinct(*args, **kwargs)
def distinct(self, **options):
return self._database[self.active_table].distinct(**options)
@check_active_table
@auto_reconnect
@ -216,10 +249,14 @@ class DbConnector:
@auto_reconnect
def delete_one(self, filter, collation=None, **options):
options["collation"] = collation
return self._database[self.active_table].delete_one(filter, **options)
return self._database[self.active_table].delete_one(
filter, **options
)
@check_active_table
@auto_reconnect
def delete_many(self, filter, collation=None, **options):
options["collation"] = collation
return self._database[self.active_table].delete_many(filter, **options)
return self._database[self.active_table].delete_many(
filter, **options
)

4
pype/muster/muster.py
View file

@ -43,8 +43,10 @@ class MusterModule:
self.aShowLogin.trigger()
if "RestApiServer" in modules:
def api_show_login():
self.aShowLogin.trigger()
modules["RestApiServer"].register_callback(
"muster/show_login", api_callback, "post"
"/show_login", api_show_login, "muster", "post"
)
# Definition of Tray menu

2
pype/services/rest_api/__init__.py
View file

@ -1,4 +1,6 @@
from .rest_api import RestApiServer
from .base_class import RestApi, abort, route, register_statics
from .lib import RestMethods, CallbackResult
def tray_init(tray_widget, main_widget):

114
pype/services/rest_api/base_class.py Normal file
View file

@ -0,0 +1,114 @@
from functools import wraps
from http import HTTPStatus
from .lib import (
RestApiFactory, Splitter,
ObjAlreadyExist, AbortException,
)
def route(path, url_prefix="", methods=[], strict_match=False):
"""Decorator that register callback and all its attributes.
Callback is registered to Singleton RestApiFactory.
:param path: Specify url path when callback should be triggered.
:type path: str
:param url_prefix: Specify prefix of path, defaults to "/".
:type url_prefix: str, list, optional
:param methods: Specify request method (GET, POST, PUT, etc.) when callback will be triggered, defaults to ["GET"]
:type methods: list, str, optional
:param strict_match: Decides if callback can handle both single and multiple entities (~/projects/<project_name> && ~/projects/), defaults to False.
:type strict_match: bool
`path` may include dynamic keys that will be stored to object which can
be obtained in callback.
Example:
- registered path: "/projects/<project_name>"
- url request path: "/projects/S001_test_project"
In this case will be callback triggered and in accessible data will be
stored {"project_name": "S001_test_project"}.
`url_prefix` is optional but it is better to specify for easier filtering
of requests.
Example:
- url_prefix: `"/avalon"` or `["avalon"]`
- path: `"/projects"`
In this case request path must be "/avalon/projects" to trigger registered
callback.
"""
def decorator(callback):
RestApiFactory.register_route(
path, callback, url_prefix, methods, strict_match
)
callback.restapi = True
return callback
return decorator
def register_statics(url_prefix, dir_path):
"""Decorator that register callback and all its attributes.
Callback is registered to Singleton RestApiFactory.
:param url_prefix: Specify prefix of path, defaults to "/". (Example: "/resources")
:type url_prefix: str
:param dir_path: Path to file folder where statics are located.
:type dir_path: str
"""
RestApiFactory.register_statics((url_prefix, dir_path))
def abort(status_code=HTTPStatus.NOT_FOUND, message=None):
"""Should be used to stop registered callback
`abort` raise AbortException that is handled with request Handler which
returns entered status and may send optional message in body.
:param status_code: Status that will be send in reply of request, defaults to 404
:type status_code: int
:param message: Message to send in body, default messages are based on statuc_code in Handler, defaults to None
:type message: str, optional
...
:raises AbortException: This exception is handled in Handler to know about launched `abort`
"""
items = []
items.append(str(status_code))
if not message:
message = ""
items.append(message)
raise AbortException(Splitter.join(items))
class RestApi:
"""Base class for RestApi classes.
Use this class is required when it is necessary to have class for handling
requests and want to use decorators for registering callbacks.
It is possible to use decorators in another class only when object, of class
where decorators are, is registered to RestApiFactory.
"""
def route(path, url_prefix="", methods=[], strict_match=False):
return route(path, url_prefix, methods, strict_match)
@classmethod
def register_route(
cls, callback, path, url_prefix="", methods=[], strict_match=False
):
return route(path, methods, url_prefix, strict_match)(callback)
@classmethod
def register_statics(cls, url_prefix, dir_path):
return register_statics(url_prefix, dir_path)
@classmethod
def abort(cls, status_code=HTTPStatus.NOT_FOUND, message=None):
abort(status_code, message)
def __new__(cls, *args, **kwargs):
for obj in RestApiFactory.registered_objs:
if type(obj) == cls:
raise ObjAlreadyExist(cls)
instance = super(RestApi, cls).__new__(cls)
RestApiFactory.register_obj(instance)
return instance

9
pype/services/rest_api/lib/__init__.py Normal file
View file

@ -0,0 +1,9 @@
Splitter = "__splitter__"
from .exceptions import ObjAlreadyExist, AbortException
from .lib import RestMethods, CustomNone, CallbackResult, RequestInfo
from .factory import _RestApiFactory
RestApiFactory = _RestApiFactory()
from .handler import Handler

11
pype/services/rest_api/lib/exceptions.py Normal file
View file

@ -0,0 +1,11 @@
class ObjAlreadyExist(Exception):
"""Is used when is created multiple objects of same RestApi class."""
def __init__(self, cls=None, message=None):
if not (cls and message):
message = "RestApi object was created twice."
elif not message:
message = "{} object was created twice.".format(cls.__name__)
super().__init__(message)
class AbortException(Exception): pass

353
pype/services/rest_api/lib/factory.py Normal file
View file

@ -0,0 +1,353 @@
import os
import re
import inspect
import collections
from .lib import RestMethods
from pypeapp import Logger
log = Logger().get_logger("RestApiFactory")
def prepare_fullpath(path, prefix):
"""Concatenate registered path and prefix with right form.
:param path: Registered url path for registered callback.
:type path: str, list
:param prefix: Registered and prepared url prefix.
:type prefix: str, None
:return: concatenated prefix and path in right form
:rtype: str
"""
if isinstance(path, (list, tuple)):
path_items = path
else:
path_items = [part for part in path.split("/") if part]
fullpath = "/"
if path and prefix:
items = [part for part in prefix.split("/") if part]
items.extend(path_items)
fullpath = "/".join(items)
if path.endswith("/"):
fullpath += "/"
elif path:
fullpath = "/".join(path_items)
if path.endswith("/"):
fullpath += "/"
elif prefix:
fullpath = prefix
if not fullpath.startswith("/"):
fullpath = "/{}".format(fullpath)
return fullpath
def prepare_regex_from_path(full_path, strict_match):
"""Prepare regex based on set path.
When registered path do not contain dynamic keys regex is not set.
Dynamic keys are specified with "<" and ">" ("<{dynamic key}>").
:param full_path: Full url path (prefix + path) for registered callback.
:type full_path: str, list, None
:return: regex and keys of all groups in regex
:rtype: tuple(SRE_Pattern, list), tuple(None, None)
"""
get_indexes_regex = "<[^< >]+>"
all_founded_keys = re.findall(get_indexes_regex, full_path)
if not all_founded_keys:
return None, None
regex_path = full_path
keys = []
for key in all_founded_keys:
replacement = "(?P{}\w+)".format(key)
keys.append(key.replace("<", "").replace(">", ""))
if not strict_match:
if full_path.endswith(key):
replacement = "?{}?".format(replacement)
regex_path = regex_path.replace(key, replacement)
regex_path = "^{}$".format(regex_path)
return re.compile(regex_path), keys
def prepare_prefix(url_prefix):
"""Check if the url_prefix is set and is in correct form.
Output is None when prefix is empty or "/".
:param url_prefix: Registered prefix of registered callback.
:type url_prefix: str, list, None
:return: Url prefix of registered callback
:rtype: str, None
"""
if url_prefix is None or url_prefix.strip() == "/":
return None
elif isinstance(url_prefix, (list, tuple)):
url_prefix = "/".join(url_prefix)
else:
items = [part for part in url_prefix.split("/") if part]
url_prefix = "/".join(items)
if not url_prefix:
return None
while url_prefix.endswith("/"):
url_prefix = url_prefix[:-1]
if not url_prefix.startswith("/"):
url_prefix = "/{}".format(url_prefix)
return url_prefix
def prepare_methods(methods, callback=None):
"""Check and convert entered methods.
String `methods` is converted to list. All values are converted to
`RestMethods` enum object. Invalid methods are ignored and printed out.
:param methods: Contain rest api methods, when callback is called.
:type methods: str, list
:param callback: Registered callback, helps to identify where is invalid method.
:type callback: function, method, optional
:return: Valid methods
:rtype: list
"""
invalid_methods = collections.defaultdict(list)
if not methods:
_methods = [RestMethods.GET]
elif isinstance(methods, str) or isinstance(methods, RestMethods):
_method = RestMethods.get(methods)
_methods = []
if _method is None:
invalid_methods[methods].append(callback)
else:
_methods.append(_method)
else:
_methods = []
for method in methods:
found = False
_method = RestMethods.get(method)
if _method == None:
invalid_methods[methods].append(callback)
continue
_methods.append(_method)
for method, callbacks in invalid_methods.items():
callback_info = ""
callbacks = [cbk for cbk in callbacks if cbk]
if len(callbacks) > 0:
multiple_ind = ""
if len(callbacks) > 1:
multiple_ind = "s"
callback_items = []
for callback in callbacks:
callback_items.append("\"{}<{}>\"".format(
callback.__qualname__, callback.__globals__["__file__"]
))
callback_info = " with callback{} {}".format(
multiple_ind, "| ".join(callback_items)
)
log.warning(
("Invalid RestApi method \"{}\"{}").format(method, callback_info)
)
return _methods
def prepare_callback_info(callback):
"""Prepare data for callback handling when should be triggered."""
callback_info = inspect.getfullargspec(callback)
callback_args = callback_info.args
callback_args_len = 0
if callback_args:
callback_args_len = len(callback_args)
if type(callback).__name__ == "method":
callback_args_len -= 1
defaults = callback_info.defaults
defaults_len = 0
if defaults:
defaults_len = len(defaults)
annotations = callback_info.annotations
return {
"args": callback_args,
"args_len": callback_args_len,
"defaults": defaults,
"defaults_len": defaults_len,
"hasargs": callback_info.varargs is not None,
"haskwargs": callback_info.varkw is not None,
"annotations": annotations
}
class _RestApiFactory:
"""Factory is used to store and prepare callbacks for requests.
Should be created only one object used for all registered callbacks when
it is expected to run only one http server.
"""
registered_objs = []
unprocessed_routes = []
unprocessed_statics = []
prepared_routes = {
method: collections.defaultdict(list) for method in RestMethods
}
prepared_statics = {}
has_routes = False
def has_handlers(self):
return (self.has_routes or self.prepared_statics)
def _process_route(self, route):
return self.unprocessed_routes.pop(
self.unprocessed_routes.index(route)
)
def _process_statics(self, item):
return self.unprocessed_statics.pop(
self.unprocessed_statics.index(item)
)
def register_route(self, path, callback, url_prefix, methods, strict_match):
log.debug("Registering callback for item \"{}\"".format(
callback.__qualname__
))
route = {
"path": path,
"callback": callback,
"url_prefix": url_prefix,
"methods": methods,
"strict_match": strict_match
}
self.unprocessed_routes.append(route)
def register_obj(self, obj):
"""Register object for decorated methods in class definition"""
self.registered_objs.append(obj)
def register_statics(self, item):
log.debug("Registering statics path \"{}\"".format(item))
self.unprocessed_statics.append(item)
def _prepare_route(self, route):
"""Prepare data of registered callbacks for routes.
Registration info are prepared to easy filter during handling
of requests.
:param route: Contain all necessary info for filtering and handling callback for registered route.
:type route: dict
"""
callback = route["callback"]
methods = prepare_methods(route["methods"], callback)
url_prefix = prepare_prefix(route["url_prefix"])
fullpath = prepare_fullpath(route["path"], url_prefix)
regex, regex_keys = prepare_regex_from_path(
fullpath, route["strict_match"]
)
callback_info = prepare_callback_info(callback)
for method in methods:
self.has_routes = True
self.prepared_routes[method][url_prefix].append({
"regex": regex,
"regex_keys": regex_keys,
"fullpath": fullpath,
"callback": callback,
"callback_info": callback_info
})
def prepare_registered(self):
"""Iterate through all registered callbacks and statics and prepare them
First are processed callbacks registered with decorators in classes by
registered objects. Remaining callbacks are filtered, it is checked if
methods has `__self__` or are defined in <locals> (it is expeted they
do not requise access to object)
"""
for url_prefix, dir_path in self.unprocessed_statics:
self._process_statics((url_prefix, dir_path))
dir_path = os.path.normpath(dir_path)
if not os.path.exists(dir_path):
log.warning(
"Directory path \"{}\" was not found".format(dir_path)
)
continue
url_prefix = prepare_prefix(url_prefix)
self.prepared_statics[url_prefix] = dir_path
for obj in self.registered_objs:
method_names = [
attr for attr in dir(obj)
if inspect.ismethod(getattr(obj, attr))
]
for method_name in method_names:
method = obj.__getattribute__(method_name)
if not hasattr(method, "restapi"):
continue
if not method.restapi:
continue
for route in self.unprocessed_routes:
callback = route["callback"]
if not (
callback.__qualname__ == method.__qualname__ and
callback.__module__ == method.__module__ and
callback.__globals__["__file__"] == method.__globals__["__file__"]
):
continue
route["callback"] = method
self._process_route(route)
self._prepare_route(route)
break
for route in self.unprocessed_routes:
callback = route["callback"]
is_class_method = len(callback.__qualname__.split(".")) != 1
if is_class_method:
missing_self = True
if hasattr(callback, "__self__"):
if callback.__self__ is not None:
missing_self = False
if "<locals>" in callback.__qualname__:
pass
elif missing_self:
log.warning((
"Object of callback \"{}\" from \"{}\" is not"
" accessible for api. Register object or"
" register callback with already created object"
"(not with decorator in class).".format(
callback.__qualname__,
callback.__globals__["__file__"]
)
))
continue
self._prepare_route(route)
continue
self._prepare_route(route)

338
pype/services/rest_api/lib/handler.py Normal file
View file

@ -0,0 +1,338 @@
import os
import re
import json
import traceback
import http.server
from http import HTTPStatus
from urllib.parse import urlparse
from .lib import RestMethods, CallbackResult, RequestInfo
from .exceptions import AbortException
from . import RestApiFactory, Splitter
from pypeapp import Logger
log = Logger().get_logger("RestApiHandler")
class Handler(http.server.SimpleHTTPRequestHandler):
# TODO fill will necessary statuses
default_messages = {
HTTPStatus.BAD_REQUEST: "Bad request",
HTTPStatus.NOT_FOUND: "Not found"
}
statuses = {
"POST": {
"OK": 200,
"CREATED": 201
},
"PUT": {
"OK": 200,
"NO_CONTENT": 204
}
}
def do_GET(self):
return self._handle_request(RestMethods.GET)
def do_POST(self):
return self._handle_request(RestMethods.POST)
def do_PUT(self):
return self._handle_request(RestMethods.PUT)
def do_DELETE(self):
return self._handle_request(RestMethods.DELETE)
def do_PATCH(self):
return self._handle_request(RestMethods.PATCH)
def _handle_request(self, rest_method):
"""Handle request by registered callbacks and statics.
Statics are only for GET method request. `_handle_statics` is called
when path is matching registered statics prefix. Callbacks are filtered
by method and their prefixes. When path is matching `_handle_callback`
is called.
If any registered callback or statics match requested path 400 status
is responsed. And 500 when unexpected error happens.
"""
parsed_url = urlparse(self.path)
path = parsed_url.path
if rest_method is RestMethods.GET:
for prefix, dirpath in RestApiFactory.prepared_statics.items():
if not path.startswith(prefix):
continue
_path = path[len(prefix):]
return self._handle_statics(dirpath, _path)
matching_item = None
found_prefix = None
url_prefixes = RestApiFactory.prepared_routes[rest_method]
for url_prefix, items in url_prefixes.items():
if matching_item is not None:
break
if url_prefix is not None:
if not path.startswith(url_prefix):
continue
found_prefix = url_prefix
for item in items:
regex = item["regex"]
item_full_path = item["fullpath"]
if regex is None:
if path == item_full_path:
item["url_data"] = None
matching_item = item
break
else:
found = re.match(regex, path)
if found:
item["url_data"] = found.groupdict()
matching_item = item
break
if not matching_item:
if found_prefix is not None:
_path = path.replace(found_prefix, "")
if _path:
request_str = " \"{}\"".format(_path)
else:
request_str = ""
message = "Invalid path request{} for prefix \"{}\"".format(
request_str, found_prefix
)
else:
message = "Invalid path request \"{}\"".format(self.path)
log.debug(message)
self.send_error(HTTPStatus.BAD_REQUEST, message)
return
try:
log.debug("Triggering callback for path \"{}\"".format(path))
result = self._handle_callback(
matching_item, parsed_url, rest_method
)
return self._handle_callback_result(result, rest_method)
except AbortException as exc:
status_code, message = str(exc).split(Splitter)
status_code = int(status_code)
if not message:
message = self.default_messages.get(
status_code, "UnexpectedError"
)
self.send_response(status_code)
self.send_header("Content-type", "text/html")
self.send_header("Content-Length", len(message))
self.end_headers()
self.wfile.write(message.encode())
return message
except Exception as exc:
log_message = "Unexpected Exception was raised (this is bug!)"
log.error(log_message, exc_info=True)
replace_helper = 0
items = [log_message]
items += traceback.extract_tb(exc.__traceback__).format()
message = "\n".join(items)
self.send_response(HTTPStatus.INTERNAL_SERVER_ERROR)
self.send_header("Content-type", "text/html")
self.send_header("Content-Length", len(message))
self.end_headers()
self.wfile.write(message.encode())
return message
def _handle_callback_result(self, result, rest_method):
"""Send response to request based on result of callback.
:param result: Result returned by callback.
:type result: None, bool, dict, list, CallbackResult
:param rest_method: Rest api method (GET, POST, etc.).
:type rest_method: RestMethods
Response is based on result type:
- None, True - It is expected everything was OK, status 200.
- False - It is expected callback was not successful, status 400
- dict, list - Result is send under "data" key of body, status 200
- CallbackResult - object specify status and data
"""
content_type = "application/json"
status = HTTPStatus.OK
success = True
message = None
data = None
body = None
# TODO better handling of results
if isinstance(result, CallbackResult):
status = result.status_code
body_dict = {}
for key, value in result.items():
if value is not None:
body_dict[key] = value
body = json.dumps(body_dict)
elif result in [None, True]:
status = HTTPStatus.OK
success = True
message = "{} request for \"{}\" passed".format(
rest_method, self.path
)
elif result is False:
status = HTTPStatus.BAD_REQUEST
success = False
elif isinstance(result, (dict, list)):
status = HTTPStatus.OK
data = result
if status == HTTPStatus.NO_CONTENT:
self.send_response(status)
self.end_headers()
return
if not body:
body_dict = {"success": success}
if message:
body_dict["message"] = message
if not data:
data = {}
body_dict["data"] = data
body = json.dumps(body_dict)
self.send_response(status)
self.send_header("Content-type", content_type)
self.send_header("Content-Length", len(body))
self.end_headers()
self.wfile.write(body.encode())
return body
def _handle_callback(self, item, parsed_url, rest_method):
"""Prepare data from request and trigger callback.
Data are loaded from body of request if there are any.
:param item: Item stored during callback registration with all info.
:type item: dict
:param parsed_url: Url parsed with urllib (separated path, query, etc.).
:type parsed_url: ParseResult
:param rest_method: Rest api method (GET, POST, etc.).
:type rest_method: RestMethods
"""
regex_keys = item["regex_keys"]
url_data = None
if regex_keys:
url_data = {key: None for key in regex_keys}
if item["url_data"]:
for key, value in item["url_data"].items():
url_data[key] = value
in_data = None
cont_len = self.headers.get("Content-Length")
if cont_len:
content_length = int(cont_len)
in_data_str = self.rfile.read(content_length)
if in_data_str:
in_data = json.loads(in_data_str)
request_info = RequestInfo(
url_data=url_data,
request_data=in_data,
query=parsed_url.query,
fragment=parsed_url.fragment,
params=parsed_url.params,
method=rest_method,
handler=self
)
callback = item["callback"]
callback_info = item["callback_info"]
_args_len = callback_info["args_len"]
_has_args = callback_info["hasargs"]
_has_kwargs = callback_info["haskwargs"]
args = []
kwargs = {}
if _args_len == 0:
if _has_args:
args.append(request_info)
elif _has_kwargs:
kwargs["request_info"] = request_info
else:
args.append(request_info)
return callback(*args, **kwargs)
def _handle_statics(self, dirpath, path):
"""Return static file in response when file exist in registered destination."""
path = os.path.normpath(dirpath + path)
ctype = self.guess_type(path)
try:
file_obj = open(path, "rb")
except OSError:
self.send_error(HTTPStatus.NOT_FOUND, "File not found")
return None
try:
file_stat = os.fstat(file_obj.fileno())
# Use browser cache if possible
if ("If-Modified-Since" in self.headers
and "If-None-Match" not in self.headers):
# compare If-Modified-Since and time of last file modification
try:
ims = http.server.email.utils.parsedate_to_datetime(
self.headers["If-Modified-Since"])
except (TypeError, IndexError, OverflowError, ValueError):
# ignore ill-formed values
pass
else:
if ims.tzinfo is None:
# obsolete format with no timezone, cf.
# https://tools.ietf.org/html/rfc7231#section-7.1.1.1
ims = ims.replace(tzinfo=datetime.timezone.utc)
if ims.tzinfo is datetime.timezone.utc:
# compare to UTC datetime of last modification
last_modif = datetime.datetime.fromtimestamp(
file_stat.st_mtime, datetime.timezone.utc)
# remove microseconds, like in If-Modified-Since
last_modif = last_modif.replace(microsecond=0)
if last_modif <= ims:
self.send_response(HTTPStatus.NOT_MODIFIED)
self.end_headers()
file_obj.close()
return None
self.send_response(HTTPStatus.OK)
self.send_header("Content-type", ctype)
self.send_header("Content-Length", str(file_stat[6]))
self.send_header("Last-Modified",
self.date_time_string(file_stat.st_mtime))
self.end_headers()
self.wfile.write(file_obj.read())
return file_obj
except:
self.log.error("Failed to read data from file \"{}\"".format(path))
finally:
file_obj.close()

207
pype/services/rest_api/lib/lib.py Normal file
View file

@ -0,0 +1,207 @@
import os
import re
import enum
from http import HTTPStatus
from urllib.parse import urlencode, parse_qs
from pypeapp import Logger
log = Logger().get_logger("RestApiServer")
class RestMethods(enum.Enum):
GET = "GET"
POST = "POST"
PUT = "PUT"
PATCH = "PATCH"
DELETE = "DELETE"
def __eq__(self, other):
if isinstance(other, self.__class__):
return self.value == other.value
elif isinstance(other, str):
return self.value.lower() == other.lower()
return self.value == other
def __hash__(self):
return enum.Enum.__hash__(self)
@classmethod
def get(cls, name, default=None):
for meth in cls:
if meth == name:
return meth
return default
class CustomNone:
"""Created object can be used as custom None (not equal to None)"""
def __init__(self, name):
self._name = name
def __bool__(self):
return False
def __eq__(self, other):
if type(other) == type(self):
if other._name == self._name:
return True
return False
def __str__(self):
return self._name
def __repr__(self):
return self._name
class HandlerDict(dict):
def __init__(self, data=None, *args, **kwargs):
if not data:
data = {}
super().__init__(data, *args, **kwargs)
def __repr__(self):
return "<{}> {}".format(self.__class__.__name__, str(dict(self)))
class Params(HandlerDict): pass
class UrlData(HandlerDict): pass
class RequestData(HandlerDict): pass
class Query(HandlerDict):
"""Class for url query convert to dict and string"""
def __init__(self, query):
if isinstance(query, dict):
pass
else:
query = parse_qs(query)
super().__init__(query)
def get_string(self):
return urlencode(dict(self), doseq=True)
class Fragment(HandlerDict):
"""Class for url fragment convert to dict and string"""
def __init__(self, fragment):
if isinstance(fragment, dict):
_fragment = fragment
else:
_fragment = {}
for frag in fragment.split("&"):
if not frag:
continue
items = frag.split("=")
value = None
key = items[0]
if len(items) == 2:
value = items[1]
elif len(items) > 2:
value = "=".join(items[1:])
_fragment[key] = value
super().__init__(_fragment)
def get_string(self):
items = []
for parts in dict(self).items():
items.append(
"=".join([p for p in parts if p])
)
return "&".join(items)
class RequestInfo:
"""Object that can be passed to callback as argument.
Contain necessary data for handling request.
Object is created to single use and can be used similar to dict.
:param url_data: Data collected from path when path with dynamic keys is matching.
:type url_data: dict, None
:param request_data: Data of body from request.
:type request_data: dict, None
:param query: Query from url path of reques.
:type query: str, None
:param fragment: Fragment from url path of reques.
:type fragment: str, None
:param params: Parems from url path of reques.
:type params: str, None
:param method: Method of request (GET, POST, etc.)
:type method: RestMethods
:param handler: Handler handling request from http server.
:type handler: Handler
"""
def __init__(
self, url_data, request_data, query, fragment, params, method, handler
):
self.url_data = UrlData(url_data)
self.request_data = RequestData(request_data)
self.query = Query(query)
self.fragment = Fragment(fragment)
self.params = Params(params)
self.method = method
self.handler = handler
def __getitem__(self, key):
return self.__getattribute__(key)
def __hash__(self):
return {
"url_data": self.url_data,
"request_data": self. request_data,
"query": self.query,
"fragment": self.fragment,
"params": self.params,
"method": self.method,
"handler": self.handler
}
def items(self):
return dict(self).items()
class CallbackResult:
"""Can be used as return value of callback.
It is possible to specify status code, success boolean, message and data
for specify head and body of request response. `abort` should be rather used
when result is error.
:param status_code: Status code of result.
:type status_code: int
:param success: Success is key in body, may be used for handling response.
:type success: bool
:param message: Similar to success, message is key in body and may be used for handling response.
:type message: str, None
:param data: Data is also key for body in response.
:type data: dict, None
"""
_data = {}
def __init__(
self, status_code=HTTPStatus.OK, success=True, message=None, data=None,
**kwargs
):
self.status_code = status_code
self._data = {
"success": success,
"message": message,
"data": data
}
for k, v in kwargs.items():
self._data[k] = v
def __getitem__(self, key):
return self._data[key]
def __iter__(self):
for key in self._data:
yield key
def get(self, key, default=None):
return self._data.get(key, default)
def items(self):
return self._data.items()

359
pype/services/rest_api/rest_api.py
View file

@ -1,260 +1,127 @@
import os
import json
import enum
import re
import collections
import threading
from inspect import signature
import socket
import http.server
from http import HTTPStatus
import socketserver
from Qt import QtCore
from .lib import RestApiFactory, Handler
from .base_class import route, register_statics
from pypeapp import config, Logger
log = Logger().get_logger("RestApiServer")
class RestMethods(enum.Enum):
GET = "GET"
POST = "POST"
PUT = "PUT"
PATCH = "PATCH"
DELETE = "DELETE"
class RestApiServer:
"""Rest Api allows to access statics or callbacks with http requests.
def __repr__(self):
return str(self.value)
To register statics use `register_statics`.
def __eq__(self, other):
if isinstance(other, str):
return self.value == other
return self == other
To register callback use `register_callback` method or use `route` decorator.
`route` decorator should be used with not-class functions, it is possible
to use within class when inherits `RestApi` (defined in `base_class.py`)
or created object, with used decorator, is registered with `register_obj`.
def __hash__(self):
return enum.Enum.__hash__(self)
.. code-block:: python
@route("/username", url_prefix="/api", methods=["get"], strict_match=False)
def get_username():
return {"username": getpass.getuser()}
def __str__(self):
return str(self.value)
In that case response to `localhost:{port}/api/username` will be status
`200` with body including `{"data": {"username": getpass.getuser()}}`
Callback may expect one argument which will be filled with request
info. Data object has attributes: `request_data`, `query`,
`fragment`, `params`, `method`, `handler`, `url_data`.
request_data - Data from request body if there are any.
query - query from url path (?identificator=name)
fragment - fragment from url path (#reset_credentials)
params - params from url path
method - request method (GET, POST, PUT, etc.)
handler - Handler object of HttpServer Handler
url_data - dynamic url keys from registered path with their values
class Handler(http.server.SimpleHTTPRequestHandler):
Dynamic url keys may be set with path argument.
.. code-block:: python
from rest_api import route
def do_GET(self):
self.process_request(RestMethods.GET)
def do_POST(self):
"""Common code for POST.
This trigger callbacks on specific paths.
If request contain data and callback func has arg data are sent to
callback too.
Send back return values of callbacks.
"""
self.process_request(RestMethods.POST)
def process_request(self, rest_method):
"""Because processing is technically the same for now so it is used
the same way
"""
in_data = None
cont_len = self.headers.get("Content-Length")
if cont_len:
content_length = int(cont_len)
in_data_str = self.rfile.read(content_length)
if in_data_str:
in_data = json.loads(in_data_str)
registered_callbacks = self.server.registered_callbacks[rest_method]
path_items = [part.lower() for part in self.path.split("/") if part]
request_path = "/".join(path_items)
callback = registered_callbacks.get(request_path)
result = None
if callback:
log.debug(
"Triggering callbacks for path \"{}\"".format(request_path)
)
try:
params = signature(callback).parameters
if len(params) > 0 and in_data:
result = callback(in_data)
else:
result = callback()
self.send_response(HTTPStatus.OK)
if result in [None, True] or isinstance(result, (str, int)):
message = str(result)
if result in [None, True]:
message = "{} request for \"{}\" passed".format(
rest_method, self.path
)
self.handle_result(rest_method, message=message)
return
if isinstance(result, (dict, list)):
message = json.dumps(result).encode()
self.handle_result(rest_method, final_output=message)
return
except Exception:
message = "{} request for \"{}\" failed".format(
rest_method, self.path
)
log.error(message, exc_info=True)
self.send_response(HTTPStatus.BAD_REQUEST)
self.handle_result(rest_method, message=message, success=False)
return
self.handle_result(rest_method)
else:
message = (
"{} request for \"{}\" don't have registered callback"
).format(rest_method, self.path)
log.debug(message)
self.send_response(HTTPStatus.NOT_FOUND)
self.handle_result(rest_method, message=message, success=False)
def handle_result(
self, rest_method, final_output=None, message=None, success=True,
content_type="application/json"
):
self.send_header("Content-type", content_type)
if final_output:
output = final_output
else:
if not message:
output = json.dumps({
"success": False,
"message": (
"{} request for \"{}\" has unexpected result"
).format(rest_method, self.path)
}).encode()
else:
output = json.dumps({
"success": success, "message": message
}).encode()
if isinstance(output, str):
self.send_header("Content-Length", len(output))
self.end_headers()
if output:
self.wfile.write(output)
class AdditionalArgsTCPServer(socketserver.TCPServer):
def __init__(self, registered_callbacks, *args, **kwargs):
self.registered_callbacks = registered_callbacks
super(AdditionalArgsTCPServer, self).__init__(*args, **kwargs)
class RestApiServer(QtCore.QThread):
""" Listener for REST requests.
It is possible to register callbacks for url paths.
Be careful about crossreferencing to different QThreads it is not allowed.
"""
def __init__(self):
super(RestApiServer, self).__init__()
self.registered_callbacks = {
RestMethods.GET: collections.defaultdict(list),
RestMethods.POST: collections.defaultdict(list),
RestMethods.PUT: collections.defaultdict(list),
RestMethods.PATCH: collections.defaultdict(list),
RestMethods.DELETE: collections.defaultdict(list)
all_projects = {
"Proj1": {"proj_data": []},
"Proj2": {"proj_data": []},
}
@route("/projects/<project_name>", url_prefix="/api", methods=["get"], strict_match=False)
def get_projects(request_info):
project_name = request_info.url_data["project_name"]
if not project_name:
return all_projects
return all_projects.get(project_name)
This example should end with status 404 if project is not found. In that
case is best to use `abort` method.
.. code-block:: python
from rest_api import abort
@route("/projects/<project_name>", url_prefix="/api", methods=["get"], strict_match=False)
def get_projects(request_info):
project_name = request_info.url_data["project_name"]
if not project_name:
return all_projects
project = all_projects.get(project_name)
if not project:
abort(404, "Project \"{}\".format(project_name) was not found")
return project
`strict_match` allows to handle not only specific entity but all entity types.
E.g. "/projects/<project_name>" with set `strict_match` to False will handle also
"/projects" or "/projects/" path. It is necessary to set `strict_match` to
True when should handle only single entity.
Callback may return many types. For more information read docstring of
`_handle_callback_result` defined in handler.
"""
def __init__(self):
self.qaction = None
self.failed_icon = None
self._is_running = False
try:
self.presets = config.get_presets().get(
"services", {}).get(
"rest_api", {}
)
self.presets = config.get_presets()["services"]["rest_api"]
except Exception:
self.presets = {"default_port": 8011, "exclude_ports": []}
log.debug((
"There are not set presets for RestApiModule."
" Using defaults \"{}\""
).format(str(self.presets)))
self.port = self.find_port()
port = self.find_port()
self.rest_api_thread = RestApiThread(self, port)
statics_dir = os.path.sep.join([os.environ["PYPE_MODULE_ROOT"], "res"])
self.register_statics("/res", statics_dir)
os.environ["PYPE_STATICS_SERVER"] = "{}/res".format(
os.environ["PYPE_REST_API_URL"]
)
def set_qaction(self, qaction, failed_icon):
self.qaction = qaction
self.failed_icon = failed_icon
def register_callback(self, path, callback, rest_method=RestMethods.POST):
if isinstance(path, (list, set)):
path = "/".join([part.lower() for part in path])
elif isinstance(path, str):
path = "/".join(
[part.lower() for part in str(path).split("/") if part]
)
if isinstance(rest_method, str):
rest_method = str(rest_method).upper()
if path in self.registered_callbacks[rest_method]:
log.error(
"Path \"{}\" has already registered callback.".format(path)
)
return False
log.debug(
"Registering callback for path \"{}\"".format(path)
def register_callback(
self, path, callback, url_prefix="", methods=[], strict_match=False
):
RestApiFactory.register_route(
path, callback, url_prefix, methods, strict_match
)
self.registered_callbacks[rest_method][path] = callback
return True
def register_statics(self, url_prefix, dir_path):
register_statics(url_prefix, dir_path)
def tray_start(self):
self.start()
@property
def is_running(self):
return self._is_running
def stop(self):
self._is_running = False
def run(self):
self._is_running = True
if not self.registered_callbacks:
log.info("Any registered callbacks for Rest Api server.")
return
try:
log.debug(
"Running Rest Api server on URL:"
" \"http://localhost:{}\"".format(self.port)
)
with AdditionalArgsTCPServer(
self.registered_callbacks,
("", self.port),
Handler
) as httpd:
while self._is_running:
httpd.handle_request()
except Exception:
log.warning(
"Rest Api Server service has failed", exc_info=True
)
self._is_running = False
if self.qaction and self.failed_icon:
self.qaction.setIcon(self.failed_icon)
def register_obj(self, obj):
RestApiFactory.register_obj(obj)
def find_port(self):
start_port = self.presets["default_port"]
@ -276,3 +143,53 @@ class RestApiServer(QtCore.QThread):
found_port
)
return found_port
def tray_start(self):
RestApiFactory.prepare_registered()
if not RestApiFactory.has_handlers():
log.debug("There are not registered any handlers for RestApi")
return
self.rest_api_thread.start()
@property
def is_running(self):
return self.rest_api_thread.is_running
def stop(self):
self.rest_api_thread.is_running = False
def thread_stopped(self):
self._is_running = False
class RestApiThread(QtCore.QThread):
""" Listener for REST requests.
It is possible to register callbacks for url paths.
Be careful about crossreferencing to different Threads it is not allowed.
"""
def __init__(self, module, port):
super(RestApiThread, self).__init__()
self.is_running = False
self.module = module
self.port = port
def run(self):
self.is_running = True
try:
log.debug(
"Running Rest Api server on URL:"
" \"http://localhost:{}\"".format(self.port)
)
with socketserver.TCPServer(("", self.port), Handler) as httpd:
while self.is_running:
httpd.handle_request()
except Exception:
log.warning(
"Rest Api Server service has failed", exc_info=True
)
self.is_running = False
self.module.thread_stopped()

5
pype/services/statics_server/__init__.py
View file

@ -1,5 +0,0 @@
from .statics_server import StaticsServer
def tray_init(tray_widget, main_widget):
return StaticsServer()

202
pype/services/statics_server/statics_server.py
View file

@ -1,202 +0,0 @@
import os
import sys
import datetime
import socket
import http.server
from http import HTTPStatus
import urllib
import posixpath
import socketserver
from Qt import QtCore
from pypeapp import config, Logger
DIRECTORY = os.path.sep.join([os.environ['PYPE_MODULE_ROOT'], 'res'])
class Handler(http.server.SimpleHTTPRequestHandler):
def __init__(self, *args, **kwargs):
py_version = sys.version.split('.')
# If python version is 3.7 or higher
if int(py_version[0]) >= 3 and int(py_version[1]) >= 7:
super().__init__(*args, directory=DIRECTORY, **kwargs)
else:
self.directory = DIRECTORY
super().__init__(*args, **kwargs)
def send_head(self):
"""Common code for GET and HEAD commands.
This sends the response code and MIME headers.
Return value is either a file object (which has to be copied
to the outputfile by the caller unless the command was HEAD,
and must be closed by the caller under all circumstances), or
None, in which case the caller has nothing further to do.
"""
path = self.translate_path(self.path)
f = None
if os.path.isdir(path):
parts = urllib.parse.urlsplit(self.path)
if not parts.path.endswith('/'):
# redirect browser - doing basically what apache does
self.send_response(HTTPStatus.MOVED_PERMANENTLY)
new_parts = (parts[0], parts[1], parts[2] + '/',
parts[3], parts[4])
new_url = urllib.parse.urlunsplit(new_parts)
self.send_header("Location", new_url)
self.end_headers()
return None
for index in "index.html", "index.htm":
index = os.path.join(path, index)
if os.path.exists(index):
path = index
break
else:
return self.list_directory(path)
ctype = self.guess_type(path)
try:
f = open(path, 'rb')
except OSError:
self.send_error(HTTPStatus.NOT_FOUND, "File not found")
return None
try:
fs = os.fstat(f.fileno())
# Use browser cache if possible
if ("If-Modified-Since" in self.headers
and "If-None-Match" not in self.headers):
# compare If-Modified-Since and time of last file modification
try:
ims = http.server.email.utils.parsedate_to_datetime(
self.headers["If-Modified-Since"])
except (TypeError, IndexError, OverflowError, ValueError):
# ignore ill-formed values
pass
else:
if ims.tzinfo is None:
# obsolete format with no timezone, cf.
# https://tools.ietf.org/html/rfc7231#section-7.1.1.1
ims = ims.replace(tzinfo=datetime.timezone.utc)
if ims.tzinfo is datetime.timezone.utc:
# compare to UTC datetime of last modification
last_modif = datetime.datetime.fromtimestamp(
fs.st_mtime, datetime.timezone.utc)
# remove microseconds, like in If-Modified-Since
last_modif = last_modif.replace(microsecond=0)
if last_modif <= ims:
self.send_response(HTTPStatus.NOT_MODIFIED)
self.end_headers()
f.close()
return None
self.send_response(HTTPStatus.OK)
self.send_header("Content-type", ctype)
self.send_header("Content-Length", str(fs[6]))
self.send_header("Last-Modified",
self.date_time_string(fs.st_mtime))
self.end_headers()
return f
except:
f.close()
raise
def translate_path(self, path):
"""Translate a /-separated PATH to the local filename syntax.
Components that mean special things to the local file system
(e.g. drive or directory names) are ignored. (XXX They should
probably be diagnosed.)
"""
# abandon query parameters
path = path.split('?',1)[0]
path = path.split('#',1)[0]
# Don't forget explicit trailing slash when normalizing. Issue17324
trailing_slash = path.rstrip().endswith('/')
try:
path = urllib.parse.unquote(path, errors='surrogatepass')
except UnicodeDecodeError:
path = urllib.parse.unquote(path)
path = posixpath.normpath(path)
words = path.split('/')
words = filter(None, words)
path = self.directory
for word in words:
if os.path.dirname(word) or word in (os.curdir, os.pardir):
# Ignore components that are not a simple file/directory name
continue
path = os.path.join(path, word)
if trailing_slash:
path += '/'
return path
class StaticsServer(QtCore.QThread):
""" Measure user's idle time in seconds.
Idle time resets on keyboard/mouse input.
Is able to emit signals at specific time idle.
"""
def __init__(self):
super(StaticsServer, self).__init__()
self.qaction = None
self.failed_icon = None
self._is_running = False
self.log = Logger().get_logger(self.__class__.__name__)
try:
self.presets = config.get_presets().get(
'services', {}).get('statics_server')
except Exception:
self.presets = {'default_port': 8010, 'exclude_ports': []}
self.port = self.find_port()
def set_qaction(self, qaction, failed_icon):
self.qaction = qaction
self.failed_icon = failed_icon
def tray_start(self):
self.start()
@property
def is_running(self):
return self._is_running
def stop(self):
self._is_running = False
def run(self):
self._is_running = True
try:
with socketserver.TCPServer(("", self.port), Handler) as httpd:
while self._is_running:
httpd.handle_request()
except Exception:
self.log.warning(
'Statics Server service has failed', exc_info=True
)
self._is_running = False
if self.qaction and self.failed_icon:
self.qaction.setIcon(self.failed_icon)
def find_port(self):
start_port = self.presets['default_port']
exclude_ports = self.presets['exclude_ports']
found_port = None
# port check takes time so it's lowered to 100 ports
for port in range(start_port, start_port+100):
if port in exclude_ports:
continue
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
result = sock.connect_ex(('localhost', port))
if result != 0:
found_port = port
if found_port is not None:
break
if found_port is None:
return None
os.environ['PYPE_STATICS_SERVER'] = 'http://localhost:{}'.format(found_port)
return found_port