Frame13/ayon-core
1
0
Fork 1
mirror of https://github.com/ynput/ayon-core.git synced 2025-12-24 21:04:40 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

[Automated] Merged develop into main

Browse source
This commit is contained in:
pypebot 2022-07-27 05:56:07 +02:00 • committed by GitHub
commit 7aaa9c1c14
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
42 changed files with 778 additions and 456 deletions
Download patch file Download diff file Expand all files Collapse all files

10
openpype/client/__init__.py
View file

@ -1,3 +1,7 @@
from .mongo import (
OpenPypeMongoConnection,
)
from .entities import (
get_projects,
get_project,
@ -25,6 +29,8 @@ from .entities import (
get_last_version_by_subset_name,
get_output_link_versions,
version_is_latest,
get_representation_by_id,
get_representation_by_name,
get_representations,
@ -40,6 +46,8 @@ from .entities import (
)
__all__ = (
"OpenPypeMongoConnection",
"get_projects",
"get_project",
"get_whole_project",
@ -66,6 +74,8 @@ __all__ = (
"get_last_version_by_subset_name",
"get_output_link_versions",
"version_is_latest",
"get_representation_by_id",
"get_representation_by_name",
"get_representations",

98
openpype/client/entities.py
View file

@ -12,7 +12,7 @@ import collections
import six
from bson.objectid import ObjectId
from openpype.lib.mongo import OpenPypeMongoConnection
from .mongo import OpenPypeMongoConnection
def _get_project_database():
@ -20,7 +20,21 @@ def _get_project_database():
return OpenPypeMongoConnection.get_mongo_client()[db_name]
def _get_project_connection(project_name):
def get_project_connection(project_name):
"""Direct access to mongo collection.
We're trying to avoid using direct access to mongo. This should be used
only for Create, Update and Remove operations until there are implemented
api calls for that.
Args:
project_name(str): Project name for which collection should be
returned.
Returns:
pymongo.Collection: Collection realated to passed project.
"""
if not project_name:
raise ValueError("Invalid project name {}".format(str(project_name)))
return _get_project_database()[project_name]
@ -93,7 +107,7 @@ def get_project(project_name, active=True, inactive=False, fields=None):
{"data.active": False},
]
conn = _get_project_connection(project_name)
conn = get_project_connection(project_name)
return conn.find_one(query_filter, _prepare_fields(fields))
@ -108,7 +122,7 @@ def get_whole_project(project_name):
project collection.
"""
conn = _get_project_connection(project_name)
conn = get_project_connection(project_name)
return conn.find({})
@ -131,7 +145,7 @@ def get_asset_by_id(project_name, asset_id, fields=None):
return None
query_filter = {"type": "asset", "_id": asset_id}
conn = _get_project_connection(project_name)
conn = get_project_connection(project_name)
return conn.find_one(query_filter, _prepare_fields(fields))
@ -153,7 +167,7 @@ def get_asset_by_name(project_name, asset_name, fields=None):
return None
query_filter = {"type": "asset", "name": asset_name}
conn = _get_project_connection(project_name)
conn = get_project_connection(project_name)
return conn.find_one(query_filter, _prepare_fields(fields))
@ -223,7 +237,7 @@ def _get_assets(
return []
query_filter["data.visualParent"] = {"$in": parent_ids}
conn = _get_project_connection(project_name)
conn = get_project_connection(project_name)
return conn.find(query_filter, _prepare_fields(fields))
@ -323,7 +337,7 @@ def get_asset_ids_with_subsets(project_name, asset_ids=None):
return []
subset_query["parent"] = {"$in": asset_ids}
conn = _get_project_connection(project_name)
conn = get_project_connection(project_name)
result = conn.aggregate([
{
"$match": subset_query
@ -363,7 +377,7 @@ def get_subset_by_id(project_name, subset_id, fields=None):
return None
query_filters = {"type": "subset", "_id": subset_id}
conn = _get_project_connection(project_name)
conn = get_project_connection(project_name)
return conn.find_one(query_filters, _prepare_fields(fields))
@ -394,7 +408,7 @@ def get_subset_by_name(project_name, subset_name, asset_id, fields=None):
"name": subset_name,
"parent": asset_id
}
conn = _get_project_connection(project_name)
conn = get_project_connection(project_name)
return conn.find_one(query_filters, _prepare_fields(fields))
@ -467,7 +481,7 @@ def get_subsets(
return []
query_filter["$or"] = or_query
conn = _get_project_connection(project_name)
conn = get_project_connection(project_name)
return conn.find(query_filter, _prepare_fields(fields))
@ -491,7 +505,7 @@ def get_subset_families(project_name, subset_ids=None):
return set()
subset_filter["_id"] = {"$in": list(subset_ids)}
conn = _get_project_connection(project_name)
conn = get_project_connection(project_name)
result = list(conn.aggregate([
{"$match": subset_filter},
{"$project": {
@ -529,7 +543,7 @@ def get_version_by_id(project_name, version_id, fields=None):
"type": {"$in": ["version", "hero_version"]},
"_id": version_id
}
conn = _get_project_connection(project_name)
conn = get_project_connection(project_name)
return conn.find_one(query_filter, _prepare_fields(fields))
@ -552,7 +566,7 @@ def get_version_by_name(project_name, version, subset_id, fields=None):
if not subset_id:
return None
conn = _get_project_connection(project_name)
conn = get_project_connection(project_name)
query_filter = {
"type": "version",
"parent": subset_id,
@ -561,6 +575,42 @@ def get_version_by_name(project_name, version, subset_id, fields=None):
return conn.find_one(query_filter, _prepare_fields(fields))
def version_is_latest(project_name, version_id):
"""Is version the latest from it's subset.
Note:
Hero versions are considered as latest.
Todo:
Maybe raise exception when version was not found?
Args:
project_name (str):Name of project where to look for queried entities.
version_id (Union[str, ObjectId]): Version id which is checked.
Returns:
bool: True if is latest version from subset else False.
"""
version_id = _convert_id(version_id)
if not version_id:
return False
version_doc = get_version_by_id(
project_name, version_id, fields=["_id", "type", "parent"]
)
# What to do when version is not found?
if not version_doc:
return False
if version_doc["type"] == "hero_version":
return True
last_version = get_last_version_by_subset_id(
project_name, version_doc["parent"], fields=["_id"]
)
return last_version["_id"] == version_id
def _get_versions(
project_name,
subset_ids=None,
@ -606,7 +656,7 @@ def _get_versions(
else:
query_filter["name"] = {"$in": versions}
conn = _get_project_connection(project_name)
conn = get_project_connection(project_name)
return conn.find(query_filter, _prepare_fields(fields))
@ -765,7 +815,7 @@ def get_output_link_versions(project_name, version_id, fields=None):
if not version_id:
return []
conn = _get_project_connection(project_name)
conn = get_project_connection(project_name)
# Does make sense to look for hero versions?
query_filter = {
"type": "version",
@ -830,7 +880,7 @@ def get_last_versions(project_name, subset_ids, fields=None):
{"$group": group_item}
]
conn = _get_project_connection(project_name)
conn = get_project_connection(project_name)
aggregate_result = conn.aggregate(aggregation_pipeline)
if limit_query:
output = {}
@ -948,7 +998,7 @@ def get_representation_by_id(project_name, representation_id, fields=None):
if representation_id is not None:
query_filter["_id"] = _convert_id(representation_id)
conn = _get_project_connection(project_name)
conn = get_project_connection(project_name)
return conn.find_one(query_filter, _prepare_fields(fields))
@ -981,7 +1031,7 @@ def get_representation_by_name(
"parent": version_id
}
conn = _get_project_connection(project_name)
conn = get_project_connection(project_name)
return conn.find_one(query_filter, _prepare_fields(fields))
@ -1044,7 +1094,7 @@ def _get_representations(
return []
query_filter["$or"] = or_query
conn = _get_project_connection(project_name)
conn = get_project_connection(project_name)
return conn.find(query_filter, _prepare_fields(fields))
@ -1255,7 +1305,7 @@ def get_thumbnail_id_from_source(project_name, src_type, src_id):
query_filter = {"_id": _convert_id(src_id)}
conn = _get_project_connection(project_name)
conn = get_project_connection(project_name)
src_doc = conn.find_one(query_filter, {"data.thumbnail_id"})
if src_doc:
return src_doc.get("data", {}).get("thumbnail_id")
@ -1288,7 +1338,7 @@ def get_thumbnails(project_name, thumbnail_ids, fields=None):
"type": "thumbnail",
"_id": {"$in": thumbnail_ids}
}
conn = _get_project_connection(project_name)
conn = get_project_connection(project_name)
return conn.find(query_filter, _prepare_fields(fields))
@ -1309,7 +1359,7 @@ def get_thumbnail(project_name, thumbnail_id, fields=None):
if not thumbnail_id:
return None
query_filter = {"type": "thumbnail", "_id": _convert_id(thumbnail_id)}
conn = _get_project_connection(project_name)
conn = get_project_connection(project_name)
return conn.find_one(query_filter, _prepare_fields(fields))
@ -1340,7 +1390,7 @@ def get_workfile_info(
"task_name": task_name,
"filename": filename
}
conn = _get_project_connection(project_name)
conn = get_project_connection(project_name)
return conn.find_one(query_filter, _prepare_fields(fields))

210
openpype/client/mongo.py Normal file
View file

@ -0,0 +1,210 @@
import os
import sys
import time
import logging
import pymongo
import certifi
if sys.version_info[0] == 2:
from urlparse import urlparse, parse_qs
else:
from urllib.parse import urlparse, parse_qs
class MongoEnvNotSet(Exception):
pass
def _decompose_url(url):
"""Decompose mongo url to basic components.
Used for creation of MongoHandler which expect mongo url components as
separated kwargs. Components are at the end not used as we're setting
connection directly this is just a dumb components for MongoHandler
validation pass.
"""
# Use first url from passed url
# - this is because it is possible to pass multiple urls for multiple
# replica sets which would crash on urlparse otherwise
# - please don't use comma in username of password
url = url.split(",")[0]
components = {
"scheme": None,
"host": None,
"port": None,
"username": None,
"password": None,
"auth_db": None
}
result = urlparse(url)
if result.scheme is None:
_url = "mongodb://{}".format(url)
result = urlparse(_url)
components["scheme"] = result.scheme
components["host"] = result.hostname
try:
components["port"] = result.port
except ValueError:
raise RuntimeError("invalid port specified")
components["username"] = result.username
components["password"] = result.password
try:
components["auth_db"] = parse_qs(result.query)['authSource'][0]
except KeyError:
# no auth db provided, mongo will use the one we are connecting to
pass
return components
def get_default_components():
mongo_url = os.environ.get("OPENPYPE_MONGO")
if mongo_url is None:
raise MongoEnvNotSet(
"URL for Mongo logging connection is not set."
)
return _decompose_url(mongo_url)
def should_add_certificate_path_to_mongo_url(mongo_url):
"""Check if should add ca certificate to mongo url.
Since 30.9.2021 cloud mongo requires newer certificates that are not
available on most of workstation. This adds path to certifi certificate
which is valid for it. To add the certificate path url must have scheme
'mongodb+srv' or has 'ssl=true' or 'tls=true' in url query.
"""
parsed = urlparse(mongo_url)
query = parse_qs(parsed.query)
lowered_query_keys = set(key.lower() for key in query.keys())
add_certificate = False
# Check if url 'ssl' or 'tls' are set to 'true'
for key in ("ssl", "tls"):
if key in query and "true" in query["ssl"]:
add_certificate = True
break
# Check if url contains 'mongodb+srv'
if not add_certificate and parsed.scheme == "mongodb+srv":
add_certificate = True
# Check if url does already contain certificate path
if add_certificate and "tlscafile" in lowered_query_keys:
add_certificate = False
return add_certificate
def validate_mongo_connection(mongo_uri):
"""Check if provided mongodb URL is valid.
Args:
mongo_uri (str): URL to validate.
Raises:
ValueError: When port in mongo uri is not valid.
pymongo.errors.InvalidURI: If passed mongo is invalid.
pymongo.errors.ServerSelectionTimeoutError: If connection timeout
passed so probably couldn't connect to mongo server.
"""
client = OpenPypeMongoConnection.create_connection(
mongo_uri, retry_attempts=1
)
client.close()
class OpenPypeMongoConnection:
"""Singleton MongoDB connection.
Keeps MongoDB connections by url.
"""
mongo_clients = {}
log = logging.getLogger("OpenPypeMongoConnection")
@staticmethod
def get_default_mongo_url():
return os.environ["OPENPYPE_MONGO"]
@classmethod
def get_mongo_client(cls, mongo_url=None):
if mongo_url is None:
mongo_url = cls.get_default_mongo_url()
connection = cls.mongo_clients.get(mongo_url)
if connection:
# Naive validation of existing connection
try:
connection.server_info()
with connection.start_session():
pass
except Exception:
connection = None
if not connection:
cls.log.debug("Creating mongo connection to {}".format(mongo_url))
connection = cls.create_connection(mongo_url)
cls.mongo_clients[mongo_url] = connection
return connection
@classmethod
def create_connection(cls, mongo_url, timeout=None, retry_attempts=None):
parsed = urlparse(mongo_url)
# Force validation of scheme
if parsed.scheme not in ["mongodb", "mongodb+srv"]:
raise pymongo.errors.InvalidURI((
"Invalid URI scheme:"
" URI must begin with 'mongodb://' or 'mongodb+srv://'"
))
if timeout is None:
timeout = int(os.environ.get("AVALON_TIMEOUT") or 1000)
kwargs = {
"serverSelectionTimeoutMS": timeout
}
if should_add_certificate_path_to_mongo_url(mongo_url):
kwargs["ssl_ca_certs"] = certifi.where()
mongo_client = pymongo.MongoClient(mongo_url, **kwargs)
if retry_attempts is None:
retry_attempts = 3
elif not retry_attempts:
retry_attempts = 1
last_exc = None
valid = False
t1 = time.time()
for attempt in range(1, retry_attempts + 1):
try:
mongo_client.server_info()
with mongo_client.start_session():
pass
valid = True
break
except Exception as exc:
last_exc = exc
if attempt < retry_attempts:
cls.log.warning(
"Attempt {} failed. Retrying... ".format(attempt)
)
time.sleep(1)
if not valid:
raise last_exc
cls.log.info("Connected to {}, delay {:.3f}s".format(
mongo_url, time.time() - t1
))
return mongo_client

4
openpype/hosts/aftereffects/api/pipeline.py
View file

@ -1,5 +1,4 @@
import os
import sys
from Qt import QtWidgets
@ -15,6 +14,7 @@ from openpype.pipeline import (
AVALON_CONTAINER_ID,
legacy_io,
)
from openpype.pipeline.load import any_outdated_containers
import openpype.hosts.aftereffects
from openpype.lib import register_event_callback
@ -136,7 +136,7 @@ def ls():
def check_inventory():
"""Checks loaded containers if they are of highest version"""
if not lib.any_outdated():
if not any_outdated_containers():
return
# Warn about outdated containers.

15
openpype/hosts/harmony/api/pipeline.py
View file

@ -4,17 +4,15 @@ import logging
import pyblish.api
from openpype import lib
from openpype.client import get_representation_by_id
from openpype.lib import register_event_callback
from openpype.pipeline import (
legacy_io,
register_loader_plugin_path,
register_creator_plugin_path,
deregister_loader_plugin_path,
deregister_creator_plugin_path,
AVALON_CONTAINER_ID,
)
from openpype.pipeline.load import get_outdated_containers
from openpype.pipeline.context_tools import get_current_project_asset
import openpype.hosts.harmony
import openpype.hosts.harmony.api as harmony
@ -108,16 +106,7 @@ def check_inventory():
in Harmony.
"""
project_name = legacy_io.active_project()
outdated_containers = []
for container in ls():
representation_id = container['representation']
representation_doc = get_representation_by_id(
project_name, representation_id, fields=["parent"]
)
if representation_doc and not lib.is_latest(representation_doc):
outdated_containers.append(container)
outdated_containers = get_outdated_containers()
if not outdated_containers:
return

9
openpype/hosts/harmony/plugins/load/load_background.py
View file

@ -5,8 +5,8 @@ from openpype.pipeline import (
load,
get_representation_path,
)
from openpype.pipeline.context_tools import is_representation_from_latest
import openpype.hosts.harmony.api as harmony
import openpype.lib
copy_files = """function copyFile(srcFilename, dstFilename)
@ -280,9 +280,7 @@ class BackgroundLoader(load.LoaderPlugin):
)
def update(self, container, representation):
path = get_representation_path(representation)
with open(path) as json_file:
data = json.load(json_file)
@ -300,10 +298,9 @@ class BackgroundLoader(load.LoaderPlugin):
bg_folder = os.path.dirname(path)
path = get_representation_path(representation)
print(container)
is_latest = is_representation_from_latest(representation)
for layer in sorted(layers):
file_to_import = [
os.path.join(bg_folder, layer).replace("\\", "/")
@ -347,7 +344,7 @@ class BackgroundLoader(load.LoaderPlugin):
}
%s
""" % (sig, sig)
if openpype.lib.is_latest(representation):
if is_latest:
harmony.send({"function": func, "args": [node, "green"]})
else:
harmony.send({"function": func, "args": [node, "red"]})

4
openpype/hosts/harmony/plugins/load/load_imagesequence.py
View file

@ -10,8 +10,8 @@ from openpype.pipeline import (
load,
get_representation_path,
)
from openpype.pipeline.context_tools import is_representation_from_latest
import openpype.hosts.harmony.api as harmony
import openpype.lib
class ImageSequenceLoader(load.LoaderPlugin):
@ -109,7 +109,7 @@ class ImageSequenceLoader(load.LoaderPlugin):
)
# Colour node.
if openpype.lib.is_latest(representation):
if is_representation_from_latest(representation):
harmony.send(
{
"function": "PypeHarmony.setColor",

4
openpype/hosts/harmony/plugins/load/load_template.py
View file

@ -10,8 +10,8 @@ from openpype.pipeline import (
load,
get_representation_path,
)
from openpype.pipeline.context_tools import is_representation_from_latest
import openpype.hosts.harmony.api as harmony
import openpype.lib
class TemplateLoader(load.LoaderPlugin):
@ -83,7 +83,7 @@ class TemplateLoader(load.LoaderPlugin):
self_name = self.__class__.__name__
update_and_replace = False
if openpype.lib.is_latest(representation):
if is_representation_from_latest(representation):
self._set_green(node)
else:
self._set_red(node)

4
openpype/hosts/houdini/api/pipeline.py
View file

@ -12,13 +12,13 @@ from openpype.pipeline import (
register_loader_plugin_path,
AVALON_CONTAINER_ID,
)
from openpype.pipeline.load import any_outdated_containers
import openpype.hosts.houdini
from openpype.hosts.houdini.api import lib
from openpype.lib import (
register_event_callback,
emit_event,
any_outdated,
)
from .lib import get_asset_fps
@ -245,7 +245,7 @@ def on_open():
# ensure it is using correct FPS for the asset
lib.validate_fps()
if any_outdated():
if any_outdated_containers():
from openpype.widgets import popup
log.warning("Scene has outdated content.")

4
openpype/hosts/maya/api/pipeline.py
View file

@ -13,7 +13,6 @@ from openpype.host import HostBase, IWorkfileHost, ILoadHost
import openpype.hosts.maya
from openpype.tools.utils import host_tools
from openpype.lib import (
any_outdated,
register_event_callback,
emit_event
)
@ -28,6 +27,7 @@ from openpype.pipeline import (
deregister_creator_plugin_path,
AVALON_CONTAINER_ID,
)
from openpype.pipeline.load import any_outdated_containers
from openpype.hosts.maya.lib import copy_workspace_mel
from . import menu, lib
from .workio import (
@ -470,7 +470,7 @@ def on_open():
lib.validate_fps()
lib.fix_incompatible_containers()
if any_outdated():
if any_outdated_containers():
log.warning("Scene has outdated content.")
# Find maya main window

2
openpype/hosts/maya/api/shader_definition_editor.py
View file

@ -6,7 +6,7 @@ Shader names are stored as simple text file over GridFS in mongodb.
"""
import os
from Qt import QtWidgets, QtCore, QtGui
from openpype.lib.mongo import OpenPypeMongoConnection
from openpype.client.mongo import OpenPypeMongoConnection
from openpype import resources
import gridfs

2
openpype/hosts/maya/plugins/publish/extract_playblast.py
View file

@ -128,8 +128,10 @@ class ExtractPlayblast(openpype.api.Extractor):
# Update preset with current panel setting
# if override_viewport_options is turned off
if not override_viewport_options:
panel = cmds.getPanel(with_focus=True)
panel_preset = capture.parse_active_view()
preset.update(panel_preset)
cmds.setFocus(panel)
path = capture.capture(**preset)

10
openpype/hosts/maya/plugins/publish/extract_thumbnail.py
View file

@ -100,6 +100,13 @@ class ExtractThumbnail(openpype.api.Extractor):
# camera.
if preset.pop("isolate_view", False) and instance.data.get("isolate"):
preset["isolate"] = instance.data["setMembers"]
# Show or Hide Image Plane
image_plane = instance.data.get("imagePlane", True)
if "viewport_options" in preset:
preset["viewport_options"]["imagePlane"] = image_plane
else:
preset["viewport_options"] = {"imagePlane": image_plane}
with lib.maintained_time():
# Force viewer to False in call to capture because we have our own
@ -110,14 +117,17 @@ class ExtractThumbnail(openpype.api.Extractor):
# Update preset with current panel setting
# if override_viewport_options is turned off
if not override_viewport_options:
panel = cmds.getPanel(with_focus=True)
panel_preset = capture.parse_active_view()
preset.update(panel_preset)
cmds.setFocus(panel)
path = capture.capture(**preset)
playblast = self._fix_playblast_output_path(path)
_, thumbnail = os.path.split(playblast)
self.log.info("file list {}".format(thumbnail))
if "representations" not in instance.data:

2
openpype/hosts/maya/plugins/publish/validate_model_name.py
View file

@ -10,7 +10,7 @@ from openpype.pipeline import legacy_io
import openpype.hosts.maya.api.action
from openpype.hosts.maya.api.shader_definition_editor import (
DEFINITION_FILENAME)
from openpype.lib.mongo import OpenPypeMongoConnection
from openpype.client.mongo import OpenPypeMongoConnection
import gridfs

1
openpype/hosts/nuke/plugins/publish/precollect_instances.py
View file

@ -94,6 +94,7 @@ class PreCollectNukeInstances(pyblish.api.ContextPlugin):
# Farm rendering
self.log.info("flagged for farm render")
instance.data["transfer"] = False
instance.data["farm"] = True
families.append("{}.farm".format(family))
family = families_ak.lower()

5
openpype/hosts/photoshop/api/pipeline.py
View file

@ -1,6 +1,5 @@
import os
from Qt import QtWidgets
from bson.objectid import ObjectId
import pyblish.api
@ -13,8 +12,8 @@ from openpype.pipeline import (
deregister_loader_plugin_path,
deregister_creator_plugin_path,
AVALON_CONTAINER_ID,
registered_host,
)
from openpype.pipeline.load import any_outdated_containers
import openpype.hosts.photoshop
from . import lib
@ -30,7 +29,7 @@ INVENTORY_PATH = os.path.join(PLUGINS_DIR, "inventory")
def check_inventory():
if not lib.any_outdated():
if not any_outdated_containers():
return
# Warn about outdated containers.

2
openpype/hosts/webpublisher/webserver_service/webpublish_routes.py
View file

@ -10,9 +10,9 @@ from aiohttp.web_response import Response
from openpype.client import (
get_projects,
get_assets,
OpenPypeMongoConnection,
)
from openpype.lib import (
OpenPypeMongoConnection,
PypeLogger,
)
from openpype.lib.remote_publish import (

3
openpype/hosts/webpublisher/webserver_service/webserver_cli.py
View file

@ -6,6 +6,7 @@ import requests
import json
import subprocess
from openpype.client import OpenPypeMongoConnection
from openpype.lib import PypeLogger
from .webpublish_routes import (
@ -121,8 +122,6 @@ def run_webserver(*args, **kwargs):
def reprocess_failed(upload_dir, webserver_url):
# log.info("check_reprocesable_records")
from openpype.lib import OpenPypeMongoConnection
mongo_client = OpenPypeMongoConnection.get_mongo_client()
database_name = os.environ["OPENPYPE_DATABASE_NAME"]
dbcon = mongo_client[database_name]["webpublishes"]

82
openpype/lib/avalon_context.py
View file

@ -15,11 +15,10 @@ from openpype.client import (
get_asset_by_name,
get_subset_by_name,
get_subsets,
get_version_by_id,
get_last_versions,
get_last_version_by_subset_id,
get_last_version_by_subset_name,
get_representations,
get_representation_by_id,
get_workfile_info,
)
from openpype.settings import (
@ -180,7 +179,7 @@ def with_pipeline_io(func):
return wrapped
@with_pipeline_io
@deprecated("openpype.pipeline.context_tools.is_representation_from_latest")
def is_latest(representation):
"""Return whether the representation is from latest version
@ -191,49 +190,18 @@ def is_latest(representation):
bool: Whether the representation is of latest version.
"""
project_name = legacy_io.active_project()
version = get_version_by_id(
project_name,
representation["parent"],
fields=["_id", "type", "parent"]
)
if version["type"] == "hero_version":
return True
from openpype.pipeline.context_tools import is_representation_from_latest
# Get highest version under the parent
last_version = get_last_version_by_subset_id(
project_name, version["parent"], fields=["_id"]
)
return version["_id"] == last_version["_id"]
return is_representation_from_latest(representation)
@with_pipeline_io
@deprecated("openpype.pipeline.load.any_outdated_containers")
def any_outdated():
"""Return whether the current scene has any outdated content"""
from openpype.pipeline import registered_host
project_name = legacy_io.active_project()
checked = set()
host = registered_host()
for container in host.ls():
representation = container['representation']
if representation in checked:
continue
from openpype.pipeline.load import any_outdated_containers
representation_doc = get_representation_by_id(
project_name, representation, fields=["parent"]
)
if representation_doc and not is_latest(representation_doc):
return True
elif not representation_doc:
log.debug("Container '{objectName}' has an invalid "
"representation, it is missing in the "
"database".format(**container))
checked.add(representation)
return False
return any_outdated_containers()
@deprecated("openpype.pipeline.context_tools.get_current_project_asset")
@ -313,7 +281,7 @@ def get_linked_assets(asset_doc):
return list(get_assets(project_name, link_ids))
@with_pipeline_io
@deprecated("openpype.client.get_last_version_by_subset_name")
def get_latest_version(asset_name, subset_name, dbcon=None, project_name=None):
"""Retrieve latest version from `asset_name`, and `subset_name`.
@ -334,6 +302,8 @@ def get_latest_version(asset_name, subset_name, dbcon=None, project_name=None):
if not project_name:
if not dbcon:
from openpype.pipeline import legacy_io
log.debug("Using `legacy_io` for query.")
dbcon = legacy_io
# Make sure is installed
@ -341,37 +311,9 @@ def get_latest_version(asset_name, subset_name, dbcon=None, project_name=None):
project_name = dbcon.active_project()
log.debug((
"Getting latest version for Project: \"{}\" Asset: \"{}\""
" and Subset: \"{}\""
).format(project_name, asset_name, subset_name))
# Query asset document id by asset name
asset_doc = get_asset_by_name(project_name, asset_name, fields=["_id"])
if not asset_doc:
log.info(
"Asset \"{}\" was not found in Database.".format(asset_name)
)
return None
subset_doc = get_subset_by_name(
project_name, subset_name, asset_doc["_id"]
return get_last_version_by_subset_name(
project_name, subset_name, asset_name=asset_name
)
if not subset_doc:
log.info(
"Subset \"{}\" was not found in Database.".format(subset_name)
)
return None
version_doc = get_last_version_by_subset_id(
project_name, subset_doc["_id"]
)
if not version_doc:
log.info(
"Subset \"{}\" does not have any version yet.".format(subset_name)
)
return None
return version_doc
def get_workfile_template_key_from_context(

2
openpype/lib/local_settings.py
View file

@ -34,7 +34,7 @@ from openpype.settings import (
get_system_settings
)
from .import validate_mongo_connection
from openpype.client.mongo import validate_mongo_connection
_PLACEHOLDER = object()

7
openpype/lib/log.py
View file

@ -24,12 +24,13 @@ import traceback
import threading
import copy
from . import Terminal
from .mongo import (
from openpype.client.mongo import (
MongoEnvNotSet,
get_default_components,
OpenPypeMongoConnection
OpenPypeMongoConnection,
)
from . import Terminal
try:
import log4mongo
from log4mongo.handlers import MongoHandler

229
openpype/lib/mongo.py
View file

@ -1,206 +1,61 @@
import os
import sys
import time
import logging
import pymongo
import certifi
if sys.version_info[0] == 2:
from urlparse import urlparse, parse_qs
else:
from urllib.parse import urlparse, parse_qs
import warnings
import functools
from openpype.client.mongo import (
MongoEnvNotSet,
OpenPypeMongoConnection,
)
class MongoEnvNotSet(Exception):
class MongoDeprecatedWarning(DeprecationWarning):
pass
def _decompose_url(url):
"""Decompose mongo url to basic components.
def mongo_deprecated(func):
"""Mark functions as deprecated.
Used for creation of MongoHandler which expect mongo url components as
separated kwargs. Components are at the end not used as we're setting
connection directly this is just a dumb components for MongoHandler
validation pass.
It will result in a warning being emitted when the function is used.
"""
# Use first url from passed url
# - this is because it is possible to pass multiple urls for multiple
# replica sets which would crash on urlparse otherwise
# - please don't use comma in username of password
url = url.split(",")[0]
components = {
"scheme": None,
"host": None,
"port": None,
"username": None,
"password": None,
"auth_db": None
}
result = urlparse(url)
if result.scheme is None:
_url = "mongodb://{}".format(url)
result = urlparse(_url)
components["scheme"] = result.scheme
components["host"] = result.hostname
try:
components["port"] = result.port
except ValueError:
raise RuntimeError("invalid port specified")
components["username"] = result.username
components["password"] = result.password
try:
components["auth_db"] = parse_qs(result.query)['authSource'][0]
except KeyError:
# no auth db provided, mongo will use the one we are connecting to
pass
return components
def get_default_components():
mongo_url = os.environ.get("OPENPYPE_MONGO")
if mongo_url is None:
raise MongoEnvNotSet(
"URL for Mongo logging connection is not set."
@functools.wraps(func)
def new_func(*args, **kwargs):
warnings.simplefilter("always", MongoDeprecatedWarning)
warnings.warn(
(
"Call to deprecated function '{}'."
" Function was moved to 'openpype.client.mongo'."
).format(func.__name__),
category=MongoDeprecatedWarning,
stacklevel=2
)
return _decompose_url(mongo_url)
return func(*args, **kwargs)
return new_func
@mongo_deprecated
def get_default_components():
from openpype.client.mongo import get_default_components
return get_default_components()
@mongo_deprecated
def should_add_certificate_path_to_mongo_url(mongo_url):
"""Check if should add ca certificate to mongo url.
from openpype.client.mongo import should_add_certificate_path_to_mongo_url
Since 30.9.2021 cloud mongo requires newer certificates that are not
available on most of workstation. This adds path to certifi certificate
which is valid for it. To add the certificate path url must have scheme
'mongodb+srv' or has 'ssl=true' or 'tls=true' in url query.
"""
parsed = urlparse(mongo_url)
query = parse_qs(parsed.query)
lowered_query_keys = set(key.lower() for key in query.keys())
add_certificate = False
# Check if url 'ssl' or 'tls' are set to 'true'
for key in ("ssl", "tls"):
if key in query and "true" in query["ssl"]:
add_certificate = True
break
# Check if url contains 'mongodb+srv'
if not add_certificate and parsed.scheme == "mongodb+srv":
add_certificate = True
# Check if url does already contain certificate path
if add_certificate and "tlscafile" in lowered_query_keys:
add_certificate = False
return add_certificate
return should_add_certificate_path_to_mongo_url(mongo_url)
@mongo_deprecated
def validate_mongo_connection(mongo_uri):
"""Check if provided mongodb URL is valid.
from openpype.client.mongo import validate_mongo_connection
Args:
mongo_uri (str): URL to validate.
Raises:
ValueError: When port in mongo uri is not valid.
pymongo.errors.InvalidURI: If passed mongo is invalid.
pymongo.errors.ServerSelectionTimeoutError: If connection timeout
passed so probably couldn't connect to mongo server.
"""
client = OpenPypeMongoConnection.create_connection(
mongo_uri, retry_attempts=1
)
client.close()
return validate_mongo_connection(mongo_uri)
class OpenPypeMongoConnection:
"""Singleton MongoDB connection.
Keeps MongoDB connections by url.
"""
mongo_clients = {}
log = logging.getLogger("OpenPypeMongoConnection")
@staticmethod
def get_default_mongo_url():
return os.environ["OPENPYPE_MONGO"]
@classmethod
def get_mongo_client(cls, mongo_url=None):
if mongo_url is None:
mongo_url = cls.get_default_mongo_url()
connection = cls.mongo_clients.get(mongo_url)
if connection:
# Naive validation of existing connection
try:
connection.server_info()
with connection.start_session():
pass
except Exception:
connection = None
if not connection:
cls.log.debug("Creating mongo connection to {}".format(mongo_url))
connection = cls.create_connection(mongo_url)
cls.mongo_clients[mongo_url] = connection
return connection
@classmethod
def create_connection(cls, mongo_url, timeout=None, retry_attempts=None):
parsed = urlparse(mongo_url)
# Force validation of scheme
if parsed.scheme not in ["mongodb", "mongodb+srv"]:
raise pymongo.errors.InvalidURI((
"Invalid URI scheme:"
" URI must begin with 'mongodb://' or 'mongodb+srv://'"
))
if timeout is None:
timeout = int(os.environ.get("AVALON_TIMEOUT") or 1000)
kwargs = {
"serverSelectionTimeoutMS": timeout
}
if should_add_certificate_path_to_mongo_url(mongo_url):
kwargs["ssl_ca_certs"] = certifi.where()
mongo_client = pymongo.MongoClient(mongo_url, **kwargs)
if retry_attempts is None:
retry_attempts = 3
elif not retry_attempts:
retry_attempts = 1
last_exc = None
valid = False
t1 = time.time()
for attempt in range(1, retry_attempts + 1):
try:
mongo_client.server_info()
with mongo_client.start_session():
pass
valid = True
break
except Exception as exc:
last_exc = exc
if attempt < retry_attempts:
cls.log.warning(
"Attempt {} failed. Retrying... ".format(attempt)
)
time.sleep(1)
if not valid:
raise last_exc
cls.log.info("Connected to {}, delay {:.3f}s".format(
mongo_url, time.time() - t1
))
return mongo_client
__all__ = (
"MongoEnvNotSet",
"OpenPypeMongoConnection",
"get_default_components",
"should_add_certificate_path_to_mongo_url",
"validate_mongo_connection",
)

2
openpype/lib/remote_publish.py
View file

@ -7,7 +7,7 @@ from bson.objectid import ObjectId
import pyblish.util
import pyblish.api
from openpype.lib.mongo import OpenPypeMongoConnection
from openpype.client.mongo import OpenPypeMongoConnection
from openpype.lib.plugin_tools import parse_json
ERROR_STATUS = "error"

31
openpype/modules/deadline/plugins/publish/submit_publish_job.py
View file

@ -10,8 +10,10 @@ import clique
import pyblish.api
import openpype.api
from openpype.client import get_representations
from openpype.client import (
get_last_version_by_subset_name,
get_representations,
)
from openpype.pipeline import (
get_representation_path,
legacy_io,
@ -343,8 +345,13 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
# get latest version of subset
# this will stop if subset wasn't published yet
version = openpype.api.get_latest_version(instance.data.get("asset"),
instance.data.get("subset"))
project_name = legacy_io.active_project()
version = get_last_version_by_subset_name(
project_name,
instance.data.get("subset"),
asset_name=instance.data.get("asset")
)
# get its files based on extension
subset_resources = get_resources(
project_name, version, representation.get("ext")
@ -1025,9 +1032,12 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
prev_start = None
prev_end = None
version = openpype.api.get_latest_version(asset_name=asset,
subset_name=subset
)
project_name = legacy_io.active_project()
version = get_last_version_by_subset_name(
project_name,
subset,
asset_name=asset
)
# Set prev start / end frames for comparison
if not prev_start and not prev_end:
@ -1072,7 +1082,12 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
based on 'publish' template
"""
if not version:
version = openpype.api.get_latest_version(asset, subset)
project_name = legacy_io.active_project()
version = get_last_version_by_subset_name(
project_name,
subset,
asset_name=asset
)
if version:
version = int(version["name"]) + 1
else:

8
openpype/modules/ftrack/ftrack_server/event_server_cli.py
View file

@ -1,11 +1,9 @@
import os
import sys
import signal
import datetime
import subprocess
import socket
import json
import platform
import getpass
import atexit
import time
@ -13,12 +11,14 @@ import uuid
import ftrack_api
import pymongo
from openpype.client.mongo import (
OpenPypeMongoConnection,
validate_mongo_connection,
)
from openpype.lib import (
get_openpype_execute_args,
OpenPypeMongoConnection,
get_openpype_version,
get_build_version,
validate_mongo_connection
)
from openpype_modules.ftrack import FTRACK_MODULE_DIR
from openpype_modules.ftrack.lib import credentials

2
openpype/modules/ftrack/ftrack_server/lib.py
View file

@ -24,7 +24,7 @@ except ImportError:
from ftrack_api._weakref import WeakMethod
from openpype_modules.ftrack.lib import get_ftrack_event_mongo_info
from openpype.lib import OpenPypeMongoConnection
from openpype.client import OpenPypeMongoConnection
from openpype.api import Logger
TOPIC_STATUS_SERVER = "openpype.event.server.status"

3
openpype/modules/ftrack/scripts/sub_event_storer.py
View file

@ -6,6 +6,8 @@ import socket
import pymongo
import ftrack_api
from openpype.client import OpenPypeMongoConnection
from openpype_modules.ftrack.ftrack_server.ftrack_server import FtrackServer
from openpype_modules.ftrack.ftrack_server.lib import (
SocketSession,
@ -15,7 +17,6 @@ from openpype_modules.ftrack.ftrack_server.lib import (
)
from openpype_modules.ftrack.lib import get_ftrack_event_mongo_info
from openpype.lib import (
OpenPypeMongoConnection,
get_openpype_version,
get_build_version
)

2
openpype/modules/slack/plugins/publish/integrate_slack_api.py
View file

@ -4,8 +4,8 @@ import pyblish.api
import copy
from datetime import datetime
from openpype.client import OpenPypeMongoConnection
from openpype.lib.plugin_tools import prepare_template_data
from openpype.lib import OpenPypeMongoConnection
class IntegrateSlackAPI(pyblish.api.InstancePlugin):

14
openpype/pipeline/context_tools.py
View file

@ -14,6 +14,7 @@ from openpype.client import (
get_project,
get_asset_by_id,
get_asset_by_name,
version_is_latest,
)
from openpype.modules import load_modules, ModulesManager
from openpype.settings import get_project_settings
@ -334,3 +335,16 @@ def get_current_project_asset(asset_name=None, asset_id=None, fields=None):
if not asset_name:
return None
return get_asset_by_name(project_name, asset_name, fields=fields)
def is_representation_from_latest(representation):
"""Return whether the representation is from latest version
Args:
representation (dict): The representation document from the database.
Returns:
bool: Whether the representation is of latest version.
"""
project_name = legacy_io.active_project()
return version_is_latest(project_name, representation["parent"])

8
openpype/pipeline/load/__init__.py
View file

@ -24,6 +24,10 @@ from .utils import (
loaders_from_repre_context,
loaders_from_representation,
any_outdated_containers,
get_outdated_containers,
filter_containers,
)
from .plugins import (
@ -66,6 +70,10 @@ __all__ = (
"loaders_from_repre_context",
"loaders_from_representation",
"any_outdated_containers",
"get_outdated_containers",
"filter_containers",
# plugins.py
"LoaderPlugin",
"SubsetLoaderPlugin",

169
openpype/pipeline/load/utils.py
View file

@ -4,8 +4,10 @@ import copy
import getpass
import logging
import inspect
import collections
import numbers
from openpype.host import ILoadHost
from openpype.client import (
get_project,
get_assets,
@ -15,6 +17,7 @@ from openpype.client import (
get_last_version_by_subset_id,
get_hero_version_by_subset_id,
get_version_by_name,
get_last_versions,
get_representations,
get_representation_by_id,
get_representation_by_name,
@ -28,6 +31,11 @@ from openpype.pipeline import (
log = logging.getLogger(__name__)
ContainersFilterResult = collections.namedtuple(
"ContainersFilterResult",
["latest", "outdated", "not_foud", "invalid"]
)
class HeroVersionType(object):
def __init__(self, version):
@ -685,3 +693,164 @@ def loaders_from_representation(loaders, representation):
context = get_representation_context(representation)
return loaders_from_repre_context(loaders, context)
def any_outdated_containers(host=None, project_name=None):
"""Check if there are any outdated containers in scene."""
if get_outdated_containers(host, project_name):
return True
return False
def get_outdated_containers(host=None, project_name=None):
"""Collect outdated containers from host scene.
Currently registered host and project in global session are used if
arguments are not passed.
Args:
host (ModuleType): Host implementation with 'ls' function available.
project_name (str): Name of project in which context we are.
"""
if host is None:
from openpype.pipeline import registered_host
host = registered_host()
if project_name is None:
project_name = legacy_io.active_project()
if isinstance(host, ILoadHost):
containers = host.get_containers()
else:
containers = host.ls()
return filter_containers(containers, project_name).outdated
def filter_containers(containers, project_name):
"""Filter containers and split them into 4 categories.
Categories are 'latest', 'outdated', 'invalid' and 'not_found'.
The 'lastest' containers are from last version, 'outdated' are not,
'invalid' are invalid containers (invalid content) and 'not_foud' has
some missing entity in database.
Args:
containers (Iterable[dict]): List of containers referenced into scene.
project_name (str): Name of project in which context shoud look for
versions.
Returns:
ContainersFilterResult: Named tuple with 'latest', 'outdated',
'invalid' and 'not_found' containers.
"""
# Make sure containers is list that won't change
containers = list(containers)
outdated_containers = []
uptodate_containers = []
not_found_containers = []
invalid_containers = []
output = ContainersFilterResult(
uptodate_containers,
outdated_containers,
not_found_containers,
invalid_containers
)
# Query representation docs to get it's version ids
repre_ids = {
container["representation"]
for container in containers
if container["representation"]
}
if not repre_ids:
if containers:
invalid_containers.extend(containers)
return output
repre_docs = get_representations(
project_name,
representation_ids=repre_ids,
fields=["_id", "parent"]
)
# Store representations by stringified representation id
repre_docs_by_str_id = {}
repre_docs_by_version_id = collections.defaultdict(list)
for repre_doc in repre_docs:
repre_id = str(repre_doc["_id"])
version_id = repre_doc["parent"]
repre_docs_by_str_id[repre_id] = repre_doc
repre_docs_by_version_id[version_id].append(repre_doc)
# Query version docs to get it's subset ids
# - also query hero version to be able identify if representation
# belongs to existing version
version_docs = get_versions(
project_name,
version_ids=repre_docs_by_version_id.keys(),
hero=True,
fields=["_id", "parent", "type"]
)
verisons_by_id = {}
versions_by_subset_id = collections.defaultdict(list)
hero_version_ids = set()
for version_doc in version_docs:
version_id = version_doc["_id"]
# Store versions by their ids
verisons_by_id[version_id] = version_doc
# There's no need to query subsets for hero versions
# - they are considered as latest?
if version_doc["type"] == "hero_version":
hero_version_ids.add(version_id)
continue
subset_id = version_doc["parent"]
versions_by_subset_id[subset_id].append(version_doc)
last_versions = get_last_versions(
project_name,
subset_ids=versions_by_subset_id.keys(),
fields=["_id"]
)
# Figure out which versions are outdated
outdated_version_ids = set()
for subset_id, last_version_doc in last_versions.items():
for version_doc in versions_by_subset_id[subset_id]:
version_id = version_doc["_id"]
if version_id != last_version_doc["_id"]:
outdated_version_ids.add(version_id)
# Based on all collected data figure out which containers are outdated
# - log out if there are missing representation or version documents
for container in containers:
container_name = container["objectName"]
repre_id = container["representation"]
if not repre_id:
invalid_containers.append(container)
continue
repre_doc = repre_docs_by_str_id.get(repre_id)
if not repre_doc:
log.debug((
"Container '{}' has an invalid representation."
" It is missing in the database."
).format(container_name))
not_found_containers.append(container)
continue
version_id = repre_doc["parent"]
if version_id in outdated_version_ids:
outdated_containers.append(container)
elif version_id not in verisons_by_id:
log.debug((
"Representation on container '{}' has an invalid version."
" It is missing in the database."
).format(container_name))
not_found_containers.append(container)
else:
uptodate_containers.append(container)
return output

4
openpype/pipeline/mongodb.py
View file

@ -5,6 +5,8 @@ import logging
import pymongo
from uuid import uuid4
from openpype.client import OpenPypeMongoConnection
from . import schema
@ -156,8 +158,6 @@ class AvalonMongoDB:
@property
def mongo_client(self):
from openpype.lib import OpenPypeMongoConnection
return OpenPypeMongoConnection.get_mongo_client()
@property

129
openpype/plugins/publish/extract_thumbnail.py
View file

@ -1,4 +1,5 @@
import os
import tempfile
import pyblish.api
from openpype.lib import (
@ -8,8 +9,6 @@ from openpype.lib import (
run_subprocess,
path_to_subprocess_arg,
execute,
)
@ -29,7 +28,27 @@ class ExtractThumbnail(pyblish.api.InstancePlugin):
ffmpeg_args = None
def process(self, instance):
self.log.info("subset {}".format(instance.data['subset']))
subset_name = instance.data["subset"]
instance_repres = instance.data.get("representations")
if not instance_repres:
self.log.debug((
"Instance {} does not have representations. Skipping"
).format(subset_name))
return
self.log.info(
"Processing instance with subset name {}".format(subset_name)
)
# Skip if instance have 'review' key in data set to 'False'
if not self._is_review_instance(instance):
self.log.info("Skipping - no review set on instance.")
return
# Check if already has thumbnail created
if self._already_has_thumbnail(instance_repres):
self.log.info("Thumbnail representation already present.")
return
# skip crypto passes.
# TODO: This is just a quick fix and has its own side-effects - it is
@ -37,20 +56,29 @@ class ExtractThumbnail(pyblish.api.InstancePlugin):
# This must be solved properly, maybe using tags on
# representation that can be determined much earlier and
# with better precision.
if 'crypto' in instance.data['subset'].lower():
if "crypto" in subset_name.lower():
self.log.info("Skipping crypto passes.")
return
# Skip if review not set.
if not instance.data.get("review", True):
self.log.info("Skipping - no review set on instance.")
return
if self._already_has_thumbnail(instance):
self.log.info("Thumbnail representation already present.")
return
filtered_repres = self._get_filtered_repres(instance)
if not filtered_repres:
self.log.info((
"Instance don't have representations"
" that can be used as source for thumbnail. Skipping"
))
return
# Create temp directory for thumbnail
# - this is to avoid "override" of source file
dst_staging = tempfile.mkdtemp(prefix="pyblish_tmp_")
self.log.debug(
"Create temp directory {} for thumbnail".format(dst_staging)
)
# Store new staging to cleanup paths
instance.context.data["cleanupFullPaths"].append(dst_staging)
thumbnail_created = False
oiio_supported = is_oiio_supported()
for repre in filtered_repres:
repre_files = repre["files"]
if not isinstance(repre_files, (list, tuple)):
@ -59,41 +87,43 @@ class ExtractThumbnail(pyblish.api.InstancePlugin):
file_index = int(float(len(repre_files)) * 0.5)
input_file = repre_files[file_index]
stagingdir = os.path.normpath(repre["stagingDir"])
full_input_path = os.path.join(stagingdir, input_file)
src_staging = os.path.normpath(repre["stagingDir"])
full_input_path = os.path.join(src_staging, input_file)
self.log.info("input {}".format(full_input_path))
filename = os.path.splitext(input_file)[0]
if not filename.endswith('.'):
filename += "."
jpeg_file = filename + "jpg"
full_output_path = os.path.join(stagingdir, jpeg_file)
jpeg_file = filename + ".jpg"
full_output_path = os.path.join(dst_staging, jpeg_file)
thumbnail_created = False
# Try to use FFMPEG if OIIO is not supported (for cases when
# oiiotool isn't available)
if not is_oiio_supported():
thumbnail_created = self.create_thumbnail_ffmpeg(full_input_path, full_output_path) # noqa
else:
if oiio_supported:
self.log.info("Trying to convert with OIIO")
# If the input can read by OIIO then use OIIO method for
# conversion otherwise use ffmpeg
self.log.info("Trying to convert with OIIO") # noqa
thumbnail_created = self.create_thumbnail_oiio(full_input_path, full_output_path) # noqa
thumbnail_created = self.create_thumbnail_oiio(
full_input_path, full_output_path
)
if not thumbnail_created:
self.log.info("Converting with FFMPEG because input can't be read by OIIO.") # noqa
thumbnail_created = self.create_thumbnail_ffmpeg(full_input_path, full_output_path) # noqa
# Skip the rest of the process if the thumbnail wasn't created
# Try to use FFMPEG if OIIO is not supported or for cases when
# oiiotool isn't available
if not thumbnail_created:
self.log.warning("Thumbanil has not been created.")
return
if oiio_supported:
self.log.info((
"Converting with FFMPEG because input"
" can't be read by OIIO."
))
thumbnail_created = self.create_thumbnail_ffmpeg(
full_input_path, full_output_path
)
# Skip representation and try next one if wasn't created
if not thumbnail_created:
continue
new_repre = {
"name": "thumbnail",
"ext": "jpg",
"files": jpeg_file,
"stagingDir": stagingdir,
"stagingDir": dst_staging,
"thumbnail": True,
"tags": ["thumbnail"]
}
@ -106,12 +136,21 @@ class ExtractThumbnail(pyblish.api.InstancePlugin):
# There is no need to create more then one thumbnail
break
def _already_has_thumbnail(self, instance):
for repre in instance.data.get("representations", []):
if not thumbnail_created:
self.log.warning("Thumbanil has not been created.")
def _is_review_instance(self, instance):
# TODO: We should probably handle "not creating" of thumbnail
# other way then checking for "review" key on instance data?
if instance.data.get("review", True):
return True
return False
def _already_has_thumbnail(self, repres):
for repre in repres:
self.log.info("repre {}".format(repre))
if repre["name"] == "thumbnail":
return True
return False
def _get_filtered_repres(self, instance):
@ -136,12 +175,12 @@ class ExtractThumbnail(pyblish.api.InstancePlugin):
def create_thumbnail_oiio(self, src_path, dst_path):
self.log.info("outputting {}".format(dst_path))
oiio_tool_path = get_oiio_tools_path()
oiio_cmd = [oiio_tool_path, "-a",
src_path, "-o",
dst_path
]
subprocess_exr = " ".join(oiio_cmd)
self.log.info(f"running: {subprocess_exr}")
oiio_cmd = [
oiio_tool_path,
"-a", src_path,
"-o", dst_path
]
self.log.info("running: {}".format(" ".join(oiio_cmd)))
try:
run_subprocess(oiio_cmd, logger=self.log)
return True

93
openpype/plugins/publish/integrate.py
View file

@ -10,6 +10,11 @@ from pymongo import DeleteMany, ReplaceOne, InsertOne, UpdateOne
import pyblish.api
import openpype.api
from openpype.client import (
get_representations,
get_subset_by_name,
get_version_by_name,
)
from openpype.lib.profiles_filtering import filter_profiles
from openpype.lib.file_transaction import FileTransaction
from openpype.pipeline import legacy_io
@ -156,7 +161,7 @@ class IntegrateAsset(pyblish.api.InstancePlugin):
"mvUsdOverride",
"simpleUnrealTexture"
]
exclude_families = ["clip", "render.farm"]
default_template_name = "publish"
# Representation context keys that should always be written to
@ -190,14 +195,6 @@ class IntegrateAsset(pyblish.api.InstancePlugin):
).format(instance.data["family"]))
return
# Exclude instances that also contain families from exclude families
families = set(get_instance_families(instance))
exclude = families & set(self.exclude_families)
if exclude:
self.log.debug("Instance not integrated due to exclude "
"families found: {}".format(", ".join(exclude)))
return
file_transactions = FileTransaction(log=self.log)
try:
self.register(instance, file_transactions, filtered_repres)
@ -274,6 +271,8 @@ class IntegrateAsset(pyblish.api.InstancePlugin):
return filtered_repres
def register(self, instance, file_transactions, filtered_repres):
project_name = legacy_io.active_project()
instance_stagingdir = instance.data.get("stagingDir")
if not instance_stagingdir:
self.log.info((
@ -289,19 +288,19 @@ class IntegrateAsset(pyblish.api.InstancePlugin):
template_name = self.get_template_name(instance)
subset, subset_writes = self.prepare_subset(instance)
version, version_writes = self.prepare_version(instance, subset)
subset, subset_writes = self.prepare_subset(instance, project_name)
version, version_writes = self.prepare_version(
instance, subset, project_name
)
instance.data["versionEntity"] = version
# Get existing representations (if any)
existing_repres_by_name = {
repres["name"].lower(): repres for repres in legacy_io.find(
{
"parent": version["_id"],
"type": "representation"
},
# Only care about id and name of existing representations
projection={"_id": True, "name": True}
repre_doc["name"].lower(): repre_doc
for repre_doc in get_representations(
project_name,
version_ids=[version["_id"]],
fields=["_id", "name"]
)
}
@ -426,17 +425,15 @@ class IntegrateAsset(pyblish.api.InstancePlugin):
self.log.info("Registered {} representations"
"".format(len(prepared_representations)))
def prepare_subset(self, instance):
asset = instance.data.get("assetEntity")
def prepare_subset(self, instance, project_name):
asset_doc = instance.data.get("assetEntity")
subset_name = instance.data["subset"]
self.log.debug("Subset: {}".format(subset_name))
# Get existing subset if it exists
subset = legacy_io.find_one({
"type": "subset",
"parent": asset["_id"],
"name": subset_name
})
subset_doc = get_subset_by_name(
project_name, subset_name, asset_doc["_id"]
)
# Define subset data
data = {
@ -448,68 +445,68 @@ class IntegrateAsset(pyblish.api.InstancePlugin):
data["subsetGroup"] = subset_group
bulk_writes = []
if subset is None:
if subset_doc is None:
# Create a new subset
self.log.info("Subset '%s' not found, creating ..." % subset_name)
subset = {
subset_doc = {
"_id": ObjectId(),
"schema": "openpype:subset-3.0",
"type": "subset",
"name": subset_name,
"data": data,
"parent": asset["_id"]
"parent": asset_doc["_id"]
}
bulk_writes.append(InsertOne(subset))
bulk_writes.append(InsertOne(subset_doc))
else:
# Update existing subset data with new data and set in database.
# We also change the found subset in-place so we don't need to
# re-query the subset afterwards
subset["data"].update(data)
subset_doc["data"].update(data)
bulk_writes.append(UpdateOne(
{"type": "subset", "_id": subset["_id"]},
{"type": "subset", "_id": subset_doc["_id"]},
{"$set": {
"data": subset["data"]
"data": subset_doc["data"]
}}
))
self.log.info("Prepared subset: {}".format(subset_name))
return subset, bulk_writes
def prepare_version(self, instance, subset):
return subset_doc, bulk_writes
def prepare_version(self, instance, subset_doc, project_name):
version_number = instance.data["version"]
version = {
version_doc = {
"schema": "openpype:version-3.0",
"type": "version",
"parent": subset["_id"],
"parent": subset_doc["_id"],
"name": version_number,
"data": self.create_version_data(instance)
}
existing_version = legacy_io.find_one({
'type': 'version',
'parent': subset["_id"],
'name': version_number
}, projection={"_id": True})
existing_version = get_version_by_name(
project_name,
version_number,
subset_doc["_id"],
fields=["_id"]
)
if existing_version:
self.log.debug("Updating existing version ...")
version["_id"] = existing_version["_id"]
version_doc["_id"] = existing_version["_id"]
else:
self.log.debug("Creating new version ...")
version["_id"] = ObjectId()
version_doc["_id"] = ObjectId()
bulk_writes = [ReplaceOne(
filter={"_id": version["_id"]},
replacement=version,
filter={"_id": version_doc["_id"]},
replacement=version_doc,
upsert=True
)]
self.log.info("Prepared version: v{0:03d}".format(version["name"]))
self.log.info("Prepared version: v{0:03d}".format(version_doc["name"]))
return version, bulk_writes
return version_doc, bulk_writes
def prepare_representation(self, repre,
template_name,

6
openpype/plugins/publish/validate_containers.py
View file

@ -1,5 +1,5 @@
import pyblish.api
import openpype.lib
from openpype.pipeline.load import any_outdated_containers
class ShowInventory(pyblish.api.Action):
@ -19,10 +19,10 @@ class ValidateContainers(pyblish.api.ContextPlugin):
label = "Validate Containers"
order = pyblish.api.ValidatorOrder
hosts = ["maya", "houdini", "nuke", "harmony", "photoshop"]
hosts = ["maya", "houdini", "nuke", "harmony", "photoshop", "aftereffects"]
optional = True
actions = [ShowInventory]
def process(self, context):
if openpype.lib.any_outdated():
if any_outdated_containers():
raise ValueError("There are outdated containers in the scene.")

15
openpype/settings/handlers.py
View file

@ -7,6 +7,8 @@ from abc import ABCMeta, abstractmethod
import six
import openpype.version
from openpype.client.mongo import OpenPypeMongoConnection
from openpype.client.entities import get_project_connection, get_project
from .constants import (
GLOBAL_SETTINGS_KEY,
@ -337,9 +339,6 @@ class MongoSettingsHandler(SettingsHandler):
def __init__(self):
# Get mongo connection
from openpype.lib import OpenPypeMongoConnection
from openpype.pipeline import AvalonMongoDB
settings_collection = OpenPypeMongoConnection.get_mongo_client()
self._anatomy_keys = None
@ -362,7 +361,6 @@ class MongoSettingsHandler(SettingsHandler):
self.collection_name = collection_name
self.collection = settings_collection[database_name][collection_name]
self.avalon_db = AvalonMongoDB()
self.system_settings_cache = CacheValues()
self.project_settings_cache = collections.defaultdict(CacheValues)
@ -607,16 +605,14 @@ class MongoSettingsHandler(SettingsHandler):
new_data = data_cache.data_copy()
# Prepare avalon project document
collection = self.avalon_db.database[project_name]
project_doc = collection.find_one({
"type": "project"
})
project_doc = get_project(project_name)
if not project_doc:
raise ValueError((
"Project document of project \"{}\" does not exists."
" Create project first."
).format(project_name))
collection = get_project_connection(project_name)
# Project's data
update_dict_data = {}
project_doc_data = project_doc.get("data") or {}
@ -1145,8 +1141,7 @@ class MongoSettingsHandler(SettingsHandler):
document, version
)
else:
collection = self.avalon_db.database[project_name]
project_doc = collection.find_one({"type": "project"})
project_doc = get_project(project_name)
self.project_anatomy_cache[project_name].update_data(
self.project_doc_to_anatomy_data(project_doc),
self._current_version

1
openpype/tests/test_lib_restructuralization.py
View file

@ -22,7 +22,6 @@ def test_backward_compatibility(printer):
from openpype.lib import any_outdated
from openpype.lib import get_asset
from openpype.lib import get_linked_assets
from openpype.lib import get_latest_version
from openpype.lib import get_ffprobe_streams
from openpype.hosts.fusion.lib import switch_item

4
openpype/tools/publisher/publish_report_viewer/model.py
View file

@ -1,9 +1,9 @@
import uuid
import html
from Qt import QtCore, QtGui
import pyblish.api
from openpype.tools.utils.lib import html_escape
from .constants import (
ITEM_ID_ROLE,
ITEM_IS_GROUP_ROLE,
@ -46,7 +46,7 @@ class InstancesModel(QtGui.QStandardItemModel):
all_removed = True
for instance_item in instance_items:
item = QtGui.QStandardItem(instance_item.label)
instance_label = html.escape(instance_item.label)
instance_label = html_escape(instance_item.label)
item.setData(instance_label, ITEM_LABEL_ROLE)
item.setData(instance_item.errored, ITEM_ERRORED_ROLE)
item.setData(instance_item.id, ITEM_ID_ROLE)

4
openpype/tools/publisher/widgets/card_view_widgets.py
View file

@ -22,13 +22,13 @@ Only one item can be selected at a time.
import re
import collections
import html
from Qt import QtWidgets, QtCore
from openpype.widgets.nice_checkbox import NiceCheckbox
from openpype.tools.utils import BaseClickableFrame
from openpype.tools.utils.lib import html_escape
from .widgets import (
AbstractInstanceView,
ContextWarningLabel,
@ -308,7 +308,7 @@ class InstanceCardWidget(CardWidget):
self._last_variant = variant
self._last_subset_name = subset_name
# Make `variant` bold
label = html.escape(self.instance.label)
label = html_escape(self.instance.label)
found_parts = set(re.findall(variant, label, re.IGNORECASE))
if found_parts:
for part in found_parts:

6
openpype/tools/publisher/widgets/list_view_widgets.py
View file

@ -23,12 +23,12 @@ selection can be enabled disabled using checkbox or keyboard key presses:
```
"""
import collections
import html
from Qt import QtWidgets, QtCore, QtGui
from openpype.style import get_objected_colors
from openpype.widgets.nice_checkbox import NiceCheckbox
from openpype.tools.utils.lib import html_escape
from .widgets import AbstractInstanceView
from ..constants import (
INSTANCE_ID_ROLE,
@ -114,7 +114,7 @@ class InstanceListItemWidget(QtWidgets.QWidget):
self.instance = instance
instance_label = html.escape(instance.label)
instance_label = html_escape(instance.label)
subset_name_label = QtWidgets.QLabel(instance_label, self)
subset_name_label.setObjectName("ListViewSubsetName")
@ -181,7 +181,7 @@ class InstanceListItemWidget(QtWidgets.QWidget):
# Check subset name
label = self.instance.label
if label != self._instance_label_widget.text():
self._instance_label_widget.setText(html.escape(label))
self._instance_label_widget.setText(html_escape(label))
# Check active state
self.set_active(self.instance["active"])
# Check valid states

13
openpype/tools/utils/lib.py
View file

@ -37,6 +37,19 @@ def center_window(window):
window.move(geo.topLeft())
def html_escape(text):
"""Basic escape of html syntax symbols in text."""
return (
text
.replace("&", "&amp;")
.replace("<", "&lt;")
.replace(">", "&gt;")
.replace('"', "&quot;")
.replace("'", "&#x27;")
)
def set_style_property(widget, property_name, property_value):
"""Set widget's property that may affect style.

11
openpype/vendor/python/common/capture.py vendored
View file

@ -665,7 +665,10 @@ def _applied_camera_options(options, panel):
_iteritems = getattr(options, "iteritems", options.items)
for opt, value in _iteritems():
_safe_setAttr(camera + "." + opt, value)
if cmds.getAttr(camera + "." + opt, lock=True):
continue
else:
_safe_setAttr(camera + "." + opt, value)
try:
yield
@ -673,7 +676,11 @@ def _applied_camera_options(options, panel):
if old_options:
_iteritems = getattr(old_options, "iteritems", old_options.items)
for opt, value in _iteritems():
_safe_setAttr(camera + "." + opt, value)
#
if cmds.getAttr(camera + "." + opt, lock=True):
continue
else:
_safe_setAttr(camera + "." + opt, value)
@contextlib.contextmanager