Frame13/ayon-core
1
0
Fork 1
mirror of https://github.com/ynput/ayon-core.git synced 2025-12-24 21:04:40 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #2222 from pypeclub/feature/separate_webpublisher_logic

Browse source 
Webpublisher: Separate webpublisher logic
This commit is contained in:
Jakub Trllo 2021-11-10 17:16:45 +01:00 • committed by GitHub
commit d376f7ca41
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 184 additions and 152 deletions
Download patch file Download diff file Expand all files Collapse all files

13
openpype/cli.py
View file

@ -166,7 +166,7 @@ def publish(debug, paths, targets):
@click.option("-p", "--project", help="Project")
@click.option("-t", "--targets", help="Targets", default=None,
multiple=True)
def remotepublishfromapp(debug, project, path, host, targets=None, user=None):
def remotepublishfromapp(debug, project, path, host, user=None, targets=None):
"""Start CLI publishing.
Publish collects json from paths provided as an argument.
@ -174,18 +174,19 @@ def remotepublishfromapp(debug, project, path, host, targets=None, user=None):
"""
if debug:
os.environ['OPENPYPE_DEBUG'] = '3'
PypeCommands.remotepublishfromapp(project, path, host, user,
targets=targets)
PypeCommands.remotepublishfromapp(
project, path, host, user, targets=targets
)
@main.command()
@click.argument("path")
@click.option("-d", "--debug", is_flag=True, help="Print debug messages")
@click.option("-h", "--host", help="Host")
@click.option("-u", "--user", help="User email address")
@click.option("-p", "--project", help="Project")
@click.option("-t", "--targets", help="Targets", default=None,
multiple=True)
def remotepublish(debug, project, path, host, targets=None, user=None):
def remotepublish(debug, project, path, user=None, targets=None):
"""Start CLI publishing.
Publish collects json from paths provided as an argument.
@ -193,7 +194,7 @@ def remotepublish(debug, project, path, host, targets=None, user=None):
"""
if debug:
os.environ['OPENPYPE_DEBUG'] = '3'
PypeCommands.remotepublish(project, path, host, user, targets=targets)
PypeCommands.remotepublish(project, path, user, targets=targets)
@main.command()

84
openpype/hosts/webpublisher/plugins/publish/collect_batch_data.py Normal file
View file

@ -0,0 +1,84 @@
"""Loads batch context from json and continues in publish process.
Provides:
context -> Loaded batch file.
"""
import os
import pyblish.api
from avalon import io
from openpype.lib.plugin_tools import (
parse_json,
get_batch_asset_task_info
)
from openpype.lib.remote_publish import get_webpublish_conn
class CollectBatchData(pyblish.api.ContextPlugin):
"""Collect batch data from json stored in 'OPENPYPE_PUBLISH_DATA' env dir.
The directory must contain 'manifest.json' file where batch data should be
stored.
"""
# must be really early, context values are only in json file
order = pyblish.api.CollectorOrder - 0.495
label = "Collect batch data"
host = ["webpublisher"]
def process(self, context):
batch_dir = os.environ.get("OPENPYPE_PUBLISH_DATA")
assert batch_dir, (
"Missing `OPENPYPE_PUBLISH_DATA`")
assert os.path.exists(batch_dir), \
"Folder {} doesn't exist".format(batch_dir)
project_name = os.environ.get("AVALON_PROJECT")
if project_name is None:
raise AssertionError(
"Environment `AVALON_PROJECT` was not found."
"Could not set project `root` which may cause issues."
)
batch_data = parse_json(os.path.join(batch_dir, "manifest.json"))
context.data["batchDir"] = batch_dir
context.data["batchData"] = batch_data
asset_name, task_name, task_type = get_batch_asset_task_info(
batch_data["context"]
)
os.environ["AVALON_ASSET"] = asset_name
io.Session["AVALON_ASSET"] = asset_name
os.environ["AVALON_TASK"] = task_name
io.Session["AVALON_TASK"] = task_name
context.data["asset"] = asset_name
context.data["task"] = task_name
context.data["taskType"] = task_type
self._set_ctx_path(batch_data)
def _set_ctx_path(self, batch_data):
dbcon = get_webpublish_conn()
batch_id = batch_data["batch"]
ctx_path = batch_data["context"]["path"]
self.log.info("ctx_path: {}".format(ctx_path))
self.log.info("batch_id: {}".format(batch_id))
if ctx_path and batch_id:
self.log.info("Updating log record")
dbcon.update_one(
{
"batch_id": batch_id,
"status": "in_progress"
},
{
"$set": {
"path": ctx_path
}
}
)

7
openpype/hosts/webpublisher/plugins/publish/collect_fps.py
View file

@ -20,9 +20,8 @@ class CollectFPS(pyblish.api.InstancePlugin):
hosts = ["webpublisher"]
def process(self, instance):
fps = instance.context.data["fps"]
instance_fps = instance.data.get("fps")
if instance_fps is None:
instance.data["fps"] = instance.context.data["fps"]
instance.data.update({
"fps": fps
})
self.log.debug(f"instance.data: {pformat(instance.data)}")

76
openpype/hosts/webpublisher/plugins/publish/collect_published_files.py
View file

@ -1,21 +1,19 @@
"""Loads publishing context from json and continues in publish process.
"""Create instances from batch data and continues in publish process.
Requires:
anatomy -> context["anatomy"] *(pyblish.api.CollectorOrder - 0.11)
CollectBatchData
Provides:
context, instances -> All data from previous publishing process.
"""
import os
import json
import clique
import tempfile
import pyblish.api
from avalon import io
import pyblish.api
from openpype.lib import prepare_template_data
from openpype.lib.plugin_tools import parse_json, get_batch_asset_task_info
from openpype.lib.plugin_tools import parse_json
class CollectPublishedFiles(pyblish.api.ContextPlugin):
@ -28,28 +26,28 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin):
order = pyblish.api.CollectorOrder - 0.490
label = "Collect rendered frames"
host = ["webpublisher"]
_context = None
targets = ["filespublish"]
# from Settings
task_type_to_family = {}
def _process_batch(self, dir_url):
task_subfolders = [
os.path.join(dir_url, o)
for o in os.listdir(dir_url)
if os.path.isdir(os.path.join(dir_url, o))]
def process(self, context):
batch_dir = context.data["batchDir"]
task_subfolders = []
for folder_name in os.listdir(batch_dir):
full_path = os.path.join(batch_dir, folder_name)
if os.path.isdir(full_path):
task_subfolders.append(full_path)
self.log.info("task_sub:: {}".format(task_subfolders))
asset_name = context.data["asset"]
task_name = context.data["task"]
task_type = context.data["taskType"]
for task_dir in task_subfolders:
task_data = parse_json(os.path.join(task_dir,
"manifest.json"))
self.log.info("task_data:: {}".format(task_data))
ctx = task_data["context"]
asset, task_name, task_type = get_batch_asset_task_info(ctx)
if task_name:
os.environ["AVALON_TASK"] = task_name
is_sequence = len(task_data["files"]) > 1
@ -60,26 +58,20 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin):
is_sequence,
extension.replace(".", ''))
subset = self._get_subset_name(family, subset_template, task_name,
task_data["variant"])
subset = self._get_subset_name(
family, subset_template, task_name, task_data["variant"]
)
version = self._get_last_version(asset_name, subset) + 1
os.environ["AVALON_ASSET"] = asset
io.Session["AVALON_ASSET"] = asset
instance = self._context.create_instance(subset)
instance.data["asset"] = asset
instance = context.create_instance(subset)
instance.data["asset"] = asset_name
instance.data["subset"] = subset
instance.data["family"] = family
instance.data["families"] = families
instance.data["version"] = \
self._get_last_version(asset, subset) + 1
instance.data["version"] = version
instance.data["stagingDir"] = tempfile.mkdtemp()
instance.data["source"] = "webpublisher"
# to store logging info into DB openpype.webpublishes
instance.data["ctx_path"] = ctx["path"]
instance.data["batch_id"] = task_data["batch"]
# to convert from email provided into Ftrack username
instance.data["user_email"] = task_data["user"]
@ -230,23 +222,3 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin):
return version[0].get("version") or 0
else:
return 0
def process(self, context):
self._context = context
batch_dir = os.environ.get("OPENPYPE_PUBLISH_DATA")
assert batch_dir, (
"Missing `OPENPYPE_PUBLISH_DATA`")
assert os.path.exists(batch_dir), \
"Folder {} doesn't exist".format(batch_dir)
project_name = os.environ.get("AVALON_PROJECT")
if project_name is None:
raise AssertionError(
"Environment `AVALON_PROJECT` was not found."
"Could not set project `root` which may cause issues."
)
self._process_batch(batch_dir)

38
openpype/hosts/webpublisher/plugins/publish/integrate_context_to_log.py
View file

@ -1,38 +0,0 @@
import os
import pyblish.api
from openpype.lib import OpenPypeMongoConnection
class IntegrateContextToLog(pyblish.api.ContextPlugin):
""" Adds context information to log document for displaying in front end"""
label = "Integrate Context to Log"
order = pyblish.api.IntegratorOrder - 0.1
hosts = ["webpublisher"]
def process(self, context):
self.log.info("Integrate Context to Log")
mongo_client = OpenPypeMongoConnection.get_mongo_client()
database_name = os.environ["OPENPYPE_DATABASE_NAME"]
dbcon = mongo_client[database_name]["webpublishes"]
for instance in context:
self.log.info("ctx_path: {}".format(instance.data.get("ctx_path")))
self.log.info("batch_id: {}".format(instance.data.get("batch_id")))
if instance.data.get("ctx_path") and instance.data.get("batch_id"):
self.log.info("Updating log record")
dbcon.update_one(
{
"batch_id": instance.data.get("batch_id"),
"status": "in_progress"
},
{"$set":
{
"path": instance.data.get("ctx_path")
}}
)
return

81
openpype/hosts/webpublisher/webserver_service/webpublish_routes.py
View file

@ -176,23 +176,48 @@ class TaskNode(Node):
class WebpublisherBatchPublishEndpoint(_RestApiEndpoint):
"""Triggers headless publishing of batch."""
async def post(self, request) -> Response:
# for postprocessing in host, currently only PS
host_map = {"photoshop": [".psd", ".psb"]}
# Validate existence of openpype executable
openpype_app = self.resource.executable
if not openpype_app or not os.path.exists(openpype_app):
msg = "Non existent OpenPype executable {}".format(openpype_app)
raise RuntimeError(msg)
# for postprocessing in host, currently only PS
output = {}
log.info("WebpublisherBatchPublishEndpoint called")
content = await request.json()
batch_path = os.path.join(self.resource.upload_dir,
content["batch"])
# Each filter have extensions which are checked on first task item
# - first filter with extensions that are on first task is used
# - filter defines command and can extend arguments dictionary
# This is used only if 'studio_processing' is enabled on batch
studio_processing_filters = [
# Photoshop filter
{
"extensions": [".psd", ".psb"],
"command": "remotepublishfromapp",
"arguments": {
# Command 'remotepublishfromapp' requires --host argument
"host": "photoshop",
# Make sure targets are set to None for cases that default
# would change
# - targets argument is not used in 'remotepublishfromapp'
"targets": None
}
}
]
add_args = {
"host": "webpublisher",
"project": content["project_name"],
"user": content["user"]
}
batch_path = os.path.join(self.resource.upload_dir, content["batch"])
# Default command and arguments
command = "remotepublish"
add_args = {
# All commands need 'project' and 'user'
"project": content["project_name"],
"user": content["user"],
"targets": ["filespublish"]
}
if content.get("studio_processing"):
log.info("Post processing called")
@ -208,32 +233,34 @@ class WebpublisherBatchPublishEndpoint(_RestApiEndpoint):
raise ValueError(
"Cannot parse batch meta in {} folder".format(task_data))
command = "remotepublishfromapp"
for host, extensions in host_map.items():
for ext in extensions:
for file_name in task_data["files"]:
if ext in file_name:
add_args["host"] = host
break
for process_filter in studio_processing_filters:
filter_extensions = process_filter.get("extensions") or []
for file_name in task_data["files"]:
file_ext = os.path.splitext(file_name)[-1].lower()
if file_ext in filter_extensions:
# Change command
command = process_filter["command"]
# Update arguments
add_args.update(
process_filter.get("arguments") or {}
)
break
if not add_args.get("host"):
raise ValueError(
"Couldn't discern host from {}".format(task_data["files"]))
openpype_app = self.resource.executable
args = [
openpype_app,
command,
batch_path
]
if not openpype_app or not os.path.exists(openpype_app):
msg = "Non existent OpenPype executable {}".format(openpype_app)
raise RuntimeError(msg)
for key, value in add_args.items():
args.append("--{}".format(key))
args.append(value)
# Skip key values where value is None
if value is not None:
args.append("--{}".format(key))
# Extend list into arguments (targets can be a list)
if isinstance(value, (tuple, list)):
args.extend(value)
else:
args.append(value)
log.info("args:: {}".format(args))

37
openpype/pype_commands.py
View file

@ -137,16 +137,13 @@ class PypeCommands:
SLEEP = 5 # seconds for another loop check for concurrently runs
WAIT_FOR = 300 # seconds to wait for conc. runs
from openpype import install, uninstall
from openpype.api import Logger
from openpype.lib import ApplicationManager
log = Logger.get_logger()
log.info("remotepublishphotoshop command")
install()
from openpype.lib import ApplicationManager
application_manager = ApplicationManager()
found_variant_key = find_variant_key(application_manager, host)
@ -220,10 +217,8 @@ class PypeCommands:
while launched_app.poll() is None:
time.sleep(0.5)
uninstall()
@staticmethod
def remotepublish(project, batch_path, host, user, targets=None):
def remotepublish(project, batch_path, user, targets=None):
"""Start headless publishing.
Used to publish rendered assets, workfiles etc.
@ -235,10 +230,9 @@ class PypeCommands:
per call of remotepublish
batch_path (str): Path batch folder. Contains subfolders with
resources (workfile, another subfolder 'renders' etc.)
targets (string): What module should be targeted
(to choose validator for example)
host (string)
user (string): email address for webpublisher
targets (list): Pyblish targets
(to choose validator for example)
Raises:
RuntimeError: When there is no path to process.
@ -246,21 +240,22 @@ class PypeCommands:
if not batch_path:
raise RuntimeError("No publish paths specified")
from openpype import install, uninstall
from openpype.api import Logger
# Register target and host
import pyblish.api
import pyblish.util
import avalon.api
from openpype.hosts.webpublisher import api as webpublisher
log = Logger.get_logger()
log = PypeLogger.get_logger()
log.info("remotepublish command")
install()
host_name = "webpublisher"
os.environ["OPENPYPE_PUBLISH_DATA"] = batch_path
os.environ["AVALON_PROJECT"] = project
os.environ["AVALON_APP"] = host_name
if host:
pyblish.api.register_host(host)
pyblish.api.register_host(host_name)
if targets:
if isinstance(targets, str):
@ -268,13 +263,6 @@ class PypeCommands:
for target in targets:
pyblish.api.register_target(target)
os.environ["OPENPYPE_PUBLISH_DATA"] = batch_path
os.environ["AVALON_PROJECT"] = project
os.environ["AVALON_APP"] = host
import avalon.api
from openpype.hosts.webpublisher import api as webpublisher
avalon.api.install(webpublisher)
log.info("Running publish ...")
@ -286,7 +274,6 @@ class PypeCommands:
publish_and_log(dbcon, _id, log)
log.info("Publish finished.")
uninstall()
@staticmethod
def extractenvironments(output_json_path, project, asset, task, app):