From f7d76617c0ea5635f9ae8ad0e6a18454da24c2be Mon Sep 17 00:00:00 2001 From: Toke Jepsen Date: Tue, 7 Nov 2023 15:51:54 +0000 Subject: [PATCH] Testing: Validate Maya Logs (#5775) * Working version * Improve launched app communication * Move imports to methods. * Update tests/integration/hosts/maya/test_publish_in_maya.py Co-authored-by: Roy Nieterau * Collect errors from process * fix startup scripts arguments * Update openpype/lib/applications.py Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> * Fix application polling * Docstring * Revert stdout and stderr * Revert subprocess.PIPE * Added missed imports If we are moving these because of testing, lets move all of them --------- Co-authored-by: Roy Nieterau Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Co-authored-by: kalisp --- openpype/hosts/maya/api/lib_rendersettings.py | 6 ++-- .../collect_deadline_server_from_instance.py | 5 ++-- .../publish/submit_blender_deadline.py | 5 ++-- .../publish/submit_houdini_remote_publish.py | 3 +- .../publish/submit_houdini_render_deadline.py | 5 ++-- .../plugins/publish/submit_max_deadline.py | 14 +++++---- .../plugins/publish/submit_maya_deadline.py | 8 ++--- .../submit_maya_remote_publish_deadline.py | 5 ++-- .../plugins/publish/submit_nuke_deadline.py | 5 ++-- .../publish/collect_otio_frame_ranges.py | 18 +++++++----- .../plugins/publish/collect_otio_review.py | 7 +++-- .../publish/collect_otio_subset_resources.py | 17 ++++++----- .../publish/extract_otio_audio_tracks.py | 8 +++-- openpype/plugins/publish/extract_otio_file.py | 5 +++- .../plugins/publish/extract_otio_review.py | 24 ++++++++++----- .../publish/extract_otio_trimming_video.py | 3 +- tests/integration/hosts/maya/lib.py | 7 ++++- .../hosts/maya/test_publish_in_maya.py | 29 +++++++++++++++++++ tests/lib/testing_classes.py | 3 +- 19 files changed, 121 insertions(+), 56 deletions(-) diff --git a/openpype/hosts/maya/api/lib_rendersettings.py b/openpype/hosts/maya/api/lib_rendersettings.py index 42cf29d0a7..20264c2cdf 100644 --- a/openpype/hosts/maya/api/lib_rendersettings.py +++ b/openpype/hosts/maya/api/lib_rendersettings.py @@ -1,7 +1,5 @@ # -*- coding: utf-8 -*- """Class for handling Render Settings.""" -from maya import cmds # noqa -import maya.mel as mel import six import sys @@ -63,6 +61,10 @@ class RenderSettings(object): def set_default_renderer_settings(self, renderer=None): """Set basic settings based on renderer.""" + # Not all hosts can import this module. + from maya import cmds + import maya.mel as mel + if not renderer: renderer = cmds.getAttr( 'defaultRenderGlobals.currentRenderer').lower() diff --git a/openpype/modules/deadline/plugins/publish/collect_deadline_server_from_instance.py b/openpype/modules/deadline/plugins/publish/collect_deadline_server_from_instance.py index 9b4f89c129..1d3dad769f 100644 --- a/openpype/modules/deadline/plugins/publish/collect_deadline_server_from_instance.py +++ b/openpype/modules/deadline/plugins/publish/collect_deadline_server_from_instance.py @@ -5,8 +5,6 @@ This is resolving index of server lists stored in `deadlineServers` instance attribute or using default server if that attribute doesn't exists. """ -from maya import cmds - import pyblish.api from openpype.pipeline.publish import KnownPublishError @@ -44,7 +42,8 @@ class CollectDeadlineServerFromInstance(pyblish.api.InstancePlugin): str: Selected Deadline Webservice URL. """ - + # Not all hosts can import this module. + from maya import cmds deadline_settings = ( render_instance.context.data ["system_settings"] diff --git a/openpype/modules/deadline/plugins/publish/submit_blender_deadline.py b/openpype/modules/deadline/plugins/publish/submit_blender_deadline.py index 4a7497b075..094f2b1821 100644 --- a/openpype/modules/deadline/plugins/publish/submit_blender_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_blender_deadline.py @@ -6,8 +6,6 @@ import getpass import attr from datetime import datetime -import bpy - from openpype.lib import is_running_from_build from openpype.pipeline import legacy_io from openpype.pipeline.farm.tools import iter_expected_files @@ -142,6 +140,9 @@ class BlenderSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): return job_info def get_plugin_info(self): + # Not all hosts can import this module. + import bpy + plugin_info = BlenderPluginInfo( SceneFile=self.scene_path, Version=bpy.app.version_string, diff --git a/openpype/modules/deadline/plugins/publish/submit_houdini_remote_publish.py b/openpype/modules/deadline/plugins/publish/submit_houdini_remote_publish.py index 39c0c3afe4..0bee42c4cb 100644 --- a/openpype/modules/deadline/plugins/publish/submit_houdini_remote_publish.py +++ b/openpype/modules/deadline/plugins/publish/submit_houdini_remote_publish.py @@ -3,7 +3,6 @@ import json from datetime import datetime import requests -import hou import pyblish.api @@ -31,6 +30,8 @@ class HoudiniSubmitPublishDeadline(pyblish.api.ContextPlugin): targets = ["deadline"] def process(self, context): + # Not all hosts can import this module. + import hou # Ensure no errors so far assert all( diff --git a/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py b/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py index 6f885c578a..abc650204b 100644 --- a/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py @@ -1,9 +1,8 @@ -import hou - import os import attr import getpass from datetime import datetime + import pyblish.api from openpype.pipeline import legacy_io @@ -119,6 +118,8 @@ class HoudiniSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): return job_info def get_plugin_info(self): + # Not all hosts can import this module. + import hou instance = self._instance context = instance.context diff --git a/openpype/modules/deadline/plugins/publish/submit_max_deadline.py b/openpype/modules/deadline/plugins/publish/submit_max_deadline.py index 073da3019a..23d4183132 100644 --- a/openpype/modules/deadline/plugins/publish/submit_max_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_max_deadline.py @@ -1,8 +1,8 @@ import os import getpass import copy - import attr + from openpype.lib import ( TextDef, BoolDef, @@ -15,11 +15,6 @@ from openpype.pipeline import ( from openpype.pipeline.publish.lib import ( replace_with_published_scene_path ) -from openpype.hosts.max.api.lib import ( - get_current_renderer, - get_multipass_setting -) -from openpype.hosts.max.api.lib_rendersettings import RenderSettings from openpype_modules.deadline import abstract_submit_deadline from openpype_modules.deadline.abstract_submit_deadline import DeadlineJobInfo from openpype.lib import is_running_from_build @@ -191,6 +186,13 @@ class MaxSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline, self.submit(self.assemble_payload(job_info, plugin_info)) def _use_published_name(self, data, project_settings): + # Not all hosts can import these modules. + from openpype.hosts.max.api.lib import ( + get_current_renderer, + get_multipass_setting + ) + from openpype.hosts.max.api.lib_rendersettings import RenderSettings + instance = self._instance job_info = copy.deepcopy(self.job_info) plugin_info = copy.deepcopy(self.plugin_info) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 7775191b12..7d532923ff 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -28,8 +28,6 @@ from collections import OrderedDict import attr -from maya import cmds - from openpype.pipeline import ( legacy_io, OpenPypePyblishPluginMixin @@ -246,6 +244,8 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline, return job_info def get_plugin_info(self): + # Not all hosts can import this module. + from maya import cmds instance = self._instance context = instance.context @@ -288,7 +288,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline, return plugin_payload def process_submission(self): - + from maya import cmds instance = self._instance filepath = self.scene_path # publish if `use_publish` else workfile @@ -675,7 +675,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline, str """ - + from maya import cmds # "vrayscene//_/" vray_settings = cmds.ls(type="VRaySettingsNode") node = vray_settings[0] diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_remote_publish_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_remote_publish_deadline.py index 0d23f44333..41a2a64ab5 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_remote_publish_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_remote_publish_deadline.py @@ -2,8 +2,6 @@ import os import attr from datetime import datetime -from maya import cmds - from openpype import AYON_SERVER_ENABLED from openpype.pipeline import legacy_io, PublishXmlValidationError from openpype.tests.lib import is_in_tests @@ -127,7 +125,8 @@ class MayaSubmitRemotePublishDeadline( job_info.EnvironmentKeyValue[key] = value def get_plugin_info(self): - + # Not all hosts can import this module. + from maya import cmds scene = self._instance.context.data["currentFile"] plugin_info = MayaPluginInfo() diff --git a/openpype/modules/deadline/plugins/publish/submit_nuke_deadline.py b/openpype/modules/deadline/plugins/publish/submit_nuke_deadline.py index 0e57c54959..fb3ab2710d 100644 --- a/openpype/modules/deadline/plugins/publish/submit_nuke_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_nuke_deadline.py @@ -7,8 +7,6 @@ from datetime import datetime import requests import pyblish.api -import nuke - from openpype import AYON_SERVER_ENABLED from openpype.pipeline import legacy_io from openpype.pipeline.publish import ( @@ -498,6 +496,9 @@ class NukeSubmitDeadline(pyblish.api.InstancePlugin, Returning: list: captured groups list """ + # Not all hosts can import this module. + import nuke + captured_groups = [] for lg_name, list_node_class in self.limit_groups.items(): for node_class in list_node_class: diff --git a/openpype/plugins/publish/collect_otio_frame_ranges.py b/openpype/plugins/publish/collect_otio_frame_ranges.py index 9a68b6e43d..4b130b0e03 100644 --- a/openpype/plugins/publish/collect_otio_frame_ranges.py +++ b/openpype/plugins/publish/collect_otio_frame_ranges.py @@ -5,15 +5,9 @@ Requires: masterLayer -> instance data attribute otioClipRange -> instance data attribute """ -# import os -import opentimelineio as otio -import pyblish.api from pprint import pformat -from openpype.pipeline.editorial import ( - get_media_range_with_retimes, - otio_range_to_frame_range, - otio_range_with_handles -) + +import pyblish.api class CollectOtioFrameRanges(pyblish.api.InstancePlugin): @@ -27,6 +21,14 @@ class CollectOtioFrameRanges(pyblish.api.InstancePlugin): hosts = ["resolve", "hiero", "flame", "traypublisher"] def process(self, instance): + # Not all hosts can import these modules. + import opentimelineio as otio + from openpype.pipeline.editorial import ( + get_media_range_with_retimes, + otio_range_to_frame_range, + otio_range_with_handles + ) + # get basic variables otio_clip = instance.data["otioClip"] workfile_start = instance.data["workfileFrameStart"] diff --git a/openpype/plugins/publish/collect_otio_review.py b/openpype/plugins/publish/collect_otio_review.py index f0157282a1..0e4d596213 100644 --- a/openpype/plugins/publish/collect_otio_review.py +++ b/openpype/plugins/publish/collect_otio_review.py @@ -11,10 +11,10 @@ Provides: instance -> families (adding ["review", "ftrack"]) """ -import opentimelineio as otio -import pyblish.api from pprint import pformat +import pyblish.api + class CollectOtioReview(pyblish.api.InstancePlugin): """Get matching otio track from defined review layer""" @@ -25,6 +25,9 @@ class CollectOtioReview(pyblish.api.InstancePlugin): hosts = ["resolve", "hiero", "flame"] def process(self, instance): + # Not all hosts can import this module. + import opentimelineio as otio + # get basic variables otio_review_clips = [] otio_timeline = instance.context.data["otioTimeline"] diff --git a/openpype/plugins/publish/collect_otio_subset_resources.py b/openpype/plugins/publish/collect_otio_subset_resources.py index f659791d95..739f5bb726 100644 --- a/openpype/plugins/publish/collect_otio_subset_resources.py +++ b/openpype/plugins/publish/collect_otio_subset_resources.py @@ -6,18 +6,15 @@ Provides: instance -> otioReviewClips """ import os + import clique -import opentimelineio as otio import pyblish.api -from openpype.pipeline.editorial import ( - get_media_range_with_retimes, - range_from_frames, - make_sequence_collection -) + from openpype.pipeline.publish import ( get_publish_template_name ) + class CollectOtioSubsetResources(pyblish.api.InstancePlugin): """Get Resources for a subset version""" @@ -26,8 +23,14 @@ class CollectOtioSubsetResources(pyblish.api.InstancePlugin): families = ["clip"] hosts = ["resolve", "hiero", "flame"] - def process(self, instance): + # Not all hosts can import these modules. + import opentimelineio as otio + from openpype.pipeline.editorial import ( + get_media_range_with_retimes, + range_from_frames, + make_sequence_collection + ) if "audio" in instance.data["family"]: return diff --git a/openpype/plugins/publish/extract_otio_audio_tracks.py b/openpype/plugins/publish/extract_otio_audio_tracks.py index 4f17731452..4b73321f02 100644 --- a/openpype/plugins/publish/extract_otio_audio_tracks.py +++ b/openpype/plugins/publish/extract_otio_audio_tracks.py @@ -1,11 +1,12 @@ import os +import tempfile + import pyblish + from openpype.lib import ( get_ffmpeg_tool_args, run_subprocess ) -import tempfile -import opentimelineio as otio class ExtractOtioAudioTracks(pyblish.api.ContextPlugin): @@ -155,6 +156,9 @@ class ExtractOtioAudioTracks(pyblish.api.ContextPlugin): Returns: list: list of audio clip dictionaries """ + # Not all hosts can import this module. + import opentimelineio as otio + output = [] # go trough all audio tracks for otio_track in otio_timeline.tracks: diff --git a/openpype/plugins/publish/extract_otio_file.py b/openpype/plugins/publish/extract_otio_file.py index 1a6a82117d..7f1cac33d7 100644 --- a/openpype/plugins/publish/extract_otio_file.py +++ b/openpype/plugins/publish/extract_otio_file.py @@ -1,6 +1,6 @@ import os + import pyblish.api -import opentimelineio as otio from openpype.pipeline import publish @@ -16,6 +16,9 @@ class ExtractOTIOFile(publish.Extractor): hosts = ["resolve", "hiero", "traypublisher"] def process(self, instance): + # Not all hosts can import this module. + import opentimelineio as otio + if not instance.context.data.get("otioTimeline"): return # create representation data diff --git a/openpype/plugins/publish/extract_otio_review.py b/openpype/plugins/publish/extract_otio_review.py index 699207df8a..ad4c807091 100644 --- a/openpype/plugins/publish/extract_otio_review.py +++ b/openpype/plugins/publish/extract_otio_review.py @@ -15,8 +15,8 @@ Provides: """ import os + import clique -import opentimelineio as otio from pyblish import api from openpype.lib import ( @@ -24,13 +24,6 @@ from openpype.lib import ( run_subprocess, ) from openpype.pipeline import publish -from openpype.pipeline.editorial import ( - otio_range_to_frame_range, - trim_media_range, - range_from_frames, - frames_to_seconds, - make_sequence_collection -) class ExtractOTIOReview(publish.Extractor): @@ -62,6 +55,13 @@ class ExtractOTIOReview(publish.Extractor): output_ext = ".jpg" def process(self, instance): + # Not all hosts can import these modules. + import opentimelineio as otio + from openpype.pipeline.editorial import ( + otio_range_to_frame_range, + make_sequence_collection + ) + # TODO: convert resulting image sequence to mp4 # get otio clip and other time info from instance clip @@ -281,6 +281,12 @@ class ExtractOTIOReview(publish.Extractor): Returns: otio.time.TimeRange: trimmed available range """ + # Not all hosts can import these modules. + from openpype.pipeline.editorial import ( + trim_media_range, + range_from_frames + ) + avl_start = int(avl_range.start_time.value) src_start = int(avl_start + start) avl_durtation = int(avl_range.duration.value) @@ -338,6 +344,8 @@ class ExtractOTIOReview(publish.Extractor): Returns: otio.time.TimeRange: trimmed available range """ + # Not all hosts can import this module. + from openpype.pipeline.editorial import frames_to_seconds # create path and frame start to destination output_path, out_frame_start = self._get_ffmpeg_output() diff --git a/openpype/plugins/publish/extract_otio_trimming_video.py b/openpype/plugins/publish/extract_otio_trimming_video.py index 67ff6c538c..2020fcde93 100644 --- a/openpype/plugins/publish/extract_otio_trimming_video.py +++ b/openpype/plugins/publish/extract_otio_trimming_video.py @@ -15,7 +15,6 @@ from openpype.lib import ( run_subprocess, ) from openpype.pipeline import publish -from openpype.pipeline.editorial import frames_to_seconds class ExtractOTIOTrimmingVideo(publish.Extractor): @@ -75,6 +74,8 @@ class ExtractOTIOTrimmingVideo(publish.Extractor): otio_range (opentime.TimeRange): range to trim to """ + # Not all hosts can import this module. + from openpype.pipeline.editorial import frames_to_seconds # create path to destination output_path = self._get_ffmpeg_output(input_file_path) diff --git a/tests/integration/hosts/maya/lib.py b/tests/integration/hosts/maya/lib.py index f27d516605..04ddb765a4 100644 --- a/tests/integration/hosts/maya/lib.py +++ b/tests/integration/hosts/maya/lib.py @@ -33,7 +33,7 @@ class MayaHostFixtures(HostFixtures): yield dest_path @pytest.fixture(scope="module") - def startup_scripts(self, monkeypatch_session): + def startup_scripts(self, monkeypatch_session, download_test_data): """Points Maya to userSetup file from input data""" startup_path = os.path.join( os.path.dirname(__file__), "input", "startup" @@ -44,6 +44,11 @@ class MayaHostFixtures(HostFixtures): "{}{}{}".format(startup_path, os.pathsep, original_pythonpath) ) + monkeypatch_session.setenv( + "MAYA_CMD_FILE_OUTPUT", + os.path.join(download_test_data, "output.log") + ) + @pytest.fixture(scope="module") def skip_compare_folders(self): yield [] diff --git a/tests/integration/hosts/maya/test_publish_in_maya.py b/tests/integration/hosts/maya/test_publish_in_maya.py index b7ee228aae..be8c74e0b8 100644 --- a/tests/integration/hosts/maya/test_publish_in_maya.py +++ b/tests/integration/hosts/maya/test_publish_in_maya.py @@ -1,3 +1,6 @@ +import re +import os + from tests.lib.assert_classes import DBAssert from tests.integration.hosts.maya.lib import MayaLocalPublishTestClass @@ -35,6 +38,32 @@ class TestPublishInMaya(MayaLocalPublishTestClass): TIMEOUT = 120 # publish timeout + def test_publish( + self, + dbcon, + publish_finished, + download_test_data + ): + """Testing Pyblish and Python logs within Maya.""" + + # All maya output via MAYA_CMD_FILE_OUTPUT env var during test run + logging_path = os.path.join(download_test_data, "output.log") + with open(logging_path, "r") as f: + logging_output = f.read() + + print(("-" * 50) + "LOGGING" + ("-" * 50)) + print(logging_output) + + # Check for pyblish errors. + error_regex = r"pyblish \(ERROR\)((.|\n)*?)((pyblish \())" + matches = re.findall(error_regex, logging_output) + assert not matches, matches[0][0] + + # Check for python errors. + error_regex = r"// Error((.|\n)*)" + matches = re.findall(error_regex, logging_output) + assert not matches, matches[0][0] + def test_db_asserts(self, dbcon, publish_finished): """Host and input data dependent expected results in DB.""" print("test_db_asserts") diff --git a/tests/lib/testing_classes.py b/tests/lib/testing_classes.py index e8e338e434..3b0611e2a0 100644 --- a/tests/lib/testing_classes.py +++ b/tests/lib/testing_classes.py @@ -10,6 +10,7 @@ import glob import platform import requests import re +import time from tests.lib.db_handler import DBHandler from tests.lib.file_handler import RemoteFileHandler @@ -334,7 +335,7 @@ class PublishTest(ModuleUnitTest): print("Creating only setup for test, not launching app") yield False return - import time + time_start = time.time() timeout = timeout or self.TIMEOUT timeout = float(timeout)