diff --git a/openpype/lib/file_transaction.py b/openpype/lib/file_transaction.py index f265b8815c..cba361a8d4 100644 --- a/openpype/lib/file_transaction.py +++ b/openpype/lib/file_transaction.py @@ -92,7 +92,9 @@ class FileTransaction(object): def process(self): # Backup any existing files for dst, (src, _) in self._transfers.items(): - if dst == src or not os.path.exists(dst): + self.log.debug("Checking file ... {} -> {}".format(src, dst)) + path_same = self._same_paths(src, dst) + if path_same or not os.path.exists(dst): continue # Backup original file @@ -105,7 +107,8 @@ class FileTransaction(object): # Copy the files to transfer for dst, (src, opts) in self._transfers.items(): - if dst == src: + path_same = self._same_paths(src, dst) + if path_same: self.log.debug( "Source and destionation are same files {} -> {}".format( src, dst)) @@ -182,3 +185,10 @@ class FileTransaction(object): else: self.log.critical("An unexpected error occurred.") six.reraise(*sys.exc_info()) + + def _same_paths(self, src, dst): + # handles same paths but with C:/project vs c:/project + if os.path.exists(src) and os.path.exists(dst): + return os.path.samefile(src, dst) + + return src == dst diff --git a/openpype/pipeline/anatomy.py b/openpype/pipeline/anatomy.py index 627f7198bb..72f30d85cb 100644 --- a/openpype/pipeline/anatomy.py +++ b/openpype/pipeline/anatomy.py @@ -1112,17 +1112,21 @@ class RootItem(FormatObject): result = False output = str(path) - root_paths = list(self.cleaned_data.values()) mod_path = self.clean_path(path) - for root_path in root_paths: + for root_os, root_path in self.cleaned_data.items(): # Skip empty paths if not root_path: continue - if mod_path.startswith(root_path): + _mod_path = mod_path # reset to original cleaned value + if root_os == "windows": + root_path = root_path.lower() + _mod_path = _mod_path.lower() + + if _mod_path.startswith(root_path): result = True replacement = "{" + self.full_key() + "}" - output = replacement + mod_path[len(root_path):] + output = replacement + _mod_path[len(root_path):] break return (result, output) diff --git a/openpype/plugins/publish/collect_resources_path.py b/openpype/plugins/publish/collect_resources_path.py index a2d5b95ab2..dcd80fbbdf 100644 --- a/openpype/plugins/publish/collect_resources_path.py +++ b/openpype/plugins/publish/collect_resources_path.py @@ -15,7 +15,11 @@ import pyblish.api class CollectResourcesPath(pyblish.api.InstancePlugin): - """Generate directory path where the files and resources will be stored""" + """Generate directory path where the files and resources will be stored. + + Collects folder name and file name from files, if exists, for in-situ + publishing. + """ label = "Collect Resources Path" order = pyblish.api.CollectorOrder + 0.495 diff --git a/openpype/plugins/publish/collect_source_for_source.py b/openpype/plugins/publish/collect_source_for_source.py new file mode 100644 index 0000000000..aa94238b4f --- /dev/null +++ b/openpype/plugins/publish/collect_source_for_source.py @@ -0,0 +1,42 @@ +""" +Requires: + instance -> currentFile + instance -> source + +Provides: + instance -> originalBasename + instance -> originalDirname +""" + +import os + +import pyblish.api + + +class CollectSourceForSource(pyblish.api.InstancePlugin): + """Collects source location of file for instance. + + Used for 'source' template name which handles in place publishing. + For this kind of publishing files are present with correct file name + pattern and correct location. + """ + + label = "Collect Source" + order = pyblish.api.CollectorOrder + 0.495 + + def process(self, instance): + # parse folder name and file name for online and source templates + # currentFile comes from hosts workfiles + # source comes from Publisher + current_file = instance.data.get("currentFile") + source = instance.data.get("source") + source_file = current_file or source + if source_file: + self.log.debug("Parsing paths for {}".format(source_file)) + if not instance.data.get("originalBasename"): + instance.data["originalBasename"] = \ + os.path.basename(source_file) + + if not instance.data.get("originalDirname"): + instance.data["originalDirname"] = \ + os.path.dirname(source_file) diff --git a/openpype/plugins/publish/extract_thumbnail.py b/openpype/plugins/publish/extract_thumbnail.py index 14b43beae8..a3c428fe97 100644 --- a/openpype/plugins/publish/extract_thumbnail.py +++ b/openpype/plugins/publish/extract_thumbnail.py @@ -91,7 +91,7 @@ class ExtractThumbnail(pyblish.api.InstancePlugin): full_input_path = os.path.join(src_staging, input_file) self.log.info("input {}".format(full_input_path)) filename = os.path.splitext(input_file)[0] - jpeg_file = filename + ".jpg" + jpeg_file = filename + "_thumb.jpg" full_output_path = os.path.join(dst_staging, jpeg_file) if oiio_supported: diff --git a/openpype/plugins/publish/extract_thumbnail_from_source.py b/openpype/plugins/publish/extract_thumbnail_from_source.py index 03df1455e2..a92f762cde 100644 --- a/openpype/plugins/publish/extract_thumbnail_from_source.py +++ b/openpype/plugins/publish/extract_thumbnail_from_source.py @@ -100,7 +100,7 @@ class ExtractThumbnailFromSource(pyblish.api.InstancePlugin): self.log.info("Thumbnail source: {}".format(thumbnail_source)) src_basename = os.path.basename(thumbnail_source) - dst_filename = os.path.splitext(src_basename)[0] + ".jpg" + dst_filename = os.path.splitext(src_basename)[0] + "_thumb.jpg" full_output_path = os.path.join(dst_staging, dst_filename) if oiio_supported: diff --git a/openpype/plugins/publish/help/validate_publish_dir.xml b/openpype/plugins/publish/help/validate_publish_dir.xml new file mode 100644 index 0000000000..9f62b264bf --- /dev/null +++ b/openpype/plugins/publish/help/validate_publish_dir.xml @@ -0,0 +1,31 @@ + + + +Source directory not collected + +## Source directory not collected + +Instance is marked for in place publishing. Its 'originalDirname' must be collected. Contact OP developer to modify collector. + + + +### __Detailed Info__ (optional) + +In place publishing uses source directory and file name in resulting path and file name of published item. For this instance + all required metadata weren't filled. This is not recoverable error, unless instance itself is removed. + Collector for this instance must be updated for instance to be published. + + + +Source file not in project dir + +## Source file not in project dir + +Path '{original_dirname}' not in project folder. Please publish from inside of project folder. + +### How to repair? + +Restart publish after you moved source file into project directory. + + + \ No newline at end of file diff --git a/openpype/plugins/publish/integrate.py b/openpype/plugins/publish/integrate.py index 457e2b4fe2..5f811ce002 100644 --- a/openpype/plugins/publish/integrate.py +++ b/openpype/plugins/publish/integrate.py @@ -270,6 +270,8 @@ class IntegrateAsset(pyblish.api.InstancePlugin): ) instance.data["versionEntity"] = version + anatomy = instance.context.data["anatomy"] + # Get existing representations (if any) existing_repres_by_name = { repre_doc["name"].lower(): repre_doc @@ -303,13 +305,17 @@ class IntegrateAsset(pyblish.api.InstancePlugin): # .ma representation. Those destination paths are pre-defined, etc. # todo: should we move or simplify this logic? resource_destinations = set() - for src, dst in instance.data.get("transfers", []): - file_transactions.add(src, dst, mode=FileTransaction.MODE_COPY) - resource_destinations.add(os.path.abspath(dst)) - for src, dst in instance.data.get("hardlinks", []): - file_transactions.add(src, dst, mode=FileTransaction.MODE_HARDLINK) - resource_destinations.add(os.path.abspath(dst)) + file_copy_modes = [ + ("transfers", FileTransaction.MODE_COPY), + ("hardlinks", FileTransaction.MODE_HARDLINK) + ] + for files_type, copy_mode in file_copy_modes: + for src, dst in instance.data.get(files_type, []): + self._validate_path_in_project_roots(anatomy, dst) + + file_transactions.add(src, dst, mode=copy_mode) + resource_destinations.add(os.path.abspath(dst)) # Bulk write to the database # We write the subset and version to the database before the File @@ -342,7 +348,6 @@ class IntegrateAsset(pyblish.api.InstancePlugin): # Compute the resource file infos once (files belonging to the # version instance instead of an individual representation) so # we can re-use those file infos per representation - anatomy = instance.context.data["anatomy"] resource_file_infos = self.get_files_info(resource_destinations, sites=sites, anatomy=anatomy) @@ -529,6 +534,20 @@ class IntegrateAsset(pyblish.api.InstancePlugin): template_data["representation"] = repre["name"] template_data["ext"] = repre["ext"] + stagingdir = repre.get("stagingDir") + if not stagingdir: + # Fall back to instance staging dir if not explicitly + # set for representation in the instance + self.log.debug(( + "Representation uses instance staging dir: {}" + ).format(instance_stagingdir)) + stagingdir = instance_stagingdir + + if not stagingdir: + raise KnownPublishError( + "No staging directory set for representation: {}".format(repre) + ) + # optionals # retrieve additional anatomy data from representation if exists for key, anatomy_key in { @@ -548,20 +567,6 @@ class IntegrateAsset(pyblish.api.InstancePlugin): if value is not None: template_data[anatomy_key] = value - stagingdir = repre.get("stagingDir") - if not stagingdir: - # Fall back to instance staging dir if not explicitly - # set for representation in the instance - self.log.debug(( - "Representation uses instance staging dir: {}" - ).format(instance_stagingdir)) - stagingdir = instance_stagingdir - - if not stagingdir: - raise KnownPublishError( - "No staging directory set for representation: {}".format(repre) - ) - self.log.debug("Anatomy template name: {}".format(template_name)) anatomy = instance.context.data["anatomy"] publish_template_category = anatomy.templates[template_name] @@ -569,6 +574,25 @@ class IntegrateAsset(pyblish.api.InstancePlugin): is_udim = bool(repre.get("udim")) + # handle publish in place + if "originalDirname" in template: + # store as originalDirname only original value without project root + # if instance collected originalDirname is present, it should be + # used for all represe + # from temp to final + original_directory = ( + instance.data.get("originalDirname") or instance_stagingdir) + + _rootless = self.get_rootless_path(anatomy, original_directory) + if _rootless == original_directory: + raise KnownPublishError(( + "Destination path '{}' ".format(original_directory) + + "must be in project dir" + )) + relative_path_start = _rootless.rfind('}') + 2 + without_root = _rootless[relative_path_start:] + template_data["originalDirname"] = without_root + is_sequence_representation = isinstance(files, (list, tuple)) if is_sequence_representation: # Collection of files (sequence) @@ -587,6 +611,7 @@ class IntegrateAsset(pyblish.api.InstancePlugin): )) src_collection = src_collections[0] + template_data["originalBasename"] = src_collection.head[:-1] destination_indexes = list(src_collection.indexes) # Use last frame for minimum padding # - that should cover both 'udim' and 'frame' minimum padding @@ -671,12 +696,11 @@ class IntegrateAsset(pyblish.api.InstancePlugin): raise KnownPublishError( "This is a bug. Representation file name is full path" ) - + template_data["originalBasename"], _ = os.path.splitext(fname) # Manage anatomy template data template_data.pop("frame", None) if is_udim: template_data["udim"] = repre["udim"][0] - # Construct destination filepath from template anatomy_filled = anatomy.format(template_data) template_filled = anatomy_filled[template_name]["path"] @@ -890,3 +914,21 @@ class IntegrateAsset(pyblish.api.InstancePlugin): "hash": source_hash(path), "sites": sites } + + def _validate_path_in_project_roots(self, anatomy, file_path): + """Checks if 'file_path' starts with any of the roots. + + Used to check that published path belongs to project, eg. we are not + trying to publish to local only folder. + Args: + anatomy (Anatomy) + file_path (str) + Raises + (KnownPublishError) + """ + path = self.get_rootless_path(anatomy, file_path) + if not path: + raise KnownPublishError(( + "Destination path '{}' ".format(file_path) + + "must be in project dir" + )) diff --git a/openpype/plugins/publish/validate_publish_dir.py b/openpype/plugins/publish/validate_publish_dir.py new file mode 100644 index 0000000000..2f41127548 --- /dev/null +++ b/openpype/plugins/publish/validate_publish_dir.py @@ -0,0 +1,74 @@ +import pyblish.api +from openpype.pipeline.publish import ValidateContentsOrder +from openpype.pipeline.publish import ( + PublishXmlValidationError, + get_publish_template_name, +) + + +class ValidatePublishDir(pyblish.api.InstancePlugin): + """Validates if 'publishDir' is a project directory + + 'publishDir' is collected based on publish templates. In specific cases + ('source' template) source folder of items is used as a 'publishDir', this + validates if it is inside any project dir for the project. + (eg. files are not published from local folder, unaccessible for studio' + + """ + + order = ValidateContentsOrder + label = "Validate publish dir" + + checked_template_names = ["source"] + # validate instances might have interim family, needs to be mapped to final + family_mapping = { + "renderLayer": "render", + "renderLocal": "render" + } + + def process(self, instance): + + template_name = self._get_template_name_from_instance(instance) + + if template_name not in self.checked_template_names: + return + + original_dirname = instance.data.get("originalDirname") + if not original_dirname: + raise PublishXmlValidationError( + self, + "Instance meant for in place publishing." + " Its 'originalDirname' must be collected." + " Contact OP developer to modify collector." + ) + + anatomy = instance.context.data["anatomy"] + + success, _ = anatomy.find_root_template_from_path(original_dirname) + + formatting_data = { + "original_dirname": original_dirname, + } + msg = "Path '{}' not in project folder.".format(original_dirname) + \ + " Please publish from inside of project folder." + if not success: + raise PublishXmlValidationError(self, msg, key="not_in_dir", + formatting_data=formatting_data) + + def _get_template_name_from_instance(self, instance): + project_name = instance.context.data["projectName"] + host_name = instance.context.data["hostName"] + anatomy_data = instance.data["anatomyData"] + family = anatomy_data["family"] + family = self.family_mapping.get("family") or family + task_info = anatomy_data.get("task") or {} + + return get_publish_template_name( + project_name, + host_name, + family, + task_name=task_info.get("name"), + task_type=task_info.get("type"), + project_settings=instance.context.data["project_settings"], + logger=self.log + ) diff --git a/openpype/settings/defaults/project_anatomy/templates.json b/openpype/settings/defaults/project_anatomy/templates.json index 0ac56a4dad..32230e0625 100644 --- a/openpype/settings/defaults/project_anatomy/templates.json +++ b/openpype/settings/defaults/project_anatomy/templates.json @@ -53,11 +53,17 @@ "file": "{originalBasename}<.{@frame}><_{udim}>.{ext}", "path": "{@folder}/{@file}" }, + "source": { + "folder": "{root[work]}/{originalDirname}", + "file": "{originalBasename}<.{@frame}><_{udim}>.{ext}", + "path": "{@folder}/{@file}" + }, "__dynamic_keys_labels__": { "maya2unreal": "Maya to Unreal", "simpleUnrealTextureHero": "Simple Unreal Texture - Hero", "simpleUnrealTexture": "Simple Unreal Texture", - "online": "online" + "online": "online", + "source": "source" } } } \ No newline at end of file