Merge pull request #4398 from ynput/feature/OP-3663_Hiero-export-of-large-number-of-shots-fill-up-disk

openpype/hosts/flame/plugins/publish/extract_subset_resources.py

@@ -143,6 +143,9 @@ class ExtractSubsetResources(publish.Extractor):
         # create staging dir path
         staging_dir = self.staging_dir(instance)
 
+        # append staging dir for later cleanup
+        instance.context.data["cleanupFullPaths"].append(staging_dir)
+
         # add default preset type for thumbnail and reviewable video
         # update them with settings and override in case the same
         # are found in there
@@ -548,30 +551,3 @@ class ExtractSubsetResources(publish.Extractor):
                 "Path `{}` is containing more that one clip".format(path)
             )
         return clips[0]
-
-    def staging_dir(self, instance):
-        """Provide a temporary directory in which to store extracted files
-
-        Upon calling this method the staging directory is stored inside
-        the instance.data['stagingDir']
-        """
-        staging_dir = instance.data.get('stagingDir', None)
-        openpype_temp_dir = os.getenv("OPENPYPE_TEMP_DIR")
-
-        if not staging_dir:
-            if openpype_temp_dir and os.path.exists(openpype_temp_dir):
-                staging_dir = os.path.normpath(
-                    tempfile.mkdtemp(
-                        prefix="pyblish_tmp_",
-                        dir=openpype_temp_dir
-                    )
-                )
-            else:
-                staging_dir = os.path.normpath(
-                    tempfile.mkdtemp(prefix="pyblish_tmp_")
-                )
-            instance.data['stagingDir'] = staging_dir
-
-        instance.context.data["cleanupFullPaths"].append(staging_dir)
-
-        return staging_dir

openpype/pipeline/publish/lib.py

@@ -10,11 +10,17 @@ import six
 import pyblish.plugin
 import pyblish.api
 
-from openpype.lib import Logger, filter_profiles
+from openpype.lib import (
+    Logger,
+    filter_profiles
+)
 from openpype.settings import (
     get_project_settings,
     get_system_settings,
 )
+from openpype.pipeline import (
+    tempdir
+)
 
 from .contants import (
     DEFAULT_PUBLISH_TEMPLATE,
@@ -595,7 +601,7 @@ def context_plugin_should_run(plugin, context):
 
     Args:
         plugin (pyblish.api.Plugin): Plugin with filters.
-        context (pyblish.api.Context): Pyblish context with insances.
+        context (pyblish.api.Context): Pyblish context with instances.
 
     Returns:
         bool: Context plugin should run based on valid instances.
@@ -609,12 +615,21 @@ def context_plugin_should_run(plugin, context):
 def get_instance_staging_dir(instance):
     """Unified way how staging dir is stored and created on instances.
 
-    First check if 'stagingDir' is already set in instance data. If there is
-    not create new in tempdir.
+    First check if 'stagingDir' is already set in instance data.
+    In case there already is new tempdir will not be created.
+
+    It also supports `OPENPYPE_TMPDIR`, so studio can define own temp
+    shared repository per project or even per more granular context.
+    Template formatting is supported also with optional keys. Folder is
+    created in case it doesn't exists.
+
+    Available anatomy formatting keys:
+        - root[work | <root name key>]
+        - project[name | code]
 
     Note:
-        Staging dir does not have to be necessarily in tempdir so be carefull
-            about it's usage.
+        Staging dir does not have to be necessarily in tempdir so be careful
+        about its usage.
 
     Args:
         instance (pyblish.lib.Instance): Instance for which we want to get
@@ -623,12 +638,27 @@ def get_instance_staging_dir(instance):
     Returns:
         str: Path to staging dir of instance.
     """
+    staging_dir = instance.data.get('stagingDir')
+    if staging_dir:
+        return staging_dir
 
-    staging_dir = instance.data.get("stagingDir")
-    if not staging_dir:
+    anatomy = instance.context.data.get("anatomy")
+
+    # get customized tempdir path from `OPENPYPE_TMPDIR` env var
+    custom_temp_dir = tempdir.create_custom_tempdir(
+        anatomy.project_name, anatomy)
+
+    if custom_temp_dir:
+        staging_dir = os.path.normpath(
+            tempfile.mkdtemp(
+                prefix="pyblish_tmp_",
+                dir=custom_temp_dir
+            )
+        )
+    else:
         staging_dir = os.path.normpath(
             tempfile.mkdtemp(prefix="pyblish_tmp_")
         )
-        instance.data["stagingDir"] = staging_dir
+    instance.data['stagingDir'] = staging_dir
 
     return staging_dir

openpype/pipeline/tempdir.py

@@ -0,0 +1,59 @@
+"""
+Temporary folder operations
+"""
+
+import os
+from openpype.lib import StringTemplate
+from openpype.pipeline import Anatomy
+
+
+def create_custom_tempdir(project_name, anatomy=None):
+    """ Create custom tempdir
+
+    Template path formatting is supporting:
+    - optional key formatting
+    - available keys:
+        - root[work | <root name key>]
+        - project[name | code]
+
+    Args:
+        project_name (str): project name
+        anatomy (openpype.pipeline.Anatomy)[optional]: Anatomy object
+
+    Returns:
+        str | None: formatted path or None
+    """
+    openpype_tempdir = os.getenv("OPENPYPE_TMPDIR")
+    if not openpype_tempdir:
+        return
+
+    custom_tempdir = None
+    if "{" in openpype_tempdir:
+        if anatomy is None:
+            anatomy = Anatomy(project_name)
+        # create base formate data
+        data = {
+            "root": anatomy.roots,
+            "project": {
+                "name": anatomy.project_name,
+                "code": anatomy.project_code,
+            }
+        }
+        # path is anatomy template
+        custom_tempdir = StringTemplate.format_template(
+            openpype_tempdir, data).normalized()
+
+    else:
+        # path is absolute
+        custom_tempdir = openpype_tempdir
+
+    # create the dir path if it doesn't exists
+    if not os.path.exists(custom_tempdir):
+        try:
+            # create it if it doesn't exists
+            os.makedirs(custom_tempdir)
+        except IOError as error:
+            raise IOError(
+                "Path couldn't be created: {}".format(error)) from error
+
+    return custom_tempdir

website/docs/admin_environment.md

@@ -0,0 +1,30 @@
+---
+id: admin_environment
+title: Environment
+sidebar_label: Environment
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+## OPENPYPE_TMPDIR:
+ - Custom staging dir directory
+ - Supports anatomy keys formatting. ex `{root[work]}/{project[name]}/temp`
+ - supported formatting keys:
+    - root[work]
+    - project[name | code]
+
+## OPENPYPE_DEBUG
+ - setting logger to debug mode
+ - example value: "1" (to activate)
+
+## OPENPYPE_LOG_LEVEL
+ - stringified numeric value of log level. [Here for more info](https://docs.python.org/3/library/logging.html#logging-levels)
+ - example value: "10"
+
+## OPENPYPE_MONGO
+- If set it takes precedence over the one set in keyring
+- for more details on how to use it go [here](admin_use#check-for-mongodb-database-connection)
+
+## OPENPYPE_USERNAME
+- if set it overides system created username

website/docs/admin_settings_system.md

@@ -13,18 +13,23 @@ Settings applicable to the full studio.
 
 ![general_settings](assets/settings/settings_system_general.png)
 
-**`Studio Name`** - Full name of the studio (can be used as variable on some places)
+### Studio Name
+Full name of the studio (can be used as variable on some places)
 
-**`Studio Code`** - Studio acronym or a short code (can be used as variable on some places)
+### Studio Code
+Studio acronym or a short code (can be used as variable on some places)
 
-**`Admin Password`** - After setting admin password, normal user won't have access to OpenPype settings
+### Admin Password
+After setting admin password, normal user won't have access to OpenPype settings
 and Project Manager GUI. Please keep in mind that this is a studio wide password and it is meant purely
 as a simple barrier to prevent artists from accidental setting changes.
 
-**`Environment`** - Globally applied environment variables that will be appended to any OpenPype process in the studio.
+### Environment
+Globally applied environment variables that will be appended to any OpenPype process in the studio.
 
-**`Disk mapping`** - Platform dependent configuration for mapping of virtual disk(s) on an artist's OpenPype machines before OP starts up.
-Uses `subst` command, if configured volume character in `Destination` field already exists, no re-mapping is done for that character(volume).
+### Disk mapping
+- Platform dependent configuration for mapping of virtual disk(s) on an artist's OpenPype machines before OP starts up.
+- Uses `subst` command, if configured volume character in `Destination` field already exists, no re-mapping is done for that character(volume).
 
 ### FFmpeg and OpenImageIO tools
 We bundle FFmpeg tools for all platforms and OpenImageIO tools for Windows and Linux. By default, bundled tools are used, but it is possible to set environment variables `OPENPYPE_FFMPEG_PATHS` and `OPENPYPE_OIIO_PATHS` in system settings environments to look for them in different directory.
@@ -171,4 +176,4 @@ In the image before you can see that we set most of the environment variables in
 In this example MTOA will automatically will the `MAYA_VERSION`(which is set by Maya Application environment) and `MTOA_VERSION` into the `MTOA` variable. We then use the `MTOA` to set all the other variables needed for it to function within Maya.
 ![tools](assets/settings/tools_01.png)
 
-All of the tools defined in here can then be assigned to projects. You can also change the tools versions on any project level all the way down to individual asset or shot overrides. So if you just need to upgrade you render plugin for a single shot, while not risking the incompatibilities on the rest of the project, it is possible.
+All the tools defined in here can then be assigned to projects. You can also change the tools versions on any project level all the way down to individual asset or shot overrides. So if you just need to upgrade you render plugin for a single shot, while not risking the incompatibilities on the rest of the project, it is possible.

website/sidebars.js

@@ -86,6 +86,7 @@ module.exports = {
             type: "category",
             label: "Configuration",
             items: [
+                "admin_environment",
                 "admin_settings",
                 "admin_settings_system",
                 "admin_settings_project_anatomy",