From 35e0b043e1dfeb259128e84a954f2e7df2879a71 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Tue, 22 Mar 2022 14:49:15 +0100 Subject: [PATCH] added few docstrings --- openpype/tools/workfiles/lib.py | 62 ++++++++++++++++++++++++++++++- openpype/tools/workfiles/model.py | 43 +++++++++++++++++++++ 2 files changed, 103 insertions(+), 2 deletions(-) diff --git a/openpype/tools/workfiles/lib.py b/openpype/tools/workfiles/lib.py index 84f2e76450..b9a1f5b19b 100644 --- a/openpype/tools/workfiles/lib.py +++ b/openpype/tools/workfiles/lib.py @@ -10,7 +10,7 @@ import appdirs class TempPublishFilesItem(object): - """Object representing on subfolder in app temp files. + """Object representing copied workfile in app temp folfer. Args: item_id (str): Id of item used as subfolder. @@ -44,7 +44,39 @@ class TempPublishFilesItem(object): class TempPublishFiles(object): - """Directory where """ + """Directory where published workfiles are copied when opened. + + Directory is located in appdirs on the machine. Folder contains file + with metadata about stored files. Each item in metadata has id, filename + and expiration time. When expiration time is higher then current time the + item is removed from metadata and it's files are deleted. Files of items + are stored in subfolder named by item's id. + + Metadata file can be in theory opened and modified by multiple processes, + threads at one time. For those cases is created simple lock file which + is created before modification begins and is removed when modification + ends. Existince of the file means that it should not be modified by + any other process at the same time. + + Metadata example: + ``` + { + "96050b4a-8974-4fca-8179-7c446c478d54": { + "created": 1647880725.555, + "expiration": 1647884325.555, + "filename": "cg_pigeon_workfileModeling_v025.ma" + }, + ... + } + ``` + + ## Why is this needed + Combination of more issues. Temp files are not automatically removed by + OS on windows so using tempfiles in TEMP would lead to kill disk space of + machine. There are also cases when someone wants to open multiple files + in short period of time and want to manually remove those files so keeping + track of temporary copied files in pre-defined structure is needed. + """ minute_in_seconds = 60 hour_in_seconds = 60 * minute_in_seconds day_in_seconds = 24 * hour_in_seconds @@ -72,16 +104,26 @@ class TempPublishFiles(object): @property def life_time(self): + """How long will be new item kept in temp in seconds. + + Returns: + int: Lifetime of temp item. + """ return int(self.hour_in_seconds) @property def size(self): + """File size of existing items.""" size = 0 for item in self.get_items(): size += item.size return size def add_file(self, src_path): + """Add workfile to temp directory. + + This will create new item and source path is copied to it's directory. + """ filename = os.path.basename(src_path) item_id = str(uuid.uuid4()) @@ -105,6 +147,7 @@ class TempPublishFiles(object): @contextlib.contextmanager def _modify_data(self): + """Create lock file when data in metadata file are modified.""" start_time = time.time() timeout = 3 while os.path.exists(self._lock_path): @@ -139,6 +182,15 @@ class TempPublishFiles(object): return output def cleanup(self, check_expiration=True): + """Cleanup files based on metadata. + + Items that passed expiration are removed when this is called. Or all + files are removed when `check_expiration` is set to False. + + Args: + check_expiration (bool): All items and files are removed when set + to True. + """ data = self._get_data() now = time.time() remove_ids = set() @@ -182,6 +234,11 @@ class TempPublishFiles(object): self.cleanup(False) def get_items(self): + """Receive all items from metadata file. + + Returns: + list: Info about each item in metadata. + """ output = [] data = self._get_data() for item_id, item_data in data.items(): @@ -190,6 +247,7 @@ class TempPublishFiles(object): return output def remove_id(self, item_id): + """Remove files of item and then remove the item from metadata.""" filepath = os.path.join(self._root_dir, item_id) if os.path.exists(filepath): shutil.rmtree(filepath) diff --git a/openpype/tools/workfiles/model.py b/openpype/tools/workfiles/model.py index 563a2fc558..4d772c58e0 100644 --- a/openpype/tools/workfiles/model.py +++ b/openpype/tools/workfiles/model.py @@ -19,6 +19,8 @@ ITEM_ID_ROLE = QtCore.Qt.UserRole + 4 class WorkAreaFilesModel(QtGui.QStandardItemModel): + """Model is looking into one folder for files with extension.""" + def __init__(self, extensions, *args, **kwargs): super(WorkAreaFilesModel, self).__init__(*args, **kwargs) @@ -64,6 +66,7 @@ class WorkAreaFilesModel(QtGui.QStandardItemModel): return self._empty_root_item def set_root(self, root): + """Change directory where to look for file.""" self._root = root if root and not os.path.exists(root): log.debug("Work Area does not exist: {}".format(root)) @@ -81,7 +84,9 @@ class WorkAreaFilesModel(QtGui.QStandardItemModel): self._items_by_filename = {} def refresh(self): + """Refresh and update model items.""" root_item = self.invisibleRootItem() + # If path is not set or does not exist then add invalid path item if not self._root or not os.path.exists(self._root): self._clear() # Add Work Area does not exist placeholder @@ -90,9 +95,14 @@ class WorkAreaFilesModel(QtGui.QStandardItemModel): self._invalid_item_visible = True return + # Clear items if previous refresh set '_invalid_item_visible' to True + # - Invalid items are not stored to '_items_by_filename' so they would + # not be removed if self._invalid_item_visible: self._clear() + # Check for new items that should be added and items that should be + # removed new_items = [] items_to_remove = set(self._items_by_filename.keys()) for filename in os.listdir(self._root): @@ -106,6 +116,7 @@ class WorkAreaFilesModel(QtGui.QStandardItemModel): modified = os.path.getmtime(filepath) + # Use existing item or create new one if filename in items_to_remove: items_to_remove.remove(filename) item = self._items_by_filename[filename] @@ -118,16 +129,20 @@ class WorkAreaFilesModel(QtGui.QStandardItemModel): item.setData(self._file_icon, QtCore.Qt.DecorationRole) new_items.append(item) self._items_by_filename[filename] = item + # Update data that may be different item.setData(filepath, FILEPATH_ROLE) item.setData(modified, DATE_MODIFIED_ROLE) + # Add new items if there are any if new_items: root_item.appendRows(new_items) + # Remove items that are no longer available for filename in items_to_remove: item = self._items_by_filename.pop(filename) root_item.removeRow(item.row()) + # Add empty root item if there are not filenames that could be shown if root_item.rowCount() > 0: self._invalid_item_visible = False else: @@ -136,9 +151,11 @@ class WorkAreaFilesModel(QtGui.QStandardItemModel): root_item.appendRow(item) def has_valid_items(self): + """Directory has files that are listed in items.""" return not self._invalid_item_visible def flags(self, index): + # Use flags of first column for all columns if index.column() != 0: index = self.index(index.row(), 0, index.parent()) return super(WorkAreaFilesModel, self).flags(index) @@ -147,6 +164,7 @@ class WorkAreaFilesModel(QtGui.QStandardItemModel): if role is None: role = QtCore.Qt.DisplayRole + # Handle roles for first column if index.column() == 1: if role == QtCore.Qt.DecorationRole: return None @@ -174,6 +192,22 @@ class WorkAreaFilesModel(QtGui.QStandardItemModel): class PublishFilesModel(QtGui.QStandardItemModel): + """Model filling files with published files calculated from representation. + + This model looks for workfile family representations based on selected + asset and task. + + Asset must set to be able look for representations that could be used. + Task is used to filter representations by task. + Model has few filter criteria for filling. + - First criteria is that version document must have "workfile" in + "data.families". + - Second cirteria is that representation must have extension same as + defined extensions + - If task is set then representation must have 'task["name"]' with same + name. + """ + def __init__(self, extensions, dbcon, anatomy, *args, **kwargs): super(PublishFilesModel, self).__init__(*args, **kwargs) @@ -225,6 +259,12 @@ class PublishFilesModel(QtGui.QStandardItemModel): return self._empty_root_item def set_context(self, asset_id, task_name): + """Change context to asset and task. + + Args: + asset_id (ObjectId): Id of selected asset. + task_name (str): Name of selected task. + """ self._asset_id = asset_id self._task_name = task_name self.refresh() @@ -242,6 +282,7 @@ class PublishFilesModel(QtGui.QStandardItemModel): def _get_workfie_representations(self): output = [] + # Get subset docs of asset subset_docs = self._dbcon.find( { "type": "subset", @@ -286,6 +327,7 @@ class PublishFilesModel(QtGui.QStandardItemModel): "context.ext": {"$in": extensions} } ) + # Filter queried representations by task name if task is set filtered_repre_docs = [] for repre_doc in repre_docs: if self._task_name is None: @@ -305,6 +347,7 @@ class PublishFilesModel(QtGui.QStandardItemModel): if task_name == self._task_name: filtered_repre_docs.append(repre_doc) + # Collect paths of representations for repre_doc in filtered_repre_docs: path = get_representation_path( repre_doc, root=self._anatomy.roots