From 31b6b79919b8a6c5d918298e707152ffcba1dcb3 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 5 Feb 2024 15:05:12 +0100 Subject: [PATCH] modified packages creation --- .gitignore | 5 +- client/openpype/version.py | 4 +- .../openpype/client => client}/pyproject.toml | 4 +- create_package.py | 357 ++++++++++++++++++ .../core/server => server}/__init__.py | 0 .../server => server}/settings/__init__.py | 0 .../core/server => server}/settings/main.py | 0 .../settings/publish_plugins.py | 0 .../core/server => server}/settings/tools.py | 0 server/version.py | 1 + server_addon/core/server/version.py | 1 - server_addon/create_ayon_addons.py | 81 +--- server_addon/openpype/server/__init__.py | 9 - 13 files changed, 368 insertions(+), 94 deletions(-) rename {server_addon/openpype/client => client}/pyproject.toml (88%) create mode 100644 create_package.py rename {server_addon/core/server => server}/__init__.py (100%) rename {server_addon/core/server => server}/settings/__init__.py (100%) rename {server_addon/core/server => server}/settings/main.py (100%) rename {server_addon/core/server => server}/settings/publish_plugins.py (100%) rename {server_addon/core/server => server}/settings/tools.py (100%) create mode 100644 server/version.py delete mode 100644 server_addon/core/server/version.py delete mode 100644 server_addon/openpype/server/__init__.py diff --git a/.gitignore b/.gitignore index 249322b667..7fda0dacdb 100644 --- a/.gitignore +++ b/.gitignore @@ -33,11 +33,10 @@ Temporary Items .apdisk -# CX_Freeze +# Package dirs ########### -/build -/dist/ /server_addon/packages/* +/package/* /vendor/bin/* /vendor/python/* diff --git a/client/openpype/version.py b/client/openpype/version.py index db6da9f656..a1c9035f50 100644 --- a/client/openpype/version.py +++ b/client/openpype/version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- -"""Package declaring Pype version.""" -__version__ = "3.18.7-nightly.1" +"""Package declaring AYON core version.""" +__version__ = "0.2.0" diff --git a/server_addon/openpype/client/pyproject.toml b/client/pyproject.toml similarity index 88% rename from server_addon/openpype/client/pyproject.toml rename to client/pyproject.toml index b5978f0498..c21ca305a7 100644 --- a/server_addon/openpype/client/pyproject.toml +++ b/client/pyproject.toml @@ -1,6 +1,6 @@ [project] -name="openpype" -description="OpenPype addon for AYON server." +name="core" +description="AYON core addon." [tool.poetry.dependencies] python = ">=3.9.1,<3.10" diff --git a/create_package.py b/create_package.py new file mode 100644 index 0000000000..cba227f759 --- /dev/null +++ b/create_package.py @@ -0,0 +1,357 @@ +"""Prepares server package from addon repo to upload to server. + +Requires Python 3.9. (Or at least 3.8+). + +This script should be called from cloned addon repo. + +It will produce 'package' subdirectory which could be pasted into server +addon directory directly (eg. into `ayon-backend/addons`). + +Format of package folder: +ADDON_REPO/package/{addon name}/{addon version} + +You can specify `--output_dir` in arguments to change output directory where +package will be created. Existing package directory will always be purged if +already present! This could be used to create package directly in server folder +if available. + +Package contains server side files directly, +client side code zipped in `private` subfolder. +""" + +import os +import sys +import re +import json +import shutil +import argparse +import platform +import logging +import collections +import zipfile +import hashlib +from pathlib import Path + +from typing import Optional + + +ADDON_NAME = "core" +ADDON_CLIENT_DIR = "openpype" + +# Patterns of directories to be skipped for server part of addon +IGNORE_DIR_PATTERNS = [ + re.compile(pattern) + for pattern in { + # Skip directories starting with '.' + r"^\.", + # Skip any pycache folders + "^__pycache__$" + } +] + +# Patterns of files to be skipped for server part of addon +IGNORE_FILE_PATTERNS = [ + re.compile(pattern) + for pattern in { + # Skip files starting with '.' + # NOTE this could be an issue in some cases + r"^\.", + # Skip '.pyc' files + r"\.pyc$" + } +] + + +def calculate_file_checksum(filepath, hash_algorithm, chunk_size=10000): + func = getattr(hashlib, hash_algorithm) + hash_obj = func() + with open(filepath, "rb") as f: + for chunk in iter(lambda: f.read(chunk_size), b""): + hash_obj.update(chunk) + return hash_obj.hexdigest() + + +class ZipFileLongPaths(zipfile.ZipFile): + """Allows longer paths in zip files. + + Regular DOS paths are limited to MAX_PATH (260) characters, including + the string's terminating NUL character. + That limit can be exceeded by using an extended-length path that + starts with the '\\?\' prefix. + """ + _is_windows = platform.system().lower() == "windows" + + def _extract_member(self, member, tpath, pwd): + if self._is_windows: + tpath = os.path.abspath(tpath) + if tpath.startswith("\\\\"): + tpath = "\\\\?\\UNC\\" + tpath[2:] + else: + tpath = "\\\\?\\" + tpath + + return super(ZipFileLongPaths, self)._extract_member( + member, tpath, pwd + ) + + +def safe_copy_file(src_path, dst_path): + """Copy file and make sure destination directory exists. + + Ignore if destination already contains directories from source. + + Args: + src_path (str): File path that will be copied. + dst_path (str): Path to destination file. + """ + + if src_path == dst_path: + return + + dst_dir = os.path.dirname(dst_path) + try: + os.makedirs(dst_dir) + except Exception: + pass + + shutil.copy2(src_path, dst_path) + + +def _value_match_regexes(value, regexes): + for regex in regexes: + if regex.search(value): + return True + return False + + +def find_files_in_subdir( + src_path, + ignore_file_patterns=None, + ignore_dir_patterns=None +): + if ignore_file_patterns is None: + ignore_file_patterns = IGNORE_FILE_PATTERNS + + if ignore_dir_patterns is None: + ignore_dir_patterns = IGNORE_DIR_PATTERNS + output = [] + + hierarchy_queue = collections.deque() + hierarchy_queue.append((src_path, [])) + while hierarchy_queue: + item = hierarchy_queue.popleft() + dirpath, parents = item + for name in os.listdir(dirpath): + path = os.path.join(dirpath, name) + if os.path.isfile(path): + if not _value_match_regexes(name, ignore_file_patterns): + items = list(parents) + items.append(name) + output.append((path, os.path.sep.join(items))) + continue + + if not _value_match_regexes(name, ignore_dir_patterns): + items = list(parents) + items.append(name) + hierarchy_queue.append((path, items)) + + return output + + +def copy_server_content(addon_output_dir, current_dir, log): + """Copies server side folders to 'addon_package_dir' + + Args: + addon_output_dir (str): package dir in addon repo dir + current_dir (str): addon repo dir + log (logging.Logger) + """ + + log.info("Copying server content") + + filepaths_to_copy = [] + server_dirpath = os.path.join(current_dir, "server") + + # Version + src_version_path = os.path.join(server_dirpath, "version.py") + dst_version_path = os.path.join(addon_output_dir, "version.py") + filepaths_to_copy.append((src_version_path, dst_version_path)) + + for item in find_files_in_subdir(server_dirpath): + src_path, dst_subpath = item + dst_path = os.path.join(addon_output_dir, dst_subpath) + filepaths_to_copy.append((src_path, dst_path)) + + # Copy files + for src_path, dst_path in filepaths_to_copy: + safe_copy_file(src_path, dst_path) + + +def zip_client_side(addon_package_dir, current_dir, log): + """Copy and zip `client` content into 'addon_package_dir'. + + Args: + addon_package_dir (str): Output package directory path. + current_dir (str): Directory path of addon source. + log (logging.Logger): Logger object. + """ + + client_dir = os.path.join(current_dir, "client") + client_addon_dir = os.path.join(client_dir, ADDON_CLIENT_DIR) + if not os.path.isdir(client_addon_dir): + raise ValueError( + f"Failed to find client directory '{client_addon_dir}'" + ) + + log.info("Preparing client code zip") + private_dir = os.path.join(addon_package_dir, "private") + + if not os.path.exists(private_dir): + os.makedirs(private_dir) + + server_dirpath = os.path.join(current_dir, "server") + src_version_path = os.path.join(server_dirpath, "version.py") + dst_version_path = os.path.join(ADDON_CLIENT_DIR, "version.py") + + zip_filepath = os.path.join(os.path.join(private_dir, "client.zip")) + with ZipFileLongPaths(zip_filepath, "w", zipfile.ZIP_DEFLATED) as zipf: + # Add client code content to zip + for path, sub_path in find_files_in_subdir(client_addon_dir): + sub_path = os.path.join(ADDON_CLIENT_DIR, sub_path) + if sub_path == dst_version_path: + continue + zipf.write(path, sub_path) + + # Add 'version.py' to client code + zipf.write(src_version_path, dst_version_path) + + shutil.copy(os.path.join(client_dir, "pyproject.toml"), private_dir) + + +def create_server_package( + output_dir: str, + addon_output_dir: str, + addon_version: str, + log: logging.Logger +): + """Create server package zip file. + + The zip file can be installed to a server using UI or rest api endpoints. + + Args: + output_dir (str): Directory path to output zip file. + addon_output_dir (str): Directory path to addon output directory. + addon_version (str): Version of addon. + log (logging.Logger): Logger object. + """ + + log.info("Creating server package") + output_path = os.path.join( + output_dir, f"{ADDON_NAME}-{addon_version}.zip" + ) + manifest_data: dict[str, str] = { + "addon_name": ADDON_NAME, + "addon_version": addon_version + } + with ZipFileLongPaths(output_path, "w", zipfile.ZIP_DEFLATED) as zipf: + # Write a manifest to zip + zipf.writestr("manifest.json", json.dumps(manifest_data, indent=4)) + + # Move addon content to zip into 'addon' directory + addon_output_dir_offset = len(addon_output_dir) + 1 + for root, _, filenames in os.walk(addon_output_dir): + if not filenames: + continue + + dst_root = "addon" + if root != addon_output_dir: + dst_root = os.path.join( + dst_root, root[addon_output_dir_offset:] + ) + for filename in filenames: + src_path = os.path.join(root, filename) + dst_path = os.path.join(dst_root, filename) + zipf.write(src_path, dst_path) + + log.info(f"Output package can be found: {output_path}") + + +def main( + output_dir: Optional[str]=None, + skip_zip: bool=False, + keep_sources: bool=False +): + log = logging.getLogger("create_package") + log.info("Start creating package") + + current_dir = os.path.dirname(os.path.abspath(__file__)) + if not output_dir: + output_dir = os.path.join(current_dir, "package") + + server_dirpath = os.path.join(current_dir, "server") + version_filepath = os.path.join(server_dirpath, "version.py") + version_content = {} + with open(version_filepath, "r") as stream: + exec(stream.read(), version_content) + addon_version = version_content["__version__"] + + new_created_version_dir = os.path.join( + output_dir, ADDON_NAME, addon_version + ) + if os.path.isdir(new_created_version_dir): + log.info(f"Purging {new_created_version_dir}") + shutil.rmtree(output_dir) + + log.info(f"Preparing package for {ADDON_NAME}-{addon_version}") + + addon_output_root = os.path.join(output_dir, ADDON_NAME) + addon_output_dir = os.path.join(addon_output_root, addon_version) + if not os.path.exists(addon_output_dir): + os.makedirs(addon_output_dir) + + copy_server_content(addon_output_dir, current_dir, log) + + zip_client_side(addon_output_dir, current_dir, log) + + # Skip server zipping + if not skip_zip: + create_server_package( + output_dir, addon_output_dir, addon_version, log + ) + # Remove sources only if zip file is created + if not keep_sources: + log.info("Removing source files for server package") + shutil.rmtree(addon_output_root) + log.info("Package creation finished") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--skip-zip", + dest="skip_zip", + action="store_true", + help=( + "Skip zipping server package and create only" + " server folder structure." + ) + ) + parser.add_argument( + "--keep-sources", + dest="keep_sources", + action="store_true", + help=( + "Keep folder structure when server package is created." + ) + ) + parser.add_argument( + "-o", "--output", + dest="output_dir", + default=None, + help=( + "Directory path where package will be created" + " (Will be purged if already exists!)" + ) + ) + + args = parser.parse_args(sys.argv[1:]) + main(args.output_dir, args.skip_zip, args.keep_sources) diff --git a/server_addon/core/server/__init__.py b/server/__init__.py similarity index 100% rename from server_addon/core/server/__init__.py rename to server/__init__.py diff --git a/server_addon/core/server/settings/__init__.py b/server/settings/__init__.py similarity index 100% rename from server_addon/core/server/settings/__init__.py rename to server/settings/__init__.py diff --git a/server_addon/core/server/settings/main.py b/server/settings/main.py similarity index 100% rename from server_addon/core/server/settings/main.py rename to server/settings/main.py diff --git a/server_addon/core/server/settings/publish_plugins.py b/server/settings/publish_plugins.py similarity index 100% rename from server_addon/core/server/settings/publish_plugins.py rename to server/settings/publish_plugins.py diff --git a/server_addon/core/server/settings/tools.py b/server/settings/tools.py similarity index 100% rename from server_addon/core/server/settings/tools.py rename to server/settings/tools.py diff --git a/server/version.py b/server/version.py new file mode 100644 index 0000000000..d3ec452c31 --- /dev/null +++ b/server/version.py @@ -0,0 +1 @@ +__version__ = "0.2.0" diff --git a/server_addon/core/server/version.py b/server_addon/core/server/version.py deleted file mode 100644 index bbab0242f6..0000000000 --- a/server_addon/core/server/version.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = "0.1.4" diff --git a/server_addon/create_ayon_addons.py b/server_addon/create_ayon_addons.py index fc7a673dcc..08443d6588 100644 --- a/server_addon/create_ayon_addons.py +++ b/server_addon/create_ayon_addons.py @@ -38,14 +38,7 @@ IGNORED_HOSTS = [ "harmony", ] -IGNORED_MODULES = [ - "ftrack", - "shotgrid", - "sync_server", - "example_addons", - "slack", - "kitsu", -] +IGNORED_MODULES = [] class ZipFileLongPaths(zipfile.ZipFile): @@ -183,66 +176,6 @@ def create_addon_zip( shutil.rmtree(str(output_dir / addon_name)) -def create_openpype_package( - addon_dir: Path, - output_dir: Path, - root_dir: Path, - create_zip: bool, - keep_source: bool -): - server_dir = addon_dir / "server" - pyproject_path = addon_dir / "client" / "pyproject.toml" - - openpype_dir = root_dir / "openpype" - version_path = openpype_dir / "version.py" - addon_version = read_addon_version(version_path) - - addon_output_dir = output_dir / "openpype" / addon_version - private_dir = addon_output_dir / "private" - if addon_output_dir.exists(): - shutil.rmtree(str(addon_output_dir)) - - # Make sure dir exists - addon_output_dir.mkdir(parents=True, exist_ok=True) - private_dir.mkdir(parents=True, exist_ok=True) - - # Copy version - shutil.copy(str(version_path), str(addon_output_dir)) - for subitem in server_dir.iterdir(): - shutil.copy(str(subitem), str(addon_output_dir / subitem.name)) - - # Copy pyproject.toml - shutil.copy( - str(pyproject_path), - (private_dir / pyproject_path.name) - ) - # Subdirs that won't be added to output zip file - ignored_subpaths = [ - ["addons"], - ["vendor", "common", "ayon_api"], - ] - ignored_subpaths.extend( - ["hosts", host_name] - for host_name in IGNORED_HOSTS - ) - ignored_subpaths.extend( - ["modules", module_name] - for module_name in IGNORED_MODULES - ) - - # Zip client - zip_filepath = private_dir / "client.zip" - with ZipFileLongPaths(zip_filepath, "w", zipfile.ZIP_DEFLATED) as zipf: - # Add client code content to zip - for path, sub_path in find_files_in_subdir( - str(openpype_dir), ignore_subdirs=ignored_subpaths - ): - zipf.write(path, f"{openpype_dir.name}/{sub_path}") - - if create_zip: - create_addon_zip(output_dir, "openpype", addon_version, keep_source) - - def create_addon_package( addon_dir: Path, output_dir: Path, @@ -316,15 +249,9 @@ def main( if not server_dir.exists(): continue - if addon_dir.name == "openpype": - create_openpype_package( - addon_dir, output_dir, root_dir, create_zip, keep_source - ) - - else: - create_addon_package( - addon_dir, output_dir, create_zip, keep_source - ) + create_addon_package( + addon_dir, output_dir, create_zip, keep_source + ) print(f"- package '{addon_dir.name}' created") print(f"Package creation finished. Output directory: {output_dir}") diff --git a/server_addon/openpype/server/__init__.py b/server_addon/openpype/server/__init__.py deleted file mode 100644 index df24c73c76..0000000000 --- a/server_addon/openpype/server/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -from ayon_server.addons import BaseServerAddon - -from .version import __version__ - - -class OpenPypeAddon(BaseServerAddon): - name = "openpype" - title = "OpenPype" - version = __version__