modified packages creation

.gitignore

@@ -33,11 +33,10 @@ Temporary Items
 .apdisk
 
 
-# CX_Freeze
+# Package dirs
 ###########
-/build
-/dist/
 /server_addon/packages/*
+/package/*
 
 /vendor/bin/*
 /vendor/python/*

client/openpype/version.py

@@ -1,3 +1,3 @@
 # -*- coding: utf-8 -*-
-"""Package declaring Pype version."""
+"""Package declaring AYON core version."""
-__version__ = "3.18.7-nightly.1"
+__version__ = "0.2.0"

client/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
-name="openpype"
+name="core"
-description="OpenPype addon for AYON server."
+description="AYON core addon."
 
 [tool.poetry.dependencies]
 python = ">=3.9.1,<3.10"

create_package.py

@@ -0,0 +1,357 @@
+"""Prepares server package from addon repo to upload to server.
+
+Requires Python 3.9. (Or at least 3.8+).
+
+This script should be called from cloned addon repo.
+
+It will produce 'package' subdirectory which could be pasted into server
+addon directory directly (eg. into `ayon-backend/addons`).
+
+Format of package folder:
+ADDON_REPO/package/{addon name}/{addon version}
+
+You can specify `--output_dir` in arguments to change output directory where
+package will be created. Existing package directory will always be purged if
+already present! This could be used to create package directly in server folder
+if available.
+
+Package contains server side files directly,
+client side code zipped in `private` subfolder.
+"""
+
+import os
+import sys
+import re
+import json
+import shutil
+import argparse
+import platform
+import logging
+import collections
+import zipfile
+import hashlib
+from pathlib import Path
+
+from typing import Optional
+
+
+ADDON_NAME = "core"
+ADDON_CLIENT_DIR = "openpype"
+
+# Patterns of directories to be skipped for server part of addon
+IGNORE_DIR_PATTERNS = [
+    re.compile(pattern)
+    for pattern in {
+        # Skip directories starting with '.'
+        r"^\.",
+        # Skip any pycache folders
+        "^__pycache__$"
+    }
+]
+
+# Patterns of files to be skipped for server part of addon
+IGNORE_FILE_PATTERNS = [
+    re.compile(pattern)
+    for pattern in {
+        # Skip files starting with '.'
+        # NOTE this could be an issue in some cases
+        r"^\.",
+        # Skip '.pyc' files
+        r"\.pyc$"
+    }
+]
+
+
+def calculate_file_checksum(filepath, hash_algorithm, chunk_size=10000):
+    func = getattr(hashlib, hash_algorithm)
+    hash_obj = func()
+    with open(filepath, "rb") as f:
+        for chunk in iter(lambda: f.read(chunk_size), b""):
+            hash_obj.update(chunk)
+    return hash_obj.hexdigest()
+
+
+class ZipFileLongPaths(zipfile.ZipFile):
+    """Allows longer paths in zip files.
+
+    Regular DOS paths are limited to MAX_PATH (260) characters, including
+    the string's terminating NUL character.
+    That limit can be exceeded by using an extended-length path that
+    starts with the '\\?\' prefix.
+    """
+    _is_windows = platform.system().lower() == "windows"
+
+    def _extract_member(self, member, tpath, pwd):
+        if self._is_windows:
+            tpath = os.path.abspath(tpath)
+            if tpath.startswith("\\\\"):
+                tpath = "\\\\?\\UNC\\" + tpath[2:]
+            else:
+                tpath = "\\\\?\\" + tpath
+
+        return super(ZipFileLongPaths, self)._extract_member(
+            member, tpath, pwd
+        )
+
+
+def safe_copy_file(src_path, dst_path):
+    """Copy file and make sure destination directory exists.
+
+    Ignore if destination already contains directories from source.
+
+    Args:
+        src_path (str): File path that will be copied.
+        dst_path (str): Path to destination file.
+    """
+
+    if src_path == dst_path:
+        return
+
+    dst_dir = os.path.dirname(dst_path)
+    try:
+        os.makedirs(dst_dir)
+    except Exception:
+        pass
+
+    shutil.copy2(src_path, dst_path)
+
+
+def _value_match_regexes(value, regexes):
+    for regex in regexes:
+        if regex.search(value):
+            return True
+    return False
+
+
+def find_files_in_subdir(
+    src_path,
+    ignore_file_patterns=None,
+    ignore_dir_patterns=None
+):
+    if ignore_file_patterns is None:
+        ignore_file_patterns = IGNORE_FILE_PATTERNS
+
+    if ignore_dir_patterns is None:
+        ignore_dir_patterns = IGNORE_DIR_PATTERNS
+    output = []
+
+    hierarchy_queue = collections.deque()
+    hierarchy_queue.append((src_path, []))
+    while hierarchy_queue:
+        item = hierarchy_queue.popleft()
+        dirpath, parents = item
+        for name in os.listdir(dirpath):
+            path = os.path.join(dirpath, name)
+            if os.path.isfile(path):
+                if not _value_match_regexes(name, ignore_file_patterns):
+                    items = list(parents)
+                    items.append(name)
+                    output.append((path, os.path.sep.join(items)))
+                continue
+
+            if not _value_match_regexes(name, ignore_dir_patterns):
+                items = list(parents)
+                items.append(name)
+                hierarchy_queue.append((path, items))
+
+    return output
+
+
+def copy_server_content(addon_output_dir, current_dir, log):
+    """Copies server side folders to 'addon_package_dir'
+
+    Args:
+        addon_output_dir (str): package dir in addon repo dir
+        current_dir (str): addon repo dir
+        log (logging.Logger)
+    """
+
+    log.info("Copying server content")
+
+    filepaths_to_copy = []
+    server_dirpath = os.path.join(current_dir, "server")
+
+    # Version
+    src_version_path = os.path.join(server_dirpath, "version.py")
+    dst_version_path = os.path.join(addon_output_dir, "version.py")
+    filepaths_to_copy.append((src_version_path, dst_version_path))
+
+    for item in find_files_in_subdir(server_dirpath):
+        src_path, dst_subpath = item
+        dst_path = os.path.join(addon_output_dir, dst_subpath)
+        filepaths_to_copy.append((src_path, dst_path))
+
+    # Copy files
+    for src_path, dst_path in filepaths_to_copy:
+        safe_copy_file(src_path, dst_path)
+
+
+def zip_client_side(addon_package_dir, current_dir, log):
+    """Copy and zip `client` content into 'addon_package_dir'.
+
+    Args:
+        addon_package_dir (str): Output package directory path.
+        current_dir (str): Directory path of addon source.
+        log (logging.Logger): Logger object.
+    """
+
+    client_dir = os.path.join(current_dir, "client")
+    client_addon_dir = os.path.join(client_dir, ADDON_CLIENT_DIR)
+    if not os.path.isdir(client_addon_dir):
+        raise ValueError(
+            f"Failed to find client directory '{client_addon_dir}'"
+        )
+
+    log.info("Preparing client code zip")
+    private_dir = os.path.join(addon_package_dir, "private")
+
+    if not os.path.exists(private_dir):
+        os.makedirs(private_dir)
+
+    server_dirpath = os.path.join(current_dir, "server")
+    src_version_path = os.path.join(server_dirpath, "version.py")
+    dst_version_path = os.path.join(ADDON_CLIENT_DIR, "version.py")
+
+    zip_filepath = os.path.join(os.path.join(private_dir, "client.zip"))
+    with ZipFileLongPaths(zip_filepath, "w", zipfile.ZIP_DEFLATED) as zipf:
+        # Add client code content to zip
+        for path, sub_path in find_files_in_subdir(client_addon_dir):
+            sub_path = os.path.join(ADDON_CLIENT_DIR, sub_path)
+            if sub_path == dst_version_path:
+                continue
+            zipf.write(path, sub_path)
+
+        # Add 'version.py' to client code
+        zipf.write(src_version_path, dst_version_path)
+
+    shutil.copy(os.path.join(client_dir, "pyproject.toml"), private_dir)
+
+
+def create_server_package(
+    output_dir: str,
+    addon_output_dir: str,
+    addon_version: str,
+    log: logging.Logger
+):
+    """Create server package zip file.
+
+    The zip file can be installed to a server using UI or rest api endpoints.
+
+    Args:
+        output_dir (str): Directory path to output zip file.
+        addon_output_dir (str): Directory path to addon output directory.
+        addon_version (str): Version of addon.
+        log (logging.Logger): Logger object.
+    """
+
+    log.info("Creating server package")
+    output_path = os.path.join(
+        output_dir, f"{ADDON_NAME}-{addon_version}.zip"
+    )
+    manifest_data: dict[str, str] = {
+        "addon_name": ADDON_NAME,
+        "addon_version": addon_version
+    }
+    with ZipFileLongPaths(output_path, "w", zipfile.ZIP_DEFLATED) as zipf:
+        # Write a manifest to zip
+        zipf.writestr("manifest.json", json.dumps(manifest_data, indent=4))
+
+        # Move addon content to zip into 'addon' directory
+        addon_output_dir_offset = len(addon_output_dir) + 1
+        for root, _, filenames in os.walk(addon_output_dir):
+            if not filenames:
+                continue
+
+            dst_root = "addon"
+            if root != addon_output_dir:
+                dst_root = os.path.join(
+                    dst_root, root[addon_output_dir_offset:]
+                )
+            for filename in filenames:
+                src_path = os.path.join(root, filename)
+                dst_path = os.path.join(dst_root, filename)
+                zipf.write(src_path, dst_path)
+
+    log.info(f"Output package can be found: {output_path}")
+
+
+def main(
+    output_dir: Optional[str]=None,
+    skip_zip: bool=False,
+    keep_sources: bool=False
+):
+    log = logging.getLogger("create_package")
+    log.info("Start creating package")
+
+    current_dir = os.path.dirname(os.path.abspath(__file__))
+    if not output_dir:
+        output_dir = os.path.join(current_dir, "package")
+
+    server_dirpath = os.path.join(current_dir, "server")
+    version_filepath = os.path.join(server_dirpath, "version.py")
+    version_content = {}
+    with open(version_filepath, "r") as stream:
+        exec(stream.read(), version_content)
+    addon_version = version_content["__version__"]
+
+    new_created_version_dir = os.path.join(
+        output_dir, ADDON_NAME, addon_version
+    )
+    if os.path.isdir(new_created_version_dir):
+        log.info(f"Purging {new_created_version_dir}")
+        shutil.rmtree(output_dir)
+
+    log.info(f"Preparing package for {ADDON_NAME}-{addon_version}")
+
+    addon_output_root = os.path.join(output_dir, ADDON_NAME)
+    addon_output_dir = os.path.join(addon_output_root, addon_version)
+    if not os.path.exists(addon_output_dir):
+        os.makedirs(addon_output_dir)
+
+    copy_server_content(addon_output_dir, current_dir, log)
+
+    zip_client_side(addon_output_dir, current_dir, log)
+
+    # Skip server zipping
+    if not skip_zip:
+        create_server_package(
+            output_dir, addon_output_dir, addon_version, log
+        )
+        # Remove sources only if zip file is created
+        if not keep_sources:
+            log.info("Removing source files for server package")
+            shutil.rmtree(addon_output_root)
+    log.info("Package creation finished")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--skip-zip",
+        dest="skip_zip",
+        action="store_true",
+        help=(
+            "Skip zipping server package and create only"
+            " server folder structure."
+        )
+    )
+    parser.add_argument(
+        "--keep-sources",
+        dest="keep_sources",
+        action="store_true",
+        help=(
+            "Keep folder structure when server package is created."
+        )
+    )
+    parser.add_argument(
+        "-o", "--output",
+        dest="output_dir",
+        default=None,
+        help=(
+            "Directory path where package will be created"
+            " (Will be purged if already exists!)"
+        )
+    )
+
+    args = parser.parse_args(sys.argv[1:])
+    main(args.output_dir, args.skip_zip, args.keep_sources)

server/version.py

@@ -0,0 +1 @@
+__version__ = "0.2.0"

server_addon/core/server/version.py

@@ -1 +0,0 @@
-__version__ = "0.1.4"

server_addon/create_ayon_addons.py

@@ -38,14 +38,7 @@ IGNORED_HOSTS = [
     "harmony",
 ]
 
-IGNORED_MODULES = [
+IGNORED_MODULES = []
-    "ftrack",
-    "shotgrid",
-    "sync_server",
-    "example_addons",
-    "slack",
-    "kitsu",
-]
 
 
 class ZipFileLongPaths(zipfile.ZipFile):
@@ -183,66 +176,6 @@ def create_addon_zip(
         shutil.rmtree(str(output_dir / addon_name))
 
 
-def create_openpype_package(
-    addon_dir: Path,
-    output_dir: Path,
-    root_dir: Path,
-    create_zip: bool,
-    keep_source: bool
-):
-    server_dir = addon_dir / "server"
-    pyproject_path = addon_dir / "client" / "pyproject.toml"
-
-    openpype_dir = root_dir / "openpype"
-    version_path = openpype_dir / "version.py"
-    addon_version = read_addon_version(version_path)
-
-    addon_output_dir = output_dir / "openpype" / addon_version
-    private_dir = addon_output_dir / "private"
-    if addon_output_dir.exists():
-        shutil.rmtree(str(addon_output_dir))
-
-    # Make sure dir exists
-    addon_output_dir.mkdir(parents=True, exist_ok=True)
-    private_dir.mkdir(parents=True, exist_ok=True)
-
-    # Copy version
-    shutil.copy(str(version_path), str(addon_output_dir))
-    for subitem in server_dir.iterdir():
-        shutil.copy(str(subitem), str(addon_output_dir / subitem.name))
-
-    # Copy pyproject.toml
-    shutil.copy(
-        str(pyproject_path),
-        (private_dir / pyproject_path.name)
-    )
-    # Subdirs that won't be added to output zip file
-    ignored_subpaths = [
-        ["addons"],
-        ["vendor", "common", "ayon_api"],
-    ]
-    ignored_subpaths.extend(
-        ["hosts", host_name]
-        for host_name in IGNORED_HOSTS
-    )
-    ignored_subpaths.extend(
-        ["modules", module_name]
-        for module_name in IGNORED_MODULES
-    )
-
-    # Zip client
-    zip_filepath = private_dir / "client.zip"
-    with ZipFileLongPaths(zip_filepath, "w", zipfile.ZIP_DEFLATED) as zipf:
-        # Add client code content to zip
-        for path, sub_path in find_files_in_subdir(
-            str(openpype_dir), ignore_subdirs=ignored_subpaths
-        ):
-            zipf.write(path, f"{openpype_dir.name}/{sub_path}")
-
-    if create_zip:
-        create_addon_zip(output_dir, "openpype", addon_version, keep_source)
-
-
 def create_addon_package(
     addon_dir: Path,
     output_dir: Path,
@@ -316,15 +249,9 @@ def main(
         if not server_dir.exists():
             continue
 
-        if addon_dir.name == "openpype":
+        create_addon_package(
-            create_openpype_package(
+            addon_dir, output_dir, create_zip, keep_source
-                addon_dir, output_dir, root_dir, create_zip, keep_source
+        )
-            )
-
-        else:
-            create_addon_package(
-                addon_dir, output_dir, create_zip, keep_source
-            )
 
         print(f"- package '{addon_dir.name}' created")
     print(f"Package creation finished. Output directory: {output_dir}")

server_addon/openpype/server/__init__.py

@@ -1,9 +0,0 @@
-from ayon_server.addons import BaseServerAddon
-
-from .version import __version__
-
-
-class OpenPypeAddon(BaseServerAddon):
-    name = "openpype"
-    title = "OpenPype"
-    version = __version__