diff --git a/.gitmodules b/.gitmodules index f08a36506c..20aa3a3e8d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,19 +2,12 @@ path = repos/avalon-core url = git@github.com:pypeclub/avalon-core.git branch = develop -[submodule "repos/pyblish-base"] - path = repos/pyblish-base - url = git@github.com:pyblish/pyblish-base.git [submodule "repos/avalon-unreal-integration"] path = repos/avalon-unreal-integration url = git@github.com:pypeclub/avalon-unreal-integration.git [submodule "repos/maya-look-assigner"] path = repos/maya-look-assigner url = git@github.com:pypeclub/maya-look-assigner.git -[submodule "repos/acre"] - path = repos/acre - url = git@github.com:antirotor/acre.git - branch = fix/unformatted-tokens [submodule "pype/modules/ftrack/python2_vendor/ftrack-python-api"] path = pype/modules/ftrack/python2_vendor/ftrack-python-api url = https://bitbucket.org/ftrack/ftrack-python-api.git diff --git a/README.md b/README.md index 4ad52c7e36..d206a2b73b 100644 --- a/README.md +++ b/README.md @@ -1,33 +1,106 @@ -# Pype -## Introduction +Pype +==== -Multi-platform open-source pipeline built around the [Avalon](https://getavalon.github.io/) platform, expanding it with extra features and integrations. Pype connects asset database, project management and time tracking into a single modular system. It has tight integration with [ftrack](https://www.ftrack.com/en/), but it can also run independently. +Introduction +------------ + +Multi-platform open-source pipeline built around the [Avalon](https://getavalon.github.io/) platform, +expanding it with extra features and integrations. Pype connects asset database, project management +and time tracking into a single modular system. It has tight integration +with [ftrack](https://www.ftrack.com/en/), but it can also run independently. To get all the key information about the project, go to [PYPE.club](http://pype.club) -## Hardware requirements +Requirements +------------ +Pype will run on most typical hardware configurations commonly found in studios around the world. +It is installed on artist computer and can take up 3Gb of space depending on number of versions +and other dependencies. -Pype should be installed centrally on a fast network storage with at least read access right for all workstations and users in the Studio. Full Deplyoyment with all dependencies and both Development and Production branches installed takes about 1GB of data, however to ensure smooth updates and general working comfort, we recommend allocating at least at least 4GB of storage dedicated to PYPE deployment. +For well functioning [ftrack](https://www.ftrack.com/en/) event server, we recommend a +linux virtual server with [Ubuntu](https://ubuntu.com/) or [CentosOS](https://www.centos.org/). +CPU and RAM allocation need differ based on the studio size, but a 2GB of RAM, with a +dual core CPU and around 4GB of storage should suffice. -For well functioning [ftrack](https://www.ftrack.com/en/) event server, we recommend a linux virtual server with [Ubuntu](https://ubuntu.com/) or [CentosOS](https://www.centos.org/). CPU and RAM allocation need differ based on the studio size, but a 2GB of RAM, with a dual core CPU and around 4GB of storage should suffice. +Pype needs running [mongodb](https://www.mongodb.com/) server with good connectivity as it is +heavily used by Pype. Depending on project size and number of artists working connection speed and +latency influence performance experienced by artists. If remote working is required, this mongodb +server must be accessible from Internet or cloud solution can be used. Reasonable backup plan +or high availability options are recommended. -## Building Pype +Building Pype +------------- ### Windows You will need [Python 3.7 and newer](https://www.python.org/downloads/) and [git](https://git-scm.com/downloads). +More tools might be needed for installing dependencies (for example for **OpenTimelineIO**) - mostly +development tools like [CMake](https://cmake.org/) and [Visual Studio](https://visualstudio.microsoft.com/cs/downloads/) Clone repository: ```sh git clone --recurse-submodules git@github.com:pypeclub/pype.git ``` -Run PowerShell script `build.ps1`. It will create *venv*, install all -required dependencies and build Pype. After it is finished, you will find -Pype in `build` folder. +#### To build Pype: -You might need more tools for installing dependencies (for example for **OpenTimelineIO**) - mostly -development tools like [CMake](https://cmake.org/) and [Visual Studio](https://visualstudio.microsoft.com/cs/downloads/) +1) Run `.\tools\create_env.ps1` to create virtual environment in `.\venv` +2) Run `.\tools\build.ps1` to build pype executables in `.\build\` + +To create distributable Pype versions, run `./tools/create_zip.ps1` - that will +create zip file with name `pype-vx.x.x.zip` parsed from current pype repository and +copy it to user data dir, or you can specify `--path /path/to/zip` to force it there. + +You can then point **Igniter** - Pype setup tool - to directory containing this zip and +it will install it on current computer. Pype is build using [CX_Freeze](https://cx-freeze.readthedocs.io/en/latest) to freeze itself and all dependencies. + +Running Pype +------------ + +Pype can by executed either from live sources (this repository) or from +*"frozen code"* - executables that can be build using steps described above. + +If Pype is executed from live sources, it will use Pype version included in them. If +it is executed from frozen code it will try to find latest Pype version installed locally +on current computer and if it is not found, it will ask for its location. On that location +pype can be either in directories or zip files. Pype will try to find latest version and +install it to user data directory (on Windows to `%LOCALAPPDATA%\pypeclub\pype`). + +### From sources +Pype can be run directly from sources by activating virtual environment: +```powershell +.\venv\Scripts\Activate.ps1 +``` +and running: +```powershell +python start.py tray +``` +This will use current Pype version with sources. You can override this with `--use-version=x.x.x` and +then Pype will try to find locally installed specified version (present in user data directory). + +### From frozen code + +You need to build Pype first. This will produce two executables - `pype.exe` and `pype_console.exe`. +First one will act as GUI application and will not create console (useful in production environments). +The second one will create console and will write output there - useful for headless application and +debugging purposes. If you need pype version installed, just run `./tools/create_zip.ps1` without +arguments and it will create zip file that pype can use. + + +Building documentation +---------------------- + +Top build API documentation, run `.\tools\make_docs.ps1`. It will create html documentation +from current sources in `.\docs\build`. + +**Note that it needs existing virtual environment.** + +Running tests +------------- + +To run tests, execute `.\tools\run_tests.ps1`. + +**Note that it needs existing virtual environment.** \ No newline at end of file diff --git a/dev_mongo.ps1 b/dev_mongo.ps1 deleted file mode 100644 index 9ad021e39d..0000000000 --- a/dev_mongo.ps1 +++ /dev/null @@ -1,2 +0,0 @@ -.\venv\Scripts\Activate.ps1 -python pype.py mongodb diff --git a/dev_settings.ps1 b/dev_settings.ps1 deleted file mode 100644 index 3eab14dc37..0000000000 --- a/dev_settings.ps1 +++ /dev/null @@ -1,2 +0,0 @@ -.\venv\Scripts\Activate.ps1 -python pype.py settings --dev diff --git a/dev_tray.ps1 b/dev_tray.ps1 deleted file mode 100644 index 44f3f69754..0000000000 --- a/dev_tray.ps1 +++ /dev/null @@ -1,2 +0,0 @@ -.\venv\Scripts\Activate.ps1 -python pype.py tray --debug diff --git a/docs/source/pype.hosts.aftereffects.rst b/docs/source/pype.hosts.aftereffects.rst new file mode 100644 index 0000000000..3c2b2dda41 --- /dev/null +++ b/docs/source/pype.hosts.aftereffects.rst @@ -0,0 +1,7 @@ +pype.hosts.aftereffects package +=============================== + +.. automodule:: pype.hosts.aftereffects + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.hosts.resolve.otio.davinci_export.rst b/docs/source/pype.hosts.resolve.otio.davinci_export.rst new file mode 100644 index 0000000000..498f96a7ed --- /dev/null +++ b/docs/source/pype.hosts.resolve.otio.davinci_export.rst @@ -0,0 +1,7 @@ +pype.hosts.resolve.otio.davinci\_export module +============================================== + +.. automodule:: pype.hosts.resolve.otio.davinci_export + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.hosts.resolve.otio.davinci_import.rst b/docs/source/pype.hosts.resolve.otio.davinci_import.rst new file mode 100644 index 0000000000..30f43cc9fe --- /dev/null +++ b/docs/source/pype.hosts.resolve.otio.davinci_import.rst @@ -0,0 +1,7 @@ +pype.hosts.resolve.otio.davinci\_import module +============================================== + +.. automodule:: pype.hosts.resolve.otio.davinci_import + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.hosts.resolve.otio.rst b/docs/source/pype.hosts.resolve.otio.rst new file mode 100644 index 0000000000..523d8937ca --- /dev/null +++ b/docs/source/pype.hosts.resolve.otio.rst @@ -0,0 +1,17 @@ +pype.hosts.resolve.otio package +=============================== + +.. automodule:: pype.hosts.resolve.otio + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +.. toctree:: + :maxdepth: 10 + + pype.hosts.resolve.otio.davinci_export + pype.hosts.resolve.otio.davinci_import + pype.hosts.resolve.otio.utils diff --git a/docs/source/pype.hosts.resolve.otio.utils.rst b/docs/source/pype.hosts.resolve.otio.utils.rst new file mode 100644 index 0000000000..765f492732 --- /dev/null +++ b/docs/source/pype.hosts.resolve.otio.utils.rst @@ -0,0 +1,7 @@ +pype.hosts.resolve.otio.utils module +==================================== + +.. automodule:: pype.hosts.resolve.otio.utils + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.hosts.resolve.todo-rendering.rst b/docs/source/pype.hosts.resolve.todo-rendering.rst new file mode 100644 index 0000000000..8ea80183ce --- /dev/null +++ b/docs/source/pype.hosts.resolve.todo-rendering.rst @@ -0,0 +1,7 @@ +pype.hosts.resolve.todo\-rendering module +========================================= + +.. automodule:: pype.hosts.resolve.todo-rendering + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.hosts.tvpaint.api.rst b/docs/source/pype.hosts.tvpaint.api.rst new file mode 100644 index 0000000000..43273e8ec5 --- /dev/null +++ b/docs/source/pype.hosts.tvpaint.api.rst @@ -0,0 +1,7 @@ +pype.hosts.tvpaint.api package +============================== + +.. automodule:: pype.hosts.tvpaint.api + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.hosts.tvpaint.rst b/docs/source/pype.hosts.tvpaint.rst new file mode 100644 index 0000000000..561be3a9dc --- /dev/null +++ b/docs/source/pype.hosts.tvpaint.rst @@ -0,0 +1,15 @@ +pype.hosts.tvpaint package +========================== + +.. automodule:: pype.hosts.tvpaint + :members: + :undoc-members: + :show-inheritance: + +Subpackages +----------- + +.. toctree:: + :maxdepth: 10 + + pype.hosts.tvpaint.api diff --git a/docs/source/pype.lib.abstract_collect_render.rst b/docs/source/pype.lib.abstract_collect_render.rst new file mode 100644 index 0000000000..d6adadc271 --- /dev/null +++ b/docs/source/pype.lib.abstract_collect_render.rst @@ -0,0 +1,7 @@ +pype.lib.abstract\_collect\_render module +========================================= + +.. automodule:: pype.lib.abstract_collect_render + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.lib.abstract_expected_files.rst b/docs/source/pype.lib.abstract_expected_files.rst new file mode 100644 index 0000000000..904aeb3375 --- /dev/null +++ b/docs/source/pype.lib.abstract_expected_files.rst @@ -0,0 +1,7 @@ +pype.lib.abstract\_expected\_files module +========================================= + +.. automodule:: pype.lib.abstract_expected_files + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.lib.abstract_metaplugins.rst b/docs/source/pype.lib.abstract_metaplugins.rst new file mode 100644 index 0000000000..9f2751b630 --- /dev/null +++ b/docs/source/pype.lib.abstract_metaplugins.rst @@ -0,0 +1,7 @@ +pype.lib.abstract\_metaplugins module +===================================== + +.. automodule:: pype.lib.abstract_metaplugins + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.lib.abstract_submit_deadline.rst b/docs/source/pype.lib.abstract_submit_deadline.rst new file mode 100644 index 0000000000..a57222add3 --- /dev/null +++ b/docs/source/pype.lib.abstract_submit_deadline.rst @@ -0,0 +1,7 @@ +pype.lib.abstract\_submit\_deadline module +========================================== + +.. automodule:: pype.lib.abstract_submit_deadline + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.lib.applications.rst b/docs/source/pype.lib.applications.rst new file mode 100644 index 0000000000..8d1ff9b2c6 --- /dev/null +++ b/docs/source/pype.lib.applications.rst @@ -0,0 +1,7 @@ +pype.lib.applications module +============================ + +.. automodule:: pype.lib.applications + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.lib.avalon_context.rst b/docs/source/pype.lib.avalon_context.rst new file mode 100644 index 0000000000..067ea3380f --- /dev/null +++ b/docs/source/pype.lib.avalon_context.rst @@ -0,0 +1,7 @@ +pype.lib.avalon\_context module +=============================== + +.. automodule:: pype.lib.avalon_context + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.lib.deprecated.rst b/docs/source/pype.lib.deprecated.rst new file mode 100644 index 0000000000..ec5ee58d67 --- /dev/null +++ b/docs/source/pype.lib.deprecated.rst @@ -0,0 +1,7 @@ +pype.lib.deprecated module +========================== + +.. automodule:: pype.lib.deprecated + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.lib.editorial.rst b/docs/source/pype.lib.editorial.rst new file mode 100644 index 0000000000..d32e495e51 --- /dev/null +++ b/docs/source/pype.lib.editorial.rst @@ -0,0 +1,7 @@ +pype.lib.editorial module +========================= + +.. automodule:: pype.lib.editorial + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.lib.env_tools.rst b/docs/source/pype.lib.env_tools.rst new file mode 100644 index 0000000000..cb470207c8 --- /dev/null +++ b/docs/source/pype.lib.env_tools.rst @@ -0,0 +1,7 @@ +pype.lib.env\_tools module +========================== + +.. automodule:: pype.lib.env_tools + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.lib.ffmpeg_utils.rst b/docs/source/pype.lib.ffmpeg_utils.rst new file mode 100644 index 0000000000..968a3f39c8 --- /dev/null +++ b/docs/source/pype.lib.ffmpeg_utils.rst @@ -0,0 +1,7 @@ +pype.lib.ffmpeg\_utils module +============================= + +.. automodule:: pype.lib.ffmpeg_utils + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.lib.path_tools.rst b/docs/source/pype.lib.path_tools.rst new file mode 100644 index 0000000000..c19c41eea3 --- /dev/null +++ b/docs/source/pype.lib.path_tools.rst @@ -0,0 +1,7 @@ +pype.lib.path\_tools module +=========================== + +.. automodule:: pype.lib.path_tools + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.lib.plugin_tools.rst b/docs/source/pype.lib.plugin_tools.rst new file mode 100644 index 0000000000..6eadc5d3be --- /dev/null +++ b/docs/source/pype.lib.plugin_tools.rst @@ -0,0 +1,7 @@ +pype.lib.plugin\_tools module +============================= + +.. automodule:: pype.lib.plugin_tools + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.lib.python_module_tools.rst b/docs/source/pype.lib.python_module_tools.rst new file mode 100644 index 0000000000..c916080bce --- /dev/null +++ b/docs/source/pype.lib.python_module_tools.rst @@ -0,0 +1,7 @@ +pype.lib.python\_module\_tools module +===================================== + +.. automodule:: pype.lib.python_module_tools + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.lib.terminal_splash.rst b/docs/source/pype.lib.terminal_splash.rst new file mode 100644 index 0000000000..06038f0f09 --- /dev/null +++ b/docs/source/pype.lib.terminal_splash.rst @@ -0,0 +1,7 @@ +pype.lib.terminal\_splash module +================================ + +.. automodule:: pype.lib.terminal_splash + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.modules.clockify.clockify_module.rst b/docs/source/pype.modules.clockify.clockify_module.rst new file mode 100644 index 0000000000..85f8e75ad1 --- /dev/null +++ b/docs/source/pype.modules.clockify.clockify_module.rst @@ -0,0 +1,7 @@ +pype.modules.clockify.clockify\_module module +============================================= + +.. automodule:: pype.modules.clockify.clockify_module + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.modules.deadline.deadline_module.rst b/docs/source/pype.modules.deadline.deadline_module.rst new file mode 100644 index 0000000000..43e7198a8b --- /dev/null +++ b/docs/source/pype.modules.deadline.deadline_module.rst @@ -0,0 +1,7 @@ +pype.modules.deadline.deadline\_module module +============================================= + +.. automodule:: pype.modules.deadline.deadline_module + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.modules.deadline.rst b/docs/source/pype.modules.deadline.rst new file mode 100644 index 0000000000..7633b2b950 --- /dev/null +++ b/docs/source/pype.modules.deadline.rst @@ -0,0 +1,15 @@ +pype.modules.deadline package +============================= + +.. automodule:: pype.modules.deadline + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +.. toctree:: + :maxdepth: 10 + + pype.modules.deadline.deadline_module diff --git a/docs/source/pype.modules.ftrack.ftrack_module.rst b/docs/source/pype.modules.ftrack.ftrack_module.rst new file mode 100644 index 0000000000..4188ffbed8 --- /dev/null +++ b/docs/source/pype.modules.ftrack.ftrack_module.rst @@ -0,0 +1,7 @@ +pype.modules.ftrack.ftrack\_module module +========================================= + +.. automodule:: pype.modules.ftrack.ftrack_module + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.modules.ftrack.lib.settings.rst b/docs/source/pype.modules.ftrack.lib.settings.rst new file mode 100644 index 0000000000..255d52178a --- /dev/null +++ b/docs/source/pype.modules.ftrack.lib.settings.rst @@ -0,0 +1,7 @@ +pype.modules.ftrack.lib.settings module +======================================= + +.. automodule:: pype.modules.ftrack.lib.settings + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.modules.ftrack.tray.ftrack_tray.rst b/docs/source/pype.modules.ftrack.tray.ftrack_tray.rst new file mode 100644 index 0000000000..147647e9b4 --- /dev/null +++ b/docs/source/pype.modules.ftrack.tray.ftrack_tray.rst @@ -0,0 +1,7 @@ +pype.modules.ftrack.tray.ftrack\_tray module +============================================ + +.. automodule:: pype.modules.ftrack.tray.ftrack_tray + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.modules.launcher_action.rst b/docs/source/pype.modules.launcher_action.rst new file mode 100644 index 0000000000..a63408e747 --- /dev/null +++ b/docs/source/pype.modules.launcher_action.rst @@ -0,0 +1,7 @@ +pype.modules.launcher\_action module +==================================== + +.. automodule:: pype.modules.launcher_action + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.modules.log_viewer.log_view_module.rst b/docs/source/pype.modules.log_viewer.log_view_module.rst new file mode 100644 index 0000000000..8d80170a9c --- /dev/null +++ b/docs/source/pype.modules.log_viewer.log_view_module.rst @@ -0,0 +1,7 @@ +pype.modules.log\_viewer.log\_view\_module module +================================================= + +.. automodule:: pype.modules.log_viewer.log_view_module + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.modules.log_viewer.rst b/docs/source/pype.modules.log_viewer.rst new file mode 100644 index 0000000000..e275d56086 --- /dev/null +++ b/docs/source/pype.modules.log_viewer.rst @@ -0,0 +1,23 @@ +pype.modules.log\_viewer package +================================ + +.. automodule:: pype.modules.log_viewer + :members: + :undoc-members: + :show-inheritance: + +Subpackages +----------- + +.. toctree:: + :maxdepth: 10 + + pype.modules.log_viewer.tray + +Submodules +---------- + +.. toctree:: + :maxdepth: 10 + + pype.modules.log_viewer.log_view_module diff --git a/docs/source/pype.modules.log_viewer.tray.app.rst b/docs/source/pype.modules.log_viewer.tray.app.rst new file mode 100644 index 0000000000..0948a05594 --- /dev/null +++ b/docs/source/pype.modules.log_viewer.tray.app.rst @@ -0,0 +1,7 @@ +pype.modules.log\_viewer.tray.app module +======================================== + +.. automodule:: pype.modules.log_viewer.tray.app + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.modules.log_viewer.tray.models.rst b/docs/source/pype.modules.log_viewer.tray.models.rst new file mode 100644 index 0000000000..4da3887600 --- /dev/null +++ b/docs/source/pype.modules.log_viewer.tray.models.rst @@ -0,0 +1,7 @@ +pype.modules.log\_viewer.tray.models module +=========================================== + +.. automodule:: pype.modules.log_viewer.tray.models + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.modules.log_viewer.tray.rst b/docs/source/pype.modules.log_viewer.tray.rst new file mode 100644 index 0000000000..5f4b92f627 --- /dev/null +++ b/docs/source/pype.modules.log_viewer.tray.rst @@ -0,0 +1,17 @@ +pype.modules.log\_viewer.tray package +===================================== + +.. automodule:: pype.modules.log_viewer.tray + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +.. toctree:: + :maxdepth: 10 + + pype.modules.log_viewer.tray.app + pype.modules.log_viewer.tray.models + pype.modules.log_viewer.tray.widgets diff --git a/docs/source/pype.modules.log_viewer.tray.widgets.rst b/docs/source/pype.modules.log_viewer.tray.widgets.rst new file mode 100644 index 0000000000..cb57c96559 --- /dev/null +++ b/docs/source/pype.modules.log_viewer.tray.widgets.rst @@ -0,0 +1,7 @@ +pype.modules.log\_viewer.tray.widgets module +============================================ + +.. automodule:: pype.modules.log_viewer.tray.widgets + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.modules.settings_action.rst b/docs/source/pype.modules.settings_action.rst new file mode 100644 index 0000000000..10f0881ced --- /dev/null +++ b/docs/source/pype.modules.settings_action.rst @@ -0,0 +1,7 @@ +pype.modules.settings\_action module +==================================== + +.. automodule:: pype.modules.settings_action + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.modules.standalonepublish_action.rst b/docs/source/pype.modules.standalonepublish_action.rst new file mode 100644 index 0000000000..d51dbcefa0 --- /dev/null +++ b/docs/source/pype.modules.standalonepublish_action.rst @@ -0,0 +1,7 @@ +pype.modules.standalonepublish\_action module +============================================= + +.. automodule:: pype.modules.standalonepublish_action + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.modules.sync_server.rst b/docs/source/pype.modules.sync_server.rst new file mode 100644 index 0000000000..a26dc7e212 --- /dev/null +++ b/docs/source/pype.modules.sync_server.rst @@ -0,0 +1,16 @@ +pype.modules.sync\_server package +================================= + +.. automodule:: pype.modules.sync_server + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +.. toctree:: + :maxdepth: 10 + + pype.modules.sync_server.sync_server + pype.modules.sync_server.utils diff --git a/docs/source/pype.modules.sync_server.sync_server.rst b/docs/source/pype.modules.sync_server.sync_server.rst new file mode 100644 index 0000000000..36d6aa68ed --- /dev/null +++ b/docs/source/pype.modules.sync_server.sync_server.rst @@ -0,0 +1,7 @@ +pype.modules.sync\_server.sync\_server module +============================================= + +.. automodule:: pype.modules.sync_server.sync_server + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.modules.sync_server.utils.rst b/docs/source/pype.modules.sync_server.utils.rst new file mode 100644 index 0000000000..325d5e435d --- /dev/null +++ b/docs/source/pype.modules.sync_server.utils.rst @@ -0,0 +1,7 @@ +pype.modules.sync\_server.utils module +====================================== + +.. automodule:: pype.modules.sync_server.utils + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.modules.websocket_server.hosts.aftereffects.rst b/docs/source/pype.modules.websocket_server.hosts.aftereffects.rst new file mode 100644 index 0000000000..9f4720ae14 --- /dev/null +++ b/docs/source/pype.modules.websocket_server.hosts.aftereffects.rst @@ -0,0 +1,7 @@ +pype.modules.websocket\_server.hosts.aftereffects module +======================================================== + +.. automodule:: pype.modules.websocket_server.hosts.aftereffects + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.plugins.maya.publish.validate_vray_referenced_aovs.rst b/docs/source/pype.plugins.maya.publish.validate_vray_referenced_aovs.rst new file mode 100644 index 0000000000..16ad9666aa --- /dev/null +++ b/docs/source/pype.plugins.maya.publish.validate_vray_referenced_aovs.rst @@ -0,0 +1,7 @@ +pype.plugins.maya.publish.validate\_vray\_referenced\_aovs module +================================================================= + +.. automodule:: pype.plugins.maya.publish.validate_vray_referenced_aovs + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.settings.constants.rst b/docs/source/pype.settings.constants.rst new file mode 100644 index 0000000000..ac652089c8 --- /dev/null +++ b/docs/source/pype.settings.constants.rst @@ -0,0 +1,7 @@ +pype.settings.constants module +============================== + +.. automodule:: pype.settings.constants + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.settings.handlers.rst b/docs/source/pype.settings.handlers.rst new file mode 100644 index 0000000000..60ea0ae952 --- /dev/null +++ b/docs/source/pype.settings.handlers.rst @@ -0,0 +1,7 @@ +pype.settings.handlers module +============================= + +.. automodule:: pype.settings.handlers + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.tests.test_lib_restructuralization.rst b/docs/source/pype.tests.test_lib_restructuralization.rst new file mode 100644 index 0000000000..8d426fcb6b --- /dev/null +++ b/docs/source/pype.tests.test_lib_restructuralization.rst @@ -0,0 +1,7 @@ +pype.tests.test\_lib\_restructuralization module +================================================ + +.. automodule:: pype.tests.test_lib_restructuralization + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.tools.tray.pype_tray.rst b/docs/source/pype.tools.tray.pype_tray.rst new file mode 100644 index 0000000000..9fc49c5763 --- /dev/null +++ b/docs/source/pype.tools.tray.pype_tray.rst @@ -0,0 +1,7 @@ +pype.tools.tray.pype\_tray module +================================= + +.. automodule:: pype.tools.tray.pype_tray + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.tools.tray.rst b/docs/source/pype.tools.tray.rst new file mode 100644 index 0000000000..b28059d170 --- /dev/null +++ b/docs/source/pype.tools.tray.rst @@ -0,0 +1,15 @@ +pype.tools.tray package +======================= + +.. automodule:: pype.tools.tray + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +.. toctree:: + :maxdepth: 10 + + pype.tools.tray.pype_tray diff --git a/docs/source/pype.tools.workfiles.app.rst b/docs/source/pype.tools.workfiles.app.rst new file mode 100644 index 0000000000..a3a46b8a07 --- /dev/null +++ b/docs/source/pype.tools.workfiles.app.rst @@ -0,0 +1,7 @@ +pype.tools.workfiles.app module +=============================== + +.. automodule:: pype.tools.workfiles.app + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.tools.workfiles.model.rst b/docs/source/pype.tools.workfiles.model.rst new file mode 100644 index 0000000000..44cea32b97 --- /dev/null +++ b/docs/source/pype.tools.workfiles.model.rst @@ -0,0 +1,7 @@ +pype.tools.workfiles.model module +================================= + +.. automodule:: pype.tools.workfiles.model + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.tools.workfiles.rst b/docs/source/pype.tools.workfiles.rst new file mode 100644 index 0000000000..147c4cebbe --- /dev/null +++ b/docs/source/pype.tools.workfiles.rst @@ -0,0 +1,17 @@ +pype.tools.workfiles package +============================ + +.. automodule:: pype.tools.workfiles + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +.. toctree:: + :maxdepth: 10 + + pype.tools.workfiles.app + pype.tools.workfiles.model + pype.tools.workfiles.view diff --git a/docs/source/pype.tools.workfiles.view.rst b/docs/source/pype.tools.workfiles.view.rst new file mode 100644 index 0000000000..acd32ed250 --- /dev/null +++ b/docs/source/pype.tools.workfiles.view.rst @@ -0,0 +1,7 @@ +pype.tools.workfiles.view module +================================ + +.. automodule:: pype.tools.workfiles.view + :members: + :undoc-members: + :show-inheritance: diff --git a/igniter/__init__.py b/igniter/__init__.py index 8de58cd6d4..ffac2b023f 100644 --- a/igniter/__init__.py +++ b/igniter/__init__.py @@ -9,8 +9,10 @@ from .bootstrap_repos import BootstrapRepos def run(): + """Show Igniter dialog.""" app = QtWidgets.QApplication(sys.argv) d = InstallDialog() + d.exec_() d.show() sys.exit(app.exec_()) diff --git a/igniter/bootstrap_repos.py b/igniter/bootstrap_repos.py index 999a6daa19..38de3007b4 100644 --- a/igniter/bootstrap_repos.py +++ b/igniter/bootstrap_repos.py @@ -1,21 +1,20 @@ # -*- coding: utf-8 -*- """Bootstrap Pype repositories.""" -import sys +import functools +import logging as log import os import re -import logging as log import shutil +import sys import tempfile -from typing import Union, Callable, List -from zipfile import ZipFile from pathlib import Path -import functools - -from speedcopy import copyfile +from typing import Union, Callable, List, Tuple +from zipfile import ZipFile, BadZipFile from appdirs import user_data_dir -from pype.version import __version__ -from pype.lib import PypeSettingsRegistry +from speedcopy import copyfile + +from .user_settings import PypeSettingsRegistry from .tools import load_environments @@ -24,21 +23,24 @@ class PypeVersion: """Class for storing information about Pype version. Attributes: - major (int): [1].2.3-variant-client - minor (int): 1.[2].3-variant-client - subversion (int): 1.2.[3]-variant-client - variant (str): 1.2.3-[variant]-client - client (str): 1.2.3-variant-[client] + major (int): [1].2.3-client-variant + minor (int): 1.[2].3-client-variant + subversion (int): 1.2.[3]-client-variant + client (str): 1.2.3-[client]-variant + variant (str): 1.2.3-client-[variant] path (str): path to Pype """ major = 0 minor = 0 subversion = 0 - variant = "production" + variant = "" client = None path = None + _version_regex = re.compile( + r"(?P\d+)\.(?P\d+)\.(?P\d+)(-(?Pstaging)|-(?P.+)(-(?Pstaging)))?") # noqa: E501 + @property def version(self): """return formatted version string.""" @@ -55,15 +57,14 @@ class PypeVersion: def __init__(self, major: int = None, minor: int = None, subversion: int = None, version: str = None, - variant: str = "production", client: str = None, + variant: str = "", client: str = None, path: Path = None): self.path = path - self._version_regex = re.compile( - r"(?P\d+)\.(?P\d+)\.(?P\d+)(-?((?Pstaging)|(?P.+))(-(?P.+))?)?") # noqa: E501 - if major is None or minor is None or subversion is None: - if version is None: - raise ValueError("Need version specified in some way.") + if ( + major is None or minor is None or subversion is None + ) and version is None: + raise ValueError("Need version specified in some way.") if version: values = self._decompose_version(version) self.major = values[0] @@ -83,25 +84,27 @@ class PypeVersion: def _compose_version(self): version = "{}.{}.{}".format(self.major, self.minor, self.subversion) - if self.variant == "staging": - version = "{}-{}".format(version, self.variant) if self.client: version = "{}-{}".format(version, self.client) + if self.variant == "staging": + version = "{}-{}".format(version, self.variant) + return version - def _decompose_version(self, version_string: str) -> tuple: - m = re.match(self._version_regex, version_string) + @classmethod + def _decompose_version(cls, version_string: str) -> tuple: + m = re.search(cls._version_regex, version_string) if not m: raise ValueError( "Cannot parse version string: {}".format(version_string)) variant = None - if m.group("variant") == "staging": + if m.group("var1") == "staging" or m.group("var2") == "staging": variant = "staging" - client = m.group("client") or m.group("cli") + client = m.group("client") return (int(m.group("major")), int(m.group("minor")), int(m.group("sub")), variant, client) @@ -122,21 +125,85 @@ class PypeVersion: return hash(self.version) def __lt__(self, other): - if self.major < other.major: + if (self.major, self.minor, self.subversion) < \ + (other.major, other.minor, other.subversion): return True - if self.major <= other.major and self.minor < other.minor: - return True - if self.major <= other.major and self.minor <= other.minor and \ - self.subversion < other.subversion: + # 1.2.3-staging < 1.2.3-client-staging + if self.get_main_version() == other.get_main_version() and \ + not self.client and self.variant and \ + other.client and other.variant: return True - if self.major == other.major and self.minor == other.minor and \ - self.subversion == other.subversion and \ - self.variant == "staging": + # 1.2.3 < 1.2.3-staging + if self.get_main_version() == other.get_main_version() and \ + not self.client and self.variant and \ + not other.client and not other.variant: return True - return False + # 1.2.3 < 1.2.3-client + if self.get_main_version() == other.get_main_version() and \ + not self.client and not self.variant and \ + other.client and not other.variant: + return True + + # 1.2.3 < 1.2.3-client-staging + if self.get_main_version() == other.get_main_version() and \ + not self.client and not self.variant and other.client: + return True + + # 1.2.3-client-staging < 1.2.3-client + if self.get_main_version() == other.get_main_version() and \ + self.client and self.variant and \ + other.client and not other.variant: + return True + + # prefer path over no path + if self.version == other.version and \ + not self.path and other.path: + return True + + # prefer path with dir over path with file + return self.version == other.version and self.path and \ + other.path and self.path.is_file() and \ + other.path.is_dir() + + def is_staging(self) -> bool: + """Test if current version is staging one.""" + return self.variant == "staging" + + def get_main_version(self) -> str: + """Return main version component. + + This returns x.x.x part of version from possibly more complex one + like x.x.x-foo-bar. + + Returns: + str: main version component + + """ + return "{}.{}.{}".format(self.major, self.minor, self.subversion) + + @staticmethod + def version_in_str(string: str) -> Tuple: + """Find Pype version in given string. + + Args: + string (str): string to search. + + Returns: + tuple: True/False and PypeVersion if found. + + """ + try: + result = PypeVersion._decompose_version(string) + except ValueError: + return False, None + return True, PypeVersion(major=result[0], + minor=result[1], + subversion=result[2], + variant=result[3], + client=result[4]) class BootstrapRepos: @@ -146,15 +213,20 @@ class BootstrapRepos: data_dir (Path): local Pype installation directory. live_repo_dir (Path): path to repos directory if running live, otherwise `None`. + registry (PypeSettingsRegistry): Pype registry object. + zip_filter (list): List of files to exclude from zip + pype_filter (list): list of top level directories not to include in + zip in Pype repository. """ - def __init__(self, progress_callback: Callable = None): + def __init__(self, progress_callback: Callable = None, message=None): """Constructor. Args: progress_callback (callable): Optional callback method to report progress. + message (QtCore.Signal, optional): Signal to report messages back. """ # vendor and app used to construct user data dir @@ -163,9 +235,15 @@ class BootstrapRepos: self._log = log.getLogger(str(__class__)) self.data_dir = Path(user_data_dir(self._app, self._vendor)) self.registry = PypeSettingsRegistry() + self.zip_filter = [".pyc", "__pycache__"] + self.pype_filter = [ + "build", "docs", "tests", "repos", "tools", "venv" + ] + self._message = message # dummy progress reporter def empty_progress(x: int): + """Progress callback dummy.""" return x if not progress_callback: @@ -194,9 +272,14 @@ class BootstrapRepos: return v.path @staticmethod - def get_local_version() -> str: + def get_local_live_version() -> str: """Get version of local Pype.""" - return __version__ + + version = {} + path = Path(os.path.dirname(__file__)).parent / "pype" / "version.py" + with open(path, "r") as fp: + exec(fp.read(), version) + return version["__version__"] @staticmethod def get_version(repo_dir: Path) -> Union[str, None]: @@ -225,7 +308,7 @@ class BootstrapRepos: """Copy zip created from Pype repositories to user data dir. This detect Pype version either in local "live" Pype repository - or in user provided path. Then it will zip in in temporary directory + or in user provided path. Then it will zip it in temporary directory and finally it will move it to destination which is user data directory. Existing files will be replaced. @@ -240,7 +323,7 @@ class BootstrapRepos: # version and use it as a source. Otherwise repo_dir is user # entered location. if not repo_dir: - version = self.get_local_version() + version = self.get_local_live_version() repo_dir = self.live_repo_dir else: version = self.get_version(repo_dir) @@ -252,7 +335,7 @@ class BootstrapRepos: # create zip inside temporary directory. with tempfile.TemporaryDirectory() as temp_dir: temp_zip = \ - Path(temp_dir) / f"pype-repositories-v{version}.zip" + Path(temp_dir) / f"pype-v{version}.zip" self._log.info(f"creating zip: {temp_zip}") self._create_pype_zip(temp_zip, repo_dir) @@ -275,7 +358,7 @@ class BootstrapRepos: except shutil.Error as e: self._log.error(e) return None - return self.data_dir / temp_zip.name + return destination def _create_pype_zip( self, @@ -284,11 +367,9 @@ class BootstrapRepos: """Pack repositories and Pype into zip. We are using :mod:`zipfile` instead :meth:`shutil.make_archive` - to later implement file filter to skip git related stuff to make - it into archive. - - Todo: - Implement file filter + because we need to decide what file and directories to include in zip + and what not. They are determined by :attr:`zip_filter` on file level + and :attr:`pype_filter` on top level directory in Pype repository. Args: zip_path (str): path to zip file. @@ -296,42 +377,82 @@ class BootstrapRepos: include_pype (bool): add Pype module itself. """ - repo_files = sum(len(files) for _, _, files in os.walk(include_dir)) + include_dir = include_dir.resolve() + + def _filter_dir(path: Path, path_filter: List) -> List[Path]: + """Recursively crawl over path and filter.""" + result = [] + for item in path.iterdir(): + if item.name in path_filter: + continue + if item.name.startswith('.'): + continue + if item.is_dir(): + result.extend(_filter_dir(item, path_filter)) + else: + result.append(item) + return result + + pype_list = [] + # get filtered list of files in repositories (repos directory) + repo_list = _filter_dir(include_dir, self.zip_filter) + # count them + repo_files = len(repo_list) + + # there must be some files, otherwise `include_dir` path is wrong assert repo_files != 0, f"No repositories to include in {include_dir}" pype_inc = 0 if include_pype: - pype_files = sum(len(files) for _, _, files in os.walk( - include_dir.parent)) + # get filtered list of file in Pype repository + pype_list = _filter_dir(include_dir.parent, self.zip_filter) + pype_files = len(pype_list) repo_inc = 48.0 / float(repo_files) pype_inc = 48.0 / float(pype_files) else: repo_inc = 98.0 / float(repo_files) progress = 0 + with ZipFile(zip_path, "w") as zip_file: - for root, _, files in os.walk(include_dir.as_posix()): - for file in files: - zip_file.write( - os.path.relpath(os.path.join(root, file), - os.path.join(include_dir, '..')), - os.path.relpath(os.path.join(root, file), - os.path.join(include_dir)) - ) - progress += repo_inc - self._progress_callback(int(progress)) + file: Path + for file in repo_list: + progress += repo_inc + self._progress_callback(int(progress)) + + # archive name is relative to repos dir + arc_name = file.relative_to(include_dir) + zip_file.write(file, arc_name) + # add pype itself if include_pype: - for root, _, files in os.walk("pype"): - for file in files: - zip_file.write( - os.path.relpath(os.path.join(root, file), - os.path.join('pype', '..')), - os.path.join( - 'pype', - os.path.relpath(os.path.join(root, file), - os.path.join('pype', '..'))) - ) - progress += pype_inc - self._progress_callback(int(progress)) + pype_root = include_dir.parent.resolve() + # generate list of filtered paths + dir_filter = [pype_root / f for f in self.pype_filter] + + file: Path + for file in pype_list: + progress += pype_inc + self._progress_callback(int(progress)) + + # if file resides in filtered path, skip it + is_inside = None + df: Path + for df in dir_filter: + try: + is_inside = file.resolve().relative_to(df) + except ValueError: + pass + + if is_inside: + continue + + processed_path = file + self._log.debug(f"processing {processed_path}") + self._print(f"- processing {processed_path}", False) + + zip_file.write(file, + "pype" / file.relative_to(pype_root)) + + # test if zip is ok zip_file.testzip() self._progress_callback(100) @@ -342,10 +463,15 @@ class BootstrapRepos: This will enable Python to import modules is second-level directories in zip file. + Adding to both `sys.path` and `PYTHONPATH`, skipping duplicates. + Args: - archive (str): path to archive. + archive (Path): path to archive. """ + if not archive.is_file() and not archive.exists(): + raise ValueError("Archive is not file.") + with ZipFile(archive, "r") as zip_file: name_list = zip_file.namelist() @@ -362,8 +488,41 @@ class BootstrapRepos: os.environ["PYTHONPATH"] = os.pathsep.join(paths) + @staticmethod + def add_paths_from_directory(directory: Path) -> None: + """Add first level directories as paths to :mod:`sys.path`. + + This works the same as :meth:`add_paths_from_archive` but in + specified directory. + + Adding to both `sys.path` and `PYTHONPATH`, skipping duplicates. + + Args: + directory (Path): path to directory. + + """ + if not directory.exists() and not directory.is_dir(): + raise ValueError("directory is invalid") + + roots = [] + for item in directory.iterdir(): + if item.is_dir(): + root = item.as_posix() + if root not in roots: + roots.append(root) + sys.path.insert(0, root) + + pythonpath = os.getenv("PYTHONPATH", "") + paths = pythonpath.split(os.pathsep) + paths += roots + + os.environ["PYTHONPATH"] = os.pathsep.join(paths) + def find_pype( - self, pype_path: Path = None) -> Union[List[PypeVersion], None]: + self, + pype_path: Path = None, + staging: bool = False, + include_zips: bool = False) -> Union[List[PypeVersion], None]: """Get ordered dict of detected Pype version. Resolution order for Pype is following: @@ -374,6 +533,10 @@ class BootstrapRepos: Args: pype_path (Path, optional): Try to find Pype on the given path. + staging (bool, optional): Filter only staging version, skip them + otherwise. + include_zips (bool, optional): If set True it will try to find + Pype in zip files in given directory. Returns: dict of Path: Dictionary of detected Pype version. @@ -383,42 +546,110 @@ class BootstrapRepos: """ dir_to_search = self.data_dir - if os.getenv("PYPE_PATH"): - if Path(os.getenv("PYPE_PATH")).exists(): - dir_to_search = Path(os.getenv("PYPE_PATH")) - else: - try: - registry_dir = Path(self.registry.get_item("pypePath")) - if registry_dir.exists(): - dir_to_search = registry_dir - except ValueError: - # nothing found in registry, we'll use data dir - pass - - # if we have pyp_path specified, search only there. + # if we have pype_path specified, search only there. if pype_path: dir_to_search = pype_path + else: + if os.getenv("PYPE_PATH"): + if Path(os.getenv("PYPE_PATH")).exists(): + dir_to_search = Path(os.getenv("PYPE_PATH")) + else: + try: + registry_dir = Path( + str(self.registry.get_item("pypePath"))) + if registry_dir.exists(): + dir_to_search = registry_dir + + except ValueError: + # nothing found in registry, we'll use data dir + pass # pype installation dir doesn't exists if not dir_to_search.exists(): return None _pype_versions = [] - file_pattern = re.compile(r"^pype-repositories-v(?P\d+\.\d+\.\d*.+?).zip$") # noqa: E501 + # iterate over directory in first level and find all that might + # contain Pype. for file in dir_to_search.iterdir(): - m = re.match( - file_pattern, - file.name) - if m: - try: - _pype_versions.append( - PypeVersion( - version=m.group("version"), path=file)) - except ValueError: - # cannot parse version string - print(m) - pass + + # if file, strip extension, in case of dir not. + name = file.name if file.is_dir() else file.stem + result = PypeVersion.version_in_str(name) + + if result[0]: + detected_version: PypeVersion + detected_version = result[1] + + if file.is_dir(): + # if item is directory that might (based on it's name) + # contain Pype version, check if it really does contain + # Pype and that their versions matches. + try: + # add one 'pype' level as inside dir there should + # be many other repositories. + version_str = BootstrapRepos.get_version( + file / "pype") + version_check = PypeVersion(version=version_str) + except ValueError: + self._log.error( + f"cannot determine version from {file}") + continue + + version_main = version_check.get_main_version() + detected_main = detected_version.get_main_version() + if version_main != detected_main: + self._log.error( + (f"dir version ({detected_version}) and " + f"its content version ({version_check}) " + "doesn't match. Skipping.")) + continue + + if file.is_file(): + + if not include_zips: + continue + + # skip non-zip files + if file.suffix.lower() != ".zip": + continue + + # open zip file, look inside and parse version from Pype + # inside it. If there is none, or it is different from + # version specified in file name, skip it. + try: + with ZipFile(file, "r") as zip_file: + with zip_file.open( + "pype/pype/version.py") as version_file: + zip_version = {} + exec(version_file.read(), zip_version) + version_check = PypeVersion( + version=zip_version["__version__"]) + + version_main = version_check.get_main_version() # noqa: E501 + detected_main = detected_version.get_main_version() # noqa: E501 + + if version_main != detected_main: + self._log.error( + (f"zip version ({detected_version}) " + f"and its content version " + f"({version_check}) " + "doesn't match. Skipping.")) + continue + except BadZipFile: + self._log.error(f"{file} is not zip file") + continue + except KeyError: + self._log.error("Zip not containing Pype") + continue + + detected_version.path = file + if staging and detected_version.is_staging(): + _pype_versions.append(detected_version) + + if not staging and not detected_version.is_staging(): + _pype_versions.append(detected_version) return sorted(_pype_versions) @@ -426,16 +657,16 @@ class BootstrapRepos: def _get_pype_from_mongo(mongo_url: str) -> Union[Path, None]: """Get path from Mongo database. - This sets environment variable ``AVALON_MONGO`` for + This sets environment variable ``PYPE_MONGO`` for :mod:`pype.settings` to be able to read data from database. It will then retrieve environment variables and among them - must be ``PYPE_ROOT``. + must be ``PYPE_PATH``. Args: mongo_url (str): mongodb connection url Returns: - Path: if path from ``PYPE_ROOT`` is found. + Path: if path from ``PYPE_PATH`` is found. None: if not. """ @@ -479,11 +710,18 @@ class BootstrapRepos: self._log.error(f"{pype_path} doesn't exists.") return None - # find pype zip files in location. In that location, there can be - # either "live" Pype repository, or multiple zip files. + # test if entered path isn't user data dir + if self.data_dir == pype_path: + self._log.error("cannot point to user data dir") + return None + + # find pype zip files in location. There can be + # either "live" Pype repository, or multiple zip files or even + # multiple pype version directories. This process looks into zip + # files and directories and tries to parse `version.py` file. versions = self.find_pype(pype_path) if versions: - self._log.info(f"found Pype zips in [ {pype_path} ].") + self._log.info(f"found Pype in [ {pype_path} ]") self._log.info(f"latest version found is [ {versions[-1]} ]") destination = self.data_dir / versions[-1].path.name @@ -503,13 +741,43 @@ class BootstrapRepos: if not destination.parent.exists(): destination.parent.mkdir(parents=True) + # latest version found is directory + if versions[-1].path.is_dir(): + # zip it, copy it and extract it + # create zip inside temporary directory. + self._log.info("Creating zip from directory ...") + with tempfile.TemporaryDirectory() as temp_dir: + temp_zip = \ + Path(temp_dir) / f"pype-v{versions[-1]}.zip" + self._log.info(f"creating zip: {temp_zip}") + + self._create_pype_zip(temp_zip, versions[-1].path) + if not os.path.exists(temp_zip): + self._log.error("make archive failed.") + return None + + destination = self.data_dir / temp_zip.name + + elif versions[-1].path.is_file(): + # in this place, it must be zip file as `find_pype()` is + # checking just that. + assert versions[-1].path.suffix.lower() == ".zip", ( + "Invalid file format" + ) try: + self._log.info("Copying zip to destination ...") copyfile(versions[-1].path.as_posix(), destination.as_posix()) except OSError: self._log.error( "cannot copy detected version to user data directory", exc_info=True) return None + + # extract zip there + self._log.info("extracting zip to destination ...") + with ZipFile(versions[-1].path, "r") as zip_ref: + zip_ref.extractall(destination) + return destination # if we got here, it means that location is "live" Pype repository. @@ -518,4 +786,171 @@ class BootstrapRepos: if not repo_file.exists(): self._log.error(f"installing zip {repo_file} failed.") return None - return repo_file + + destination = self.data_dir / repo_file.stem + if destination.exists(): + try: + destination.unlink() + except OSError: + self._log.error( + f"cannot remove already existing {destination}", + exc_info=True) + return None + + destination.mkdir(parents=True) + + # extract zip there + self._log.info("extracting zip to destination ...") + with ZipFile(versions[-1].path, "r") as zip_ref: + zip_ref.extractall(destination) + + return destination + + def _print(self, message, error=False): + if self._message: + self._message.emit(message, error) + + def extract_pype(self, version: PypeVersion) -> Union[Path, None]: + """Extract zipped Pype version to user data directory. + + Args: + version (PypeVersion): Version of Pype. + + Returns: + Path: path to extracted version. + None: if something failed. + + """ + if not version.path: + raise ValueError( + f"version {version} is not associated with any file") + + destination = self.data_dir / version.path.stem + if destination.exists(): + try: + destination.unlink() + except OSError as e: + msg = f"!!! Cannot remove already existing {destination}" + self._log.error(msg) + self._log.error(e.strerror) + self._print(msg, True) + self._print(e.strerror, True) + return None + + destination.mkdir(parents=True) + + # extract zip there + self._print("Extracting zip to destination ...") + with ZipFile(version.path, "r") as zip_ref: + zip_ref.extractall(destination) + + self._print(f"Installed as {version.path.stem}") + + return destination + + def install_version(self, pype_version: PypeVersion, force: bool = False): + """Install Pype version to user data directory. + + Args: + pype_version (PypeVersion): Pype version to install. + force (bool, optional): Force overwrite existing version. + + Returns: + Path: Path to installed Pype. + + Raises: + PypeVersionExists: If not forced and this version already exist + in user data directory. + PypeVersionInvalid: If version to install is invalid. + PypeVersionIOError: If copying or zipping fail. + + """ + + # test if version is located (in user data dir) + is_inside = False + try: + is_inside = pype_version.path.resolve().relative_to( + self.data_dir) + except ValueError: + # if relative path cannot be calculated, Pype version is not + # inside user data dir + pass + + if is_inside: + raise PypeVersionExists("Pype already inside user data dir") + + # determine destination directory name + # for zip file strip suffix + destination = self.data_dir / pype_version.path.stem + + # test if destination file already exist, if so lets delete it. + # we consider path on location as authoritative place. + if destination.exists() and force: + try: + destination.unlink() + except OSError: + self._log.error( + f"cannot remove already existing {destination}", + exc_info=True) + return None + else: + raise PypeVersionExists(f"{destination} already exist.") + + # create destination parent directories even if they don't exist. + if not destination.exists(): + destination.mkdir(parents=True) + + # version is directory + if pype_version.path.is_dir(): + # create zip inside temporary directory. + self._log.info("Creating zip from directory ...") + with tempfile.TemporaryDirectory() as temp_dir: + temp_zip = \ + Path(temp_dir) / f"pype-v{pype_version}.zip" + self._log.info(f"creating zip: {temp_zip}") + + self._create_pype_zip(temp_zip, pype_version.path) + if not os.path.exists(temp_zip): + self._log.error("make archive failed.") + raise PypeVersionIOError("Zip creation failed.") + + # set zip as version source + pype_version.path = temp_zip + + elif pype_version.path.is_file(): + # check if file is zip (by extension) + if pype_version.path.suffix.lower() != ".zip": + raise PypeVersionInvalid("Invalid file format") + + try: + # copy file to destination + self._log.info("Copying zip to destination ...") + copyfile(pype_version.path.as_posix(), destination.as_posix()) + except OSError as e: + self._log.error( + "cannot copy version to user data directory", + exc_info=True) + raise PypeVersionIOError( + "can't copy version to destination") from e + + # extract zip there + self._log.info("extracting zip to destination ...") + with ZipFile(pype_version.path, "r") as zip_ref: + zip_ref.extractall(destination) + + return destination + + +class PypeVersionExists(Exception): + """Exception for handling existing Pype version.""" + pass + + +class PypeVersionInvalid(Exception): + """Exception for handling invalid Pype version.""" + pass + + +class PypeVersionIOError(Exception): + """Exception for handling IO errors in Pype version.""" + pass diff --git a/igniter/install_dialog.py b/igniter/install_dialog.py index 93c94977c6..b4fa68d89a 100644 --- a/igniter/install_dialog.py +++ b/igniter/install_dialog.py @@ -19,7 +19,7 @@ class InstallDialog(QtWidgets.QDialog): def __init__(self, parent=None): super(InstallDialog, self).__init__(parent) - self._mongo_url = "" + self._mongo_url = os.getenv("PYPE_MONGO", "") self.setWindowTitle("Pype - Configure Pype repository path") self._icon_path = os.path.join( @@ -149,10 +149,13 @@ class InstallDialog(QtWidgets.QDialog): self.setLayout(mongo_layout) def _mongo_changed(self, mongo: str): - self.parent()._mongo_url = mongo + self.parent().mongo_url = mongo def get_mongo_url(self): - return self.parent()._mongo_url + return self.parent().mongo_url + + def set_mongo_url(self, mongo: str): + self._mongo_input.setText(mongo) def set_valid(self): self._mongo_input.setStyleSheet( @@ -175,6 +178,8 @@ class InstallDialog(QtWidgets.QDialog): ) self._mongo = MongoWidget(self) + if self._mongo_url: + self._mongo.set_mongo_url(self._mongo_url) # Bottom button bar # -------------------------------------------------------------------- @@ -303,14 +308,17 @@ class InstallDialog(QtWidgets.QDialog): options |= QtWidgets.QFileDialog.DontUseNativeDialog options |= QtWidgets.QFileDialog.ShowDirsOnly - filename, _ = QtWidgets.QFileDialog.getOpenFileName( + result = QtWidgets.QFileDialog.getExistingDirectory( parent=self, caption='Select path', directory=os.getcwd(), options=options) - if filename: - filename = QtCore.QDir.toNativeSeparators(filename) + if not result: + return + + filename = result[0] + filename = QtCore.QDir.toNativeSeparators(filename) if os.path.isdir(filename): self.user_input.setText(filename) @@ -378,6 +386,7 @@ class InstallDialog(QtWidgets.QDialog): if len(self._path) < 1: self._mongo.setVisible(False) + return path def _update_console(self, msg: str, error: bool = False) -> None: """Display message in console. diff --git a/igniter/install_thread.py b/igniter/install_thread.py index 278877bdf7..ad24913ed7 100644 --- a/igniter/install_thread.py +++ b/igniter/install_thread.py @@ -1,10 +1,13 @@ # -*- coding: utf-8 -*- """Working thread for installer.""" import os +import sys +from zipfile import ZipFile from Qt.QtCore import QThread, Signal from .bootstrap_repos import BootstrapRepos +from .bootstrap_repos import PypeVersion from .tools import validate_mongo_connection @@ -43,8 +46,9 @@ class InstallThread(QThread): self.message.emit("Installing Pype ...", False) # find local version of Pype - bs = BootstrapRepos(progress_callback=self.set_progress) - local_version = bs.get_local_version() + bs = BootstrapRepos( + progress_callback=self.set_progress, message=self.message) + local_version = bs.get_local_live_version() # if user did entered nothing, we install Pype from local version. # zip content of `repos`, copy it to user data dir and append @@ -68,14 +72,71 @@ class InstallThread(QThread): os.environ["PYPE_MONGO"] = self._mongo + self.message.emit( + f"Detecting installed Pype versions in {bs.data_dir}", False) + detected = bs.find_pype(include_zips=True) + + if detected: + if PypeVersion(version=local_version) < detected[-1]: + self.message.emit(( + f"Latest installed version {detected[-1]} is newer " + f"then currently running {local_version}" + ), False) + self.message.emit("Skipping Pype install ...", False) + if detected[-1].path.suffix.lower() == ".zip": + bs.extract_pype(detected[-1]) + return + + if PypeVersion(version=local_version) == detected[-1]: + self.message.emit(( + f"Latest installed version is the same as " + f"currently running {local_version}" + ), False) + self.message.emit("Skipping Pype install ...", False) + return + + self.message.emit(( + "All installed versions are older then " + f"currently running one {local_version}" + ), False) + else: + # we cannot build install package from frozen code. + if getattr(sys, 'frozen', False): + self.message.emit("None detected.", True) + self.message.emit(("Please set path to Pype sources to " + "build installation."), False) + return + else: + self.message.emit("None detected.", False) + self.message.emit( f"We will use local Pype version {local_version}", False) + repo_file = bs.install_live_repos() if not repo_file: self.message.emit( - f"!!! install failed - {repo_file}", True) + f"!!! Install failed - {repo_file}", True) return - self.message.emit(f"installed as {repo_file}", False) + + destination = bs.data_dir / repo_file.stem + if destination.exists(): + try: + destination.unlink() + except OSError as e: + self.message.emit( + f"!!! Cannot remove already existing {destination}", + True) + self.message.emit(e.strerror, True) + return + + destination.mkdir(parents=True) + + # extract zip there + self.message.emit("Extracting zip to destination ...", False) + with ZipFile(repo_file, "r") as zip_ref: + zip_ref.extractall(destination) + + self.message.emit(f"Installed as {repo_file}", False) else: # if we have mongo connection string, validate it, set it to # user settings and get PYPE_PATH from there. @@ -87,10 +148,14 @@ class InstallThread(QThread): bs.registry.set_secure_item("pypeMongo", self._mongo) os.environ["PYPE_MONGO"] = self._mongo + if os.getenv("PYPE_PATH") == self._path: + ... + + self.message.emit(f"processing {self._path}", True) repo_file = bs.process_entered_location(self._path) if not repo_file: - self.message.emit(f"!!! Cannot install", True) + self.message.emit("!!! Cannot install", True) return def set_path(self, path: str) -> None: diff --git a/igniter/pype.ico b/igniter/pype.ico new file mode 100644 index 0000000000..746fc36ba2 Binary files /dev/null and b/igniter/pype.ico differ diff --git a/igniter/splash.txt b/igniter/splash.txt new file mode 100644 index 0000000000..833bcd4b9c --- /dev/null +++ b/igniter/splash.txt @@ -0,0 +1,413 @@ + + + + * + + + + + + + .* + + + + + + * + .* + * + + + + . + * + .* + * + . + + . + * + .* + .* + .* + * + . + . + * + .* + .* + .* + * + . + _. + /** + \ * + \* + * + * + . + __. + ---* + \ \* + \ * + \* + * + . + \___. + /* * + \ \ * + \ \* + \ * + \* + . + |____. + /* * + \|\ * + \ \ * + \ \ * + \ \* + \/. + _/_____. + /* * + / \ * + \ \ * + \ \ * + \ \__* + \/__. + __________. + --*-- ___* + \ \ \/_* + \ \ __* + \ \ \_* + \ \____\* + \/____/. + \____________ . + /* ___ \* + \ \ \/_\ * + \ \ _____* + \ \ \___/* + \ \____\ * + \/____/ . + |___________ . + /* ___ \ * + \|\ \/_\ \ * + \ \ _____/ * + \ \ \___/ * + \ \____\ / * + \/____/ \. + _/__________ . + /* ___ \ * + / \ \/_\ \ * + \ \ _____/ * + \ \ \___/ ---* + \ \____\ / \__* + \/____/ \/__. + ____________ . + --*-- ___ \ * + \ \ \/_\ \ * + \ \ _____/ * + \ \ \___/ ---- * + \ \____\ / \____\* + \/____/ \/____/. + ____________ + /\ ___ \ . + \ \ \/_\ \ * + \ \ _____/ * + \ \ \___/ ---- * + \ \____\ / \____\ . + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ . + \ \ _____/ * + \ \ \___/ ---- * + \ \____\ / \____\ . + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ . + \ \ \___/ ---- * + \ \____\ / \____\ . + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ + \ \ \___/ ---- * + \ \____\ / \____\ + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ + \ \ \___/ ---- . + \ \____\ / \____\ + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ _ + \ \ \___/ ---- + \ \____\ / \____\ + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ + \ \ \___/ ---- + \ \____\ / \____\ + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ + \ \ \___/ ---- \ + \ \____\ / \____\ + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ + \ \ \___/ ---- \ + \ \____\ / \____\ \ + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ + \ \ \___/ ---- \ + \ \____\ / \____\ __\ + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ + \ \ \___/ ---- \ + \ \____\ / \____\ \__\ + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ + \ \ \___/ ---- \ \ + \ \____\ / \____\ \__\ + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ + \ \ \___/ ---- \ \ + \ \____\ / \____\ \__\ + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___. + \ \ \___/ ---- \ \\ + \ \____\ / \____\ \__\, + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ . + \ \ \___/ ---- \ \\ + \ \____\ / \____\ \__\\, + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ _. + \ \ \___/ ---- \ \\\ + \ \____\ / \____\ \__\\\ + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ __. + \ \ \___/ ---- \ \\ \ + \ \____\ / \____\ \__\\_/. + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___. + \ \ \___/ ---- \ \\ \\ + \ \____\ / \____\ \__\\__\. + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ . + \ \ \___/ ---- \ \\ \\ + \ \____\ / \____\ \__\\__\\. + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ _. + \ \ \___/ ---- \ \\ \\\ + \ \____\ / \____\ \__\\__\\. + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ __. + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\_. + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ __. + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__. + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ ___ + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__\ + \/____/ \/____/ . + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ ___ + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__\ + \/____/ \/____/ * + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ ___ + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__\ + \/____/ \/____/ O* + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ ___ + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__\ + \/____/ \/____/ oO* + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ ___ + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__\ + \/____/ \/____/ .oO* + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ ___ + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__\ + \/____/ \/____/ ..oO* + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ ___ + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__\ + \/____/ \/____/ . .oO* + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ ___ + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__\ + \/____/ \/____/ . p.oO* + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ ___ + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__\ + \/____/ \/____/ . Py.oO* + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ ___ + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__\ + \/____/ \/____/ . PYp.oO* + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ ___ + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__\ + \/____/ \/____/ . PYPe.oO* + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ ___ + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__\ + \/____/ \/____/ . PYPE .oO* + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ ___ + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__\ + \/____/ \/____/ . PYPE c.oO* + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ ___ + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__\ + \/____/ \/____/ . PYPE C1.oO* + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ ___ + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__\ + \/____/ \/____/ . PYPE ClU.oO* + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ ___ + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__\ + \/____/ \/____/ . PYPE CluB.oO* + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ ___ + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__\ + \/____/ \/____/ . PYPE Club .oO* + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ ___ + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__\ + \/____/ \/____/ . PYPE Club . .. + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ ___ + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__\ + \/____/ \/____/ . PYPE Club . .. + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ ___ + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__\ + \/____/ \/____/ . PYPE Club . . + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ ___ + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__\ + \/____/ \/____/ . PYPE Club . diff --git a/igniter/terminal_splash.py b/igniter/terminal_splash.py new file mode 100644 index 0000000000..1a7645571e --- /dev/null +++ b/igniter/terminal_splash.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +"""Pype terminal animation.""" +import blessed +from pathlib import Path +from time import sleep + +NO_TERMINAL = False + +try: + term = blessed.Terminal() +except AttributeError: + # this happens when blessed cannot find proper terminal. + # If so, skip printing ascii art animation. + NO_TERMINAL = True + + +def play_animation(): + """Play ASCII art Pype animation.""" + if NO_TERMINAL: + return + print(term.home + term.clear) + frame_size = 7 + splash_file = Path(__file__).parent / "splash.txt" + with splash_file.open("r") as sf: + animation = sf.readlines() + + animation_length = int(len(animation) / frame_size) + current_frame = 0 + for _ in range(animation_length): + frame = "".join( + scanline + for y, scanline in enumerate( + animation[current_frame : current_frame + frame_size] + ) + ) + + with term.location(0, 0): + # term.aquamarine3_bold(frame) + print(f"{term.bold}{term.aquamarine3}{frame}{term.normal}") + + sleep(0.02) + current_frame += frame_size + print(term.move_y(7)) diff --git a/igniter/tools.py b/igniter/tools.py index a06c444e62..d9a315834a 100644 --- a/igniter/tools.py +++ b/igniter/tools.py @@ -1,14 +1,104 @@ # -*- coding: utf-8 -*- -"""Tools used in **Igniter** GUI.""" +"""Tools used in **Igniter** GUI. + +Functions ``compose_url()`` and ``decompose_url()`` are the same as in +``pype.lib`` and they are here to avoid importing pype module before its +version is decided. + +""" + import os -import sys import uuid -from urllib.parse import urlparse +from typing import Dict +from urllib.parse import urlparse, parse_qs from pymongo import MongoClient from pymongo.errors import ServerSelectionTimeoutError, InvalidURI -from pype.lib import decompose_url, compose_url + +def decompose_url(url: str) -> Dict: + """Decompose mongodb url to its separate components. + + Args: + url (str): Mongodb url. + + Returns: + dict: Dictionary of components. + + """ + components = { + "scheme": None, + "host": None, + "port": None, + "username": None, + "password": None, + "auth_db": None + } + + result = urlparse(url) + if result.scheme is None: + _url = "mongodb://{}".format(url) + result = urlparse(_url) + + components["scheme"] = result.scheme + components["host"] = result.hostname + try: + components["port"] = result.port + except ValueError: + raise RuntimeError("invalid port specified") + components["username"] = result.username + components["password"] = result.password + + try: + components["auth_db"] = parse_qs(result.query)['authSource'][0] + except KeyError: + # no auth db provided, mongo will use the one we are connecting to + pass + + return components + + +def compose_url(scheme: str = None, + host: str = None, + username: str = None, + password: str = None, + port: int = None, + auth_db: str = None) -> str: + """Compose mongodb url from its individual components. + + Args: + scheme (str, optional): + host (str, optional): + username (str, optional): + password (str, optional): + port (str, optional): + auth_db (str, optional): + + Returns: + str: mongodb url + + """ + + url = "{scheme}://" + + if username and password: + url += "{username}:{password}@" + + url += "{host}" + if port: + url += ":{port}" + + if auth_db: + url += "?authSource={auth_db}" + + return url.format(**{ + "scheme": scheme, + "host": host, + "username": username, + "password": password, + "port": port, + "auth_db": auth_db + }) def validate_mongo_connection(cnx: str) -> (bool, str): @@ -22,30 +112,29 @@ def validate_mongo_connection(cnx: str) -> (bool, str): """ parsed = urlparse(cnx) - if parsed.scheme in ["mongodb", "mongodb+srv"]: - # we have mongo connection string. Let's try if we can connect. - components = decompose_url(cnx) - mongo_args = { - "host": compose_url(**components), - "serverSelectionTimeoutMS": 1000 - } - port = components.get("port") - if port is not None: - mongo_args["port"] = int(port) - - try: - client = MongoClient(**mongo_args) - client.server_info() - except ServerSelectionTimeoutError as e: - return False, f"Cannot connect to server {cnx} - {e}" - except ValueError: - return False, f"Invalid port specified {parsed.port}" - except InvalidURI as e: - return False, str(e) - else: - return True, "Connection is successful" - else: + if parsed.scheme not in ["mongodb", "mongodb+srv"]: return False, "Not mongodb schema" + # we have mongo connection string. Let's try if we can connect. + components = decompose_url(cnx) + mongo_args = { + "host": compose_url(**components), + "serverSelectionTimeoutMS": 1000 + } + port = components.get("port") + if port is not None: + mongo_args["port"] = int(port) + + try: + client = MongoClient(**mongo_args) + client.server_info() + except ServerSelectionTimeoutError as e: + return False, f"Cannot connect to server {cnx} - {e}" + except ValueError: + return False, f"Invalid port specified {parsed.port}" + except InvalidURI as e: + return False, str(e) + else: + return True, "Connection is successful" def validate_path_string(path: str) -> (bool, str): @@ -81,26 +170,6 @@ def validate_path_string(path: str) -> (bool, str): return False, "Not implemented yet" -def add_acre_to_sys_path(): - """Add full path of acre module to sys.path on ignitation.""" - try: - # Skip if is possible to import - import acre - - except ImportError: - # Full path to acred repository related to current file - acre_dir = os.path.join( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))), - "repos", - "acre" - ) - # Add path to sys.path - sys.path.append(acre_dir) - - # Validate that acre can be imported - import acre - - def load_environments(sections: list = None) -> dict: """Load environments from Pype. @@ -114,7 +183,6 @@ def load_environments(sections: list = None) -> dict: dict of str: loaded and processed environments. """ - add_acre_to_sys_path() import acre from pype import settings @@ -131,5 +199,4 @@ def load_environments(sections: list = None) -> dict: continue merged_env = acre.append(merged_env, parsed_env) - env = acre.compute(merged_env, cleanup=True) - return env + return acre.compute(merged_env, cleanup=True) diff --git a/igniter/user_settings.py b/igniter/user_settings.py new file mode 100644 index 0000000000..00ce68cb0b --- /dev/null +++ b/igniter/user_settings.py @@ -0,0 +1,466 @@ +# -*- coding: utf-8 -*- +"""Package to deal with saving and retrieving user specific settings.""" +import os +from datetime import datetime +from abc import ABCMeta, abstractmethod +import json + +# disable lru cache in Python 2 +try: + from functools import lru_cache +except ImportError: + def lru_cache(maxsize): + def max_size(func): + def wrapper(*args, **kwargs): + value = func(*args, **kwargs) + return value + return wrapper + return max_size + +# ConfigParser was renamed in python3 to configparser +try: + import configparser +except ImportError: + import ConfigParser as configparser + +import platform + +import appdirs +import six + + +@six.add_metaclass(ABCMeta) +class ASettingRegistry(): + """Abstract class defining structure of **SettingRegistry** class. + + It is implementing methods to store secure items into keyring, otherwise + mechanism for storing common items must be implemented in abstract + methods. + + Attributes: + _name (str): Registry names. + + """ + + def __init__(self, name): + # type: (str) -> ASettingRegistry + super(ASettingRegistry, self).__init__() + + if six.PY3: + import keyring + # hack for cx_freeze and Windows keyring backend + if platform.system() == "Windows": + from keyring.backends import Windows + keyring.set_keyring(Windows.WinVaultKeyring()) + + self._name = name + self._items = {} + + def set_item(self, name, value): + # type: (str, str) -> None + """Set item to settings registry. + + Args: + name (str): Name of the item. + value (str): Value of the item. + + """ + self._set_item(name, value) + + @abstractmethod + def _set_item(self, name, value): + # type: (str, str) -> None + # Implement it + pass + + def __setitem__(self, name, value): + self._items[name] = value + self._set_item(name, value) + + def get_item(self, name): + # type: (str) -> str + """Get item from settings registry. + + Args: + name (str): Name of the item. + + Returns: + value (str): Value of the item. + + Raises: + ValueError: If item doesn't exist. + + """ + return self._get_item(name) + + @abstractmethod + def _get_item(self, name): + # type: (str) -> str + # Implement it + pass + + def __getitem__(self, name): + return self._get_item(name) + + def delete_item(self, name): + # type: (str) -> None + """Delete item from settings registry. + + Args: + name (str): Name of the item. + + """ + self._delete_item(name) + + @abstractmethod + def _delete_item(self, name): + # type: (str) -> None + """Delete item from settings. + + Note: + see :meth:`pype.lib.user_settings.ARegistrySettings.delete_item` + + """ + pass + + def __delitem__(self, name): + del self._items[name] + self._delete_item(name) + + def set_secure_item(self, name, value): + # type: (str, str) -> None + """Set sensitive item into system's keyring. + + This uses `Keyring module`_ to save sensitive stuff into system's + keyring. + + Args: + name (str): Name of the item. + value (str): Value of the item. + + .. _Keyring module: + https://github.com/jaraco/keyring + + """ + if six.PY2: + raise NotImplementedError( + "Keyring not available on Python 2 hosts") + import keyring + keyring.set_password(self._name, name, value) + + @lru_cache(maxsize=32) + def get_secure_item(self, name): + # type: (str) -> str + """Get value of sensitive item from system's keyring. + + See also `Keyring module`_ + + Args: + name (str): Name of the item. + + Returns: + value (str): Value of the item. + + Raises: + ValueError: If item doesn't exist. + + .. _Keyring module: + https://github.com/jaraco/keyring + + """ + if six.PY2: + raise NotImplementedError( + "Keyring not available on Python 2 hosts") + import keyring + value = keyring.get_password(self._name, name) + if not value: + raise ValueError( + "Item {}:{} does not exist in keyring.".format( + self._name, name)) + return value + + def delete_secure_item(self, name): + # type: (str) -> None + """Delete value stored in system's keyring. + + See also `Keyring module`_ + + Args: + name (str): Name of the item to be deleted. + + .. _Keyring module: + https://github.com/jaraco/keyring + + """ + if six.PY2: + raise NotImplementedError( + "Keyring not available on Python 2 hosts") + import keyring + self.get_secure_item.cache_clear() + keyring.delete_password(self._name, name) + + +class IniSettingRegistry(ASettingRegistry): + """Class using :mod:`configparser`. + + This class is using :mod:`configparser` (ini) files to store items. + + """ + + def __init__(self, name, path): + # type: (str, str) -> IniSettingRegistry + super(IniSettingRegistry, self).__init__(name) + # get registry file + version = os.getenv("PYPE_VERSION", "N/A") + self._registry_file = os.path.join(path, "{}.ini".format(name)) + if not os.path.exists(self._registry_file): + with open(self._registry_file, mode="w") as cfg: + print("# Settings registry", cfg) + print("# Generated by Pype {}".format(version), cfg) + now = datetime.now().strftime("%d/%m/%Y %H:%M:%S") + print("# {}".format(now), cfg) + + def set_item_section( + self, section, name, value): + # type: (str, str, str) -> None + """Set item to specific section of ini registry. + + If section doesn't exists, it is created. + + Args: + section (str): Name of section. + name (str): Name of the item. + value (str): Value of the item. + + """ + value = str(value) + config = configparser.ConfigParser() + + config.read(self._registry_file) + if not config.has_section(section): + config.add_section(section) + current = config[section] + current[name] = value + + with open(self._registry_file, mode="w") as cfg: + config.write(cfg) + + def _set_item(self, name, value): + # type: (str, str) -> None + self.set_item_section("MAIN", name, value) + + def set_item(self, name, value): + # type: (str, str) -> None + """Set item to settings ini file. + + This saves item to ``DEFAULT`` section of ini as each item there + must reside in some section. + + Args: + name (str): Name of the item. + value (str): Value of the item. + + """ + # this does the some, overridden just for different docstring. + # we cast value to str as ini options values must be strings. + super(IniSettingRegistry, self).set_item(name, str(value)) + + def get_item(self, name): + # type: (str) -> str + """Gets item from settings ini file. + + This gets settings from ``DEFAULT`` section of ini file as each item + there must reside in some section. + + Args: + name (str): Name of the item. + + Returns: + str: Value of item. + + Raises: + ValueError: If value doesn't exist. + + """ + return super(IniSettingRegistry, self).get_item(name) + + @lru_cache(maxsize=32) + def get_item_from_section(self, section, name): + # type: (str, str) -> str + """Get item from section of ini file. + + This will read ini file and try to get item value from specified + section. If that section or item doesn't exist, :exc:`ValueError` + is risen. + + Args: + section (str): Name of ini section. + name (str): Name of the item. + + Returns: + str: Item value. + + Raises: + ValueError: If value doesn't exist. + + """ + config = configparser.ConfigParser() + config.read(self._registry_file) + try: + value = config[section][name] + except KeyError: + raise ValueError( + "Registry doesn't contain value {}:{}".format(section, name)) + return value + + def _get_item(self, name): + # type: (str) -> str + return self.get_item_from_section("MAIN", name) + + def delete_item_from_section(self, section, name): + # type: (str, str) -> None + """Delete item from section in ini file. + + Args: + section (str): Section name. + name (str): Name of the item. + + Raises: + ValueError: If item doesn't exist. + + """ + self.get_item_from_section.cache_clear() + config = configparser.ConfigParser() + config.read(self._registry_file) + try: + _ = config[section][name] + except KeyError: + raise ValueError( + "Registry doesn't contain value {}:{}".format(section, name)) + config.remove_option(section, name) + + # if section is empty, delete it + if len(config[section].keys()) == 0: + config.remove_section(section) + + with open(self._registry_file, mode="w") as cfg: + config.write(cfg) + + def _delete_item(self, name): + """Delete item from default section. + + Note: + See :meth:`~pype.lib.IniSettingsRegistry.delete_item_from_section` + + """ + self.delete_item_from_section("MAIN", name) + + +class JSONSettingRegistry(ASettingRegistry): + """Class using json file as storage.""" + + def __init__(self, name, path): + # type: (str, str) -> JSONSettingRegistry + super(JSONSettingRegistry, self).__init__(name) + #: str: name of registry file + self._registry_file = os.path.join(path, "{}.json".format(name)) + now = datetime.now().strftime("%d/%m/%Y %H:%M:%S") + header = { + "__metadata__": { + "pype-version": os.getenv("PYPE_VERSION", "N/A"), + "generated": now + }, + "registry": {} + } + + if not os.path.exists(os.path.dirname(self._registry_file)): + os.makedirs(os.path.dirname(self._registry_file), exist_ok=True) + if not os.path.exists(self._registry_file): + with open(self._registry_file, mode="w") as cfg: + json.dump(header, cfg, indent=4) + + @lru_cache(maxsize=32) + def _get_item(self, name): + # type: (str) -> object + """Get item value from registry json. + + Note: + See :meth:`pype.lib.JSONSettingRegistry.get_item` + + """ + with open(self._registry_file, mode="r") as cfg: + data = json.load(cfg) + try: + value = data["registry"][name] + except KeyError: + raise ValueError( + "Registry doesn't contain value {}".format(name)) + return value + + def get_item(self, name): + # type: (str) -> object + """Get item value from registry json. + + Args: + name (str): Name of the item. + + Returns: + value of the item + + Raises: + ValueError: If item is not found in registry file. + + """ + return self._get_item(name) + + def _set_item(self, name, value): + # type: (str, object) -> None + """Set item value to registry json. + + Note: + See :meth:`pype.lib.JSONSettingRegistry.set_item` + + """ + with open(self._registry_file, "r+") as cfg: + data = json.load(cfg) + data["registry"][name] = value + cfg.truncate(0) + cfg.seek(0) + json.dump(data, cfg, indent=4) + + def set_item(self, name, value): + # type: (str, object) -> None + """Set item and its value into json registry file. + + Args: + name (str): name of the item. + value (Any): value of the item. + + """ + self._set_item(name, value) + + def _delete_item(self, name): + # type: (str) -> None + self._get_item.cache_clear() + with open(self._registry_file, "r+") as cfg: + data = json.load(cfg) + del data["registry"][name] + cfg.truncate(0) + cfg.seek(0) + json.dump(data, cfg, indent=4) + + +class PypeSettingsRegistry(JSONSettingRegistry): + """Class handling Pype general settings registry. + + Attributes: + vendor (str): Name used for path construction. + product (str): Additional name used for path construction. + + """ + + def __init__(self): + self.vendor = "pypeclub" + self.product = "pype" + path = appdirs.user_data_dir(self.product, self.vendor) + super(PypeSettingsRegistry, self).__init__("pype_settings", path) diff --git a/pype.py b/pype.py deleted file mode 100644 index 3d700aebdb..0000000000 --- a/pype.py +++ /dev/null @@ -1,268 +0,0 @@ -# -*- coding: utf-8 -*- -"""Main entry point for Pype command. - -Bootstrapping process of Pype is as follows: - -`PYPE_PATH` is checked for existence - either one from environment or -from user settings. Precedence takes the one set by environment. - -On this path we try to find zip files with `pype-repositories-v3.x.x.zip` -format. - -If no Pype repositories are found in `PYPE_PATH (user data dir) -then **Igniter** (Pype setup tool) will launch its GUI. - -It can be used to specify `PYPE_PATH` or if it is _not_ specified, current -*"live"* repositories will be used to create such zip file and copy it to -appdata dir in user home. Version will be determined by version specified -in Pype module. - -If Pype repositories zip file is found in default install location -(user data dir) or in `PYPE_PATH`, it will get list of those zips there and -use latest one or the one specified with optional `--use-version` command -line argument. If the one specified doesn't exist then latest available -version will be used. All repositories in that zip will be added -to `sys.path` and `PYTHONPATH`. - -If Pype is live (not frozen) then current version of Pype module will be -used. All directories under `repos` will be added to `sys.path` and -`PYTHONPATH`. - -Pype depends on connection to `MongoDB`_. You can specify MongoDB connection -string via `AVALON_MONGO` set in environment or it can be set in user -settings or via **Igniter** GUI. - -Todo: - Move or remove bootstrapping environments out of the code. - -.. _MongoDB: - https://www.mongodb.com/ - -""" -import os -import re -import sys -import traceback - -from igniter.tools import load_environments, add_acre_to_sys_path - -from igniter import BootstrapRepos - -try: - import acre -except ImportError: - add_acre_to_sys_path() - import acre - - -def set_environments() -> None: - """Set loaded environments. - - .. todo: - better handling of environments - - """ - # FIXME: remove everything except global - env = load_environments(["global"]) - env = acre.merge(env, dict(os.environ)) - os.environ.clear() - os.environ.update(env) - - -def set_modules_environments(): - """Set global environments for pype's modules. - - This requires to have pype in `sys.path`. - """ - - from pype.modules import ModulesManager - - modules_manager = ModulesManager() - - module_envs = modules_manager.collect_global_environments() - publish_plugin_dirs = modules_manager.collect_plugin_paths()["publish"] - - # Set pyblish plugins paths if any module want to register them - if publish_plugin_dirs: - publish_paths_str = os.environ.get("PYBLISHPLUGINPATH") or "" - publish_paths = publish_paths_str.split(os.pathsep) - _publish_paths = set() - for path in publish_paths: - if path: - _publish_paths.add(os.path.normpath(path)) - for path in publish_plugin_dirs: - _publish_paths.add(os.path.normpath(path)) - module_envs["PYBLISHPLUGINPATH"] = os.pathsep.join(_publish_paths) - - # Metge environments with current environments and update values - if module_envs: - parsed_envs = acre.parse(module_envs) - env = acre.merge(parsed_envs, dict(os.environ)) - os.environ.clear() - os.environ.update(env) - - -def boot(): - """Bootstrap Pype.""" - - from pype.lib.terminal_splash import play_animation - play_animation() - - # find pype versions - bootstrap = BootstrapRepos() - pype_versions = bootstrap.find_pype() - - # check for `--use-version=3.0.0` argument. - use_version = None - - for arg in sys.argv: - m = re.search(r"--use-version=(?P\d+\.\d+\.\d*.+?)", arg) - if m and m.group('version'): - use_version = m.group('version') - break - - if not os.getenv("PYPE_MONGO"): - try: - pype_mongo = bootstrap.registry.get_secure_item("pypeMongo") - except ValueError: - print("*** No DB connection string specified.") - print("--- launching setup UI ...") - import igniter - igniter.run() - return - else: - os.environ["PYPE_MONGO"] = pype_mongo - - set_environments() - if getattr(sys, 'frozen', False): - if not pype_versions: - import igniter - igniter.run() - - version_path = BootstrapRepos.get_version_path_from_list( - use_version, pype_versions) - if version_path: - # use specified - bootstrap.add_paths_from_archive(version_path) - - else: - if use_version is not None: - print(("!!! Specified version was not found, using " - "latest available")) - # use latest - version_path = pype_versions[-1].path - bootstrap.add_paths_from_archive(version_path) - use_version = str(pype_versions[-1]) - - os.environ["PYPE_ROOT"] = version_path.as_posix() - else: - # run through repos and add them to sys.path and PYTHONPATH - pype_root = os.path.dirname(os.path.realpath(__file__)) - local_version = bootstrap.get_local_version() - if use_version and use_version != local_version: - version_path = BootstrapRepos.get_version_path_from_list( - use_version, pype_versions) - if version_path: - # use specified - bootstrap.add_paths_from_archive(version_path) - - os.environ["PYPE_ROOT"] = pype_root - repos = os.listdir(os.path.join(pype_root, "repos")) - repos = [os.path.join(pype_root, "repos", repo) for repo in repos] - # add self to python paths - repos.insert(0, pype_root) - for repo in repos: - sys.path.append(repo) - - pythonpath = os.getenv("PYTHONPATH", "") - paths = pythonpath.split(os.pathsep) - paths += repos - os.environ["PYTHONPATH"] = os.pathsep.join(paths) - - # DEPRECATED: remove when `pype-config` dissolves into Pype for good. - # .-=-----------------------=-=. ^ .=-=--------------------------=-. - os.environ["PYPE_MODULE_ROOT"] = os.environ["PYPE_ROOT"] - - # delete Pype module from cache so it is used from specific version - try: - del sys.modules["pype"] - del sys.modules["pype.version"] - except AttributeError: - pass - - from pype import cli - from pype.lib import terminal as t - from pype.version import __version__ - print(">>> loading environments ...") - set_environments() - set_modules_environments() - - info = get_info() - info.insert(0, ">>> Using Pype from [ {} ]".format( - os.path.dirname(cli.__file__))) - - info_length = len(max(info, key=len)) - info.insert(0, f"*** Pype [{__version__}] " + "-" * info_length) - for i in info: - t.echo(i) - - try: - cli.main(obj={}, prog_name="pype") - except Exception: - exc_info = sys.exc_info() - print("!!! Pype crashed:") - traceback.print_exception(*exc_info) - sys.exit(1) - - -def get_info() -> list: - """Print additional information to console.""" - from pype.lib.mongo import get_default_components - from pype.lib.log import PypeLogger - - components = get_default_components() - - infos = [] - if not getattr(sys, 'frozen', False): - infos.append(("Pype variant", "staging")) - else: - infos.append(("Pype variant", "production")) - infos.append(("Running pype from", os.environ.get('PYPE_ROOT'))) - infos.append(("Using mongodb", components["host"])) - - if os.environ.get("FTRACK_SERVER"): - infos.append(("Using FTrack at", - os.environ.get("FTRACK_SERVER"))) - - if os.environ.get('DEADLINE_REST_URL'): - infos.append(("Using Deadline webservice at", - os.environ.get("DEADLINE_REST_URL"))) - - if os.environ.get('MUSTER_REST_URL'): - infos.append(("Using Muster at", - os.environ.get("MUSTER_REST_URL"))) - - # Reinitialize - PypeLogger.initialize() - - log_components = PypeLogger.log_mongo_url_components - if log_components["host"]: - infos.append(("Logging to MongoDB", log_components["host"])) - infos.append((" - port", log_components["port"] or "")) - infos.append((" - database", PypeLogger.log_database_name)) - infos.append((" - collection", PypeLogger.log_collection_name)) - infos.append((" - user", log_components["username"] or "")) - if log_components["auth_db"]: - infos.append((" - auth source", log_components["auth_db"])) - - maximum = max([len(i[0]) for i in infos]) - formatted = [] - for info in infos: - padding = (maximum - len(info[0])) + 1 - formatted.append( - "... {}:{}[ {} ]".format(info[0], " " * padding, info[1])) - return formatted - - -if __name__ == "__main__": - boot() diff --git a/pype/cli.py b/pype/cli.py index 9d66ff988a..62975bff31 100644 --- a/pype/cli.py +++ b/pype/cli.py @@ -1,13 +1,20 @@ # -*- coding: utf-8 -*- """Package for handling pype command line arguments.""" import os +import sys + +import click + # import sys from .pype_commands import PypeCommands -import click @click.group(invoke_without_command=True) @click.pass_context +@click.option("--use-version", + expose_value=False, help="use specified version") +@click.option("--use-staging", is_flag=True, + expose_value=False, help="use staging variants") def main(ctx): """Pype is main command serving as entry point to pipeline system. @@ -20,6 +27,7 @@ def main(ctx): @main.command() @click.option("-d", "--dev", is_flag=True, help="Settings in Dev mode") def settings(dev=False): + """Show Pype Settings UI.""" PypeCommands().launch_settings_gui(dev) @@ -38,12 +46,6 @@ def tray(debug=False): PypeCommands().launch_tray(debug) -@main.command() -def mongodb(): - """Launch local mongodb server. Useful for development.""" - PypeCommands().launch_local_mongodb() - - @main.command() @click.option("-d", "--debug", is_flag=True, help="Print debug messages") @click.option("--ftrack-url", envvar="FTRACK_SERVER", @@ -56,7 +58,7 @@ def mongodb(): envvar="FTRACK_EVENTS_PATH", help=("path to ftrack event handlers")) @click.option("--no-stored-credentials", is_flag=True, - help="dont use stored credentials") + help="don't use stored credentials") @click.option("--store-credentials", is_flag=True, help="store provided credentials") @click.option("--legacy", is_flag=True, @@ -165,41 +167,6 @@ def texturecopy(debug, project, asset, path): PypeCommands().texture_copy(project, asset, path) -@main.command() -@click.option("-k", "--keyword", help="select tests by keyword to run", - type=click.STRING) -@click.argument("id", nargs=-1, type=click.STRING) -def test(pype, keyword, id): - """Run test suite.""" - if pype: - PypeCommands().run_pype_tests(keyword, id) - - -@main.command() -def make_docs(): - """Generate documentation with Sphinx into `docs/build`.""" - PypeCommands().make_docs() - - -@main.command() -def coverage(): - """Generate code coverage report.""" - PypeCommands().pype_setup_coverage() - - -@main.command() -def clean(): - """Delete python bytecode files. - - Working throughout Pype directory, it will remove all pyc bytecode files. - This is normally not needed but there are cases when update of repostories - caused errors thanks to these files. If you encounter errors complaining - about `magic number`, run this command. - """ - # TODO: reimplement in Python - pass - - @main.command(context_settings={"ignore_unknown_options": True}) @click.option("--app", help="Registered application name") @click.option("--project", help="Project name", @@ -233,7 +200,9 @@ def launch(app, project, asset, task, Optionally you can specify ftrack credentials if needed. ARGUMENTS are passed to launched application. + """ + # TODO: this needs to switch for Settings if ftrack_server: os.environ["FTRACK_SERVER"] = ftrack_server @@ -255,6 +224,33 @@ def launch(app, project, asset, task, @main.command() -def validate_config(): - """Validate all json configuration files for errors.""" - PypeCommands().validate_jsons() +@click.option("-p", "--path", help="Path to zip file", default=None) +def generate_zip(path): + """Generate Pype zip from current sources. + + If PATH is not provided, it will create zip file in user data dir. + + """ + PypeCommands().generate_zip(path) + + +@main.command( + context_settings=dict( + ignore_unknown_options=True, + allow_extra_args=True)) +@click.argument("script", required=True, type=click.Path(exists=True)) +def run(script): + """Run python script in Pype context.""" + import runpy + + if not script: + print("Error: missing path to script file.") + else: + + args = sys.argv + args.remove("run") + args.remove(script) + sys.argv = args + args_string = " ".join(args[1:]) + print(f"... running: {script} {args_string}") + runpy.run_path(script, run_name="__main__", ) diff --git a/pype/lib/__init__.py b/pype/lib/__init__.py index 691c105b76..d10f3d199d 100644 --- a/pype/lib/__init__.py +++ b/pype/lib/__init__.py @@ -1,5 +1,16 @@ # -*- coding: utf-8 -*- +# flake8: noqa E402 """Pype module API.""" +# add vendor to sys path based on Python version +import sys +import os +import site + +# add Python version specific vendor folder +site.addsitedir( + os.path.join( + os.getenv("PYPE_ROOT", ""), + "vendor", "python", "python_{}".format(sys.version[0]))) from .terminal import Terminal from .execute import ( diff --git a/pype/lib/terminal_splash.py b/pype/lib/terminal_splash.py index 7a94f2243e..0ba2706a27 100644 --- a/pype/lib/terminal_splash.py +++ b/pype/lib/terminal_splash.py @@ -4,12 +4,20 @@ import blessed from pathlib import Path from time import sleep +NO_TERMINAL = False -term = blessed.Terminal() +try: + term = blessed.Terminal() +except AttributeError: + # this happens when blessed cannot find proper terminal. + # If so, skip printing ascii art animation. + NO_TERMINAL = True def play_animation(): """Play ASCII art Pype animation.""" + if NO_TERMINAL: + return print(term.home + term.clear) frame_size = 7 splash_file = Path(__file__).parent / "splash.txt" @@ -19,11 +27,12 @@ def play_animation(): animation_length = int(len(animation) / frame_size) current_frame = 0 for _ in range(animation_length): - frame = "" - y = 0 - for scanline in animation[current_frame:current_frame + frame_size]: - frame += scanline - y += 1 + frame = "".join( + scanline + for y, scanline in enumerate( + animation[current_frame: current_frame + frame_size] + ) + ) with term.location(0, 0): # term.aquamarine3_bold(frame) diff --git a/pype/lib/user_settings.py b/pype/lib/user_settings.py index 0b40eccb65..00ce68cb0b 100644 --- a/pype/lib/user_settings.py +++ b/pype/lib/user_settings.py @@ -28,8 +28,6 @@ import platform import appdirs import six -from ..version import __version__ - @six.add_metaclass(ABCMeta) class ASettingRegistry(): @@ -213,11 +211,12 @@ class IniSettingRegistry(ASettingRegistry): # type: (str, str) -> IniSettingRegistry super(IniSettingRegistry, self).__init__(name) # get registry file + version = os.getenv("PYPE_VERSION", "N/A") self._registry_file = os.path.join(path, "{}.ini".format(name)) if not os.path.exists(self._registry_file): with open(self._registry_file, mode="w") as cfg: print("# Settings registry", cfg) - print("# Generated by Pype {}".format(__version__), cfg) + print("# Generated by Pype {}".format(version), cfg) now = datetime.now().strftime("%d/%m/%Y %H:%M:%S") print("# {}".format(now), cfg) @@ -368,7 +367,7 @@ class JSONSettingRegistry(ASettingRegistry): now = datetime.now().strftime("%d/%m/%Y %H:%M:%S") header = { "__metadata__": { - "pype-version": __version__, + "pype-version": os.getenv("PYPE_VERSION", "N/A"), "generated": now }, "registry": {} @@ -459,9 +458,9 @@ class PypeSettingsRegistry(JSONSettingRegistry): product (str): Additional name used for path construction. """ - vendor = "pypeclub" - product = "pype" def __init__(self): + self.vendor = "pypeclub" + self.product = "pype" path = appdirs.user_data_dir(self.product, self.vendor) super(PypeSettingsRegistry, self).__init__("pype_settings", path) diff --git a/pype/modules/ftrack/ftrack_server/socket_thread.py b/pype/modules/ftrack/ftrack_server/socket_thread.py index 6a5fe2c9d6..c638c9fa03 100644 --- a/pype/modules/ftrack/ftrack_server/socket_thread.py +++ b/pype/modules/ftrack/ftrack_server/socket_thread.py @@ -57,9 +57,15 @@ class SocketThread(threading.Thread): env = os.environ.copy() env["PYPE_PROCESS_MONGO_ID"] = str(Logger.mongo_process_id) + executable_args = [ + sys.executable + ] + if getattr(sys, "frozen", False): + executable_args.append("run") + self.subproc = subprocess.Popen( [ - sys.executable, + *executable_args, self.filepath, *self.additional_args, str(self.port) diff --git a/pype/modules/ftrack/ftrack_server/sub_event_processor.py b/pype/modules/ftrack/ftrack_server/sub_event_processor.py index 77dc85b0a2..f48b2141e6 100644 --- a/pype/modules/ftrack/ftrack_server/sub_event_processor.py +++ b/pype/modules/ftrack/ftrack_server/sub_event_processor.py @@ -4,7 +4,7 @@ import signal import socket import datetime -from ftrack_server import FtrackServer +from pype.modules.ftrack.ftrack_server.ftrack_server import FtrackServer from pype.modules.ftrack.ftrack_server.lib import ( SocketSession, ProcessEventHub, diff --git a/pype/modules/ftrack/ftrack_server/sub_event_status.py b/pype/modules/ftrack/ftrack_server/sub_event_status.py index a398b019eb..07b233282f 100644 --- a/pype/modules/ftrack/ftrack_server/sub_event_status.py +++ b/pype/modules/ftrack/ftrack_server/sub_event_status.py @@ -7,7 +7,7 @@ import socket import datetime import ftrack_api -from ftrack_server import FtrackServer +from pype.modules.ftrack.ftrack_server.ftrack_server import FtrackServer from pype.modules.ftrack.ftrack_server.lib import ( SocketSession, StatusEventHub, diff --git a/pype/modules/ftrack/ftrack_server/sub_event_storer.py b/pype/modules/ftrack/ftrack_server/sub_event_storer.py index 3523e5701f..2fdd3b07f7 100644 --- a/pype/modules/ftrack/ftrack_server/sub_event_storer.py +++ b/pype/modules/ftrack/ftrack_server/sub_event_storer.py @@ -6,7 +6,7 @@ import socket import pymongo import ftrack_api -from ftrack_server import FtrackServer +from pype.modules.ftrack.ftrack_server.ftrack_server import FtrackServer from pype.modules.ftrack.ftrack_server.lib import ( SocketSession, StorerEventHub, diff --git a/pype/modules/ftrack/ftrack_server/sub_legacy_server.py b/pype/modules/ftrack/ftrack_server/sub_legacy_server.py index ecb12830c4..e09bcbb699 100644 --- a/pype/modules/ftrack/ftrack_server/sub_legacy_server.py +++ b/pype/modules/ftrack/ftrack_server/sub_legacy_server.py @@ -4,10 +4,10 @@ import datetime import signal import threading -from ftrack_server import FtrackServer import ftrack_api from pype.api import Logger from pype.modules import ModulesManager +from pype.modules.ftrack.ftrack_server.ftrack_server import FtrackServer log = Logger().get_logger("Event Server Legacy") diff --git a/pype/modules/ftrack/ftrack_server/sub_user_server.py b/pype/modules/ftrack/ftrack_server/sub_user_server.py index 58d5982ac2..b968714faf 100644 --- a/pype/modules/ftrack/ftrack_server/sub_user_server.py +++ b/pype/modules/ftrack/ftrack_server/sub_user_server.py @@ -2,7 +2,7 @@ import sys import signal import socket -from ftrack_server import FtrackServer +from pype.modules.ftrack.ftrack_server.ftrack_server import FtrackServer from pype.modules.ftrack.ftrack_server.lib import ( SocketSession, SocketBaseEventHub diff --git a/pype/modules/sync_server/providers/gdrive.py b/pype/modules/sync_server/providers/gdrive.py index 2207fdf3a3..5bc6f21b38 100644 --- a/pype/modules/sync_server/providers/gdrive.py +++ b/pype/modules/sync_server/providers/gdrive.py @@ -8,6 +8,7 @@ from googleapiclient.http import MediaFileUpload, MediaIoBaseDownload from pype.api import Logger from pype.api import get_system_settings from ..utils import time_function +import time SCOPES = ['https://www.googleapis.com/auth/drive.metadata.readonly', 'https://www.googleapis.com/auth/drive.file', @@ -42,6 +43,7 @@ class GDriveHandler(AbstractProvider): """ FOLDER_STR = 'application/vnd.google-apps.folder' MY_DRIVE_STR = 'My Drive' # name of root folder of regular Google drive + CHUNK_SIZE = 2097152 # must be divisible by 256! def __init__(self, site_name, tree=None, presets=None): self.presets = None @@ -277,7 +279,9 @@ class GDriveHandler(AbstractProvider): path = new_path_key return folder_id - def upload_file(self, source_path, path, overwrite=False): + def upload_file(self, source_path, path, + server, collection, file, representation, site, + overwrite=False): """ Uploads single file from 'source_path' to destination 'path'. It creates all folders on the path if are not existing. @@ -287,6 +291,13 @@ class GDriveHandler(AbstractProvider): path (string): absolute path with or without name of the file overwrite (boolean): replace existing file + arguments for saving progress: + server (SyncServer): server instance to call update_db on + collection (str): name of collection + file (dict): info about uploaded file (matches structure from db) + representation (dict): complete repre containing 'file' + site (str): site name + Returns: (string) file_id of created/modified file , throws FileExistsError, FileNotFoundError exceptions @@ -302,8 +313,8 @@ class GDriveHandler(AbstractProvider): path = os.path.dirname(path) else: target_name = os.path.basename(source_path) - file = self.file_path_exists(path + "/" + target_name) - if file and not overwrite: + target_file = self.file_path_exists(path + "/" + target_name) + if target_file and not overwrite: raise FileExistsError("File already exists, " "use 'overwrite' argument") @@ -316,23 +327,45 @@ class GDriveHandler(AbstractProvider): } media = MediaFileUpload(source_path, mimetype='application/octet-stream', + chunksize=self.CHUNK_SIZE, resumable=True) try: - if not file: + if not target_file: # update doesnt like parent file_metadata['parents'] = [folder_id] - file = self.service.files().create(body=file_metadata, - supportsAllDrives=True, - media_body=media, - fields='id').execute() + request = self.service.files().create(body=file_metadata, + supportsAllDrives=True, + media_body=media, + fields='id') else: - file = self.service.files().update(fileId=file["id"], - body=file_metadata, - supportsAllDrives=True, - media_body=media, - fields='id').execute() + request = self.service.files().update(fileId=target_file["id"], + body=file_metadata, + supportsAllDrives=True, + media_body=media, + fields='id') + + media.stream() + log.debug("Start Upload! {}".format(source_path)) + last_tick = status = response = None + status_val = 0 + while response is None: + if status: + status_val = float(status.progress()) + if not last_tick or \ + time.time() - last_tick >= server.LOG_PROGRESS_SEC: + last_tick = time.time() + log.debug("Uploaded %d%%." % + int(status_val * 100)) + server.update_db(collection=collection, + new_file_id=None, + file=file, + representation=representation, + site=site, + progress=status_val + ) + status, response = request.next_chunk() except errors.HttpError as ex: if ex.resp['status'] == '404': @@ -344,13 +377,14 @@ class GDriveHandler(AbstractProvider): log.warning("Forbidden received, hit quota. " "Injecting 60s delay.") - import time time.sleep(60) return False raise - return file["id"] + return response['id'] - def download_file(self, source_path, local_path, overwrite=False): + def download_file(self, source_path, local_path, + server, collection, file, representation, site, + overwrite=False): """ Downloads single file from 'source_path' (remote) to 'local_path'. It creates all folders on the local_path if are not existing. @@ -361,6 +395,13 @@ class GDriveHandler(AbstractProvider): local_path (string): absolute path with or without name of the file overwrite (boolean): replace existing file + arguments for saving progress: + server (SyncServer): server instance to call update_db on + collection (str): name of collection + file (dict): info about uploaded file (matches structure from db) + representation (dict): complete repre containing 'file' + site (str): site name + Returns: (string) file_id of created/modified file , throws FileExistsError, FileNotFoundError exceptions @@ -378,9 +419,9 @@ class GDriveHandler(AbstractProvider): else: # just folder, get file name from source target_name = os.path.basename(source_path) - file = os.path.isfile(local_path + "/" + target_name) + local_file = os.path.isfile(local_path + "/" + target_name) - if file and not overwrite: + if local_file and not overwrite: raise FileExistsError("File already exists, " "use 'overwrite' argument") @@ -389,9 +430,24 @@ class GDriveHandler(AbstractProvider): with open(local_path + "/" + target_name, "wb") as fh: downloader = MediaIoBaseDownload(fh, request) - done = False - while done is False: - status, done = downloader.next_chunk() + last_tick = status = response = None + status_val = 0 + while response is None: + if status: + status_val = float(status.progress()) + if not last_tick or \ + time.time() - last_tick >= server.LOG_PROGRESS_SEC: + last_tick = time.time() + log.debug("Downloaded %d%%." % + int(status_val * 100)) + server.update_db(collection=collection, + new_file_id=None, + file=file, + representation=representation, + site=site, + progress=status_val + ) + status, response = downloader.next_chunk() return target_name diff --git a/pype/modules/sync_server/providers/resources/gdrive.png b/pype/modules/sync_server/providers/resources/gdrive.png new file mode 100644 index 0000000000..e6c9131454 Binary files /dev/null and b/pype/modules/sync_server/providers/resources/gdrive.png differ diff --git a/pype/modules/sync_server/providers/resources/studio.png b/pype/modules/sync_server/providers/resources/studio.png new file mode 100644 index 0000000000..e95e9762f8 Binary files /dev/null and b/pype/modules/sync_server/providers/resources/studio.png differ diff --git a/pype/modules/sync_server/sync_server.py b/pype/modules/sync_server/sync_server.py index 167be665f5..84637a1d62 100644 --- a/pype/modules/sync_server/sync_server.py +++ b/pype/modules/sync_server/sync_server.py @@ -18,7 +18,7 @@ from .utils import time_function import six from pype.lib import PypeLogger -from .. import PypeModule, ITrayService +from .. import PypeModule, ITrayModule if six.PY2: web = asyncio = STATIC_DIR = WebSocketAsync = None @@ -34,7 +34,7 @@ class SyncStatus(Enum): DO_DOWNLOAD = 2 -class SyncServer(PypeModule, ITrayService): +class SyncServer(PypeModule, ITrayModule): """ Synchronization server that is syncing published files from local to any of implemented providers (like GDrive, S3 etc.) @@ -92,6 +92,7 @@ class SyncServer(PypeModule, ITrayService): # set 0 to no limit REPRESENTATION_LIMIT = 100 DEFAULT_SITE = 'studio' + LOG_PROGRESS_SEC = 5 # how often log progress to DB name = "sync_server" label = "Sync Server" @@ -116,6 +117,8 @@ class SyncServer(PypeModule, ITrayService): self.presets = None # settings for all enabled projects for sync self.sync_server_thread = None # asyncio requires new thread + self.action_show_widget = None + def connect_with_modules(self, *_a, **kw): return @@ -131,21 +134,26 @@ class SyncServer(PypeModule, ITrayService): self.presets = None self.lock = threading.Lock() + self.connection = AvalonMongoDB() + self.connection.install() try: self.presets = self.get_synced_presets() self.set_active_sites(self.presets) - self.sync_server_thread = SyncServerThread(self) + from .tray.app import SyncServerWindow + self.widget = SyncServerWindow(self) except ValueError: - log.info("No system setting for sync. Not syncing.") + log.info("No system setting for sync. Not syncing.", exc_info=True) + self.enabled = False except KeyError: log.info(( "There are not set presets for SyncServer OR " "Credentials provided are invalid, " "no syncing possible"). format(str(self.presets)), exc_info=True) + self.enabled = False def tray_start(self): """ @@ -185,6 +193,19 @@ class SyncServer(PypeModule, ITrayService): exc_info=True ) + def tray_menu(self, parent_menu): + if not self.enabled: + return + + from Qt import QtWidgets + """Add menu or action to Tray(or parent)'s menu""" + action = QtWidgets.QAction("SyncServer", parent_menu) + action.triggered.connect(self.show_widget) + parent_menu.addAction(action) + parent_menu.addSeparator() + + self.action_show_widget = action + @property def is_running(self): return self.sync_server_thread.is_running @@ -245,7 +266,8 @@ class SyncServer(PypeModule, ITrayService): settings = get_project_settings(project_name) sync_settings = settings.get("global")["sync_server"] if not sync_settings: - log.info("No project setting for Sync Server, not syncing.") + log.info("No project setting for {}, not syncing.". + format(project_name)) return {} if sync_settings.get("enabled"): return sync_settings @@ -406,8 +428,8 @@ class SyncServer(PypeModule, ITrayService): return SyncStatus.DO_NOTHING - async def upload(self, file, representation, provider_name, site_name, - tree=None, preset=None): + async def upload(self, collection, file, representation, provider_name, + site_name, tree=None, preset=None): """ Upload single 'file' of a 'representation' to 'provider'. Source url is taken from 'file' portion, where {root} placeholder @@ -418,6 +440,7 @@ class SyncServer(PypeModule, ITrayService): from GDrive), 'created_dt' - time of upload Args: + collection (str): source collection file (dictionary): of file from representation in Mongo representation (dictionary): of representation provider_name (string): gdrive, gdc etc. @@ -447,21 +470,28 @@ class SyncServer(PypeModule, ITrayService): err = "Folder {} wasn't created. Check permissions.".\ format(target_folder) raise NotADirectoryError(err) - + _, remote_site = self.get_sites_for_project(collection) loop = asyncio.get_running_loop() file_id = await loop.run_in_executor(None, handler.upload_file, local_file, remote_file, - True) + self, + collection, + file, + representation, + remote_site, + True + ) return file_id - async def download(self, file, representation, provider_name, + async def download(self, collection, file, representation, provider_name, site_name, tree=None, preset=None): """ Downloads file to local folder denoted in representation.Context. Args: + collection (str): source collection file (dictionary) : info about processed file representation (dictionary): repr that 'file' belongs to provider_name (string): 'gdrive' etc @@ -485,26 +515,37 @@ class SyncServer(PypeModule, ITrayService): local_folder = os.path.dirname(local_file) os.makedirs(local_folder, exist_ok=True) + local_site, _ = self.get_sites_for_project(collection) + loop = asyncio.get_running_loop() file_id = await loop.run_in_executor(None, handler.download_file, remote_file, local_file, - False) + False, + self, + collection, + file, + representation, + local_site + ) return file_id - def update_db(self, new_file_id, file, representation, provider_name, - error=None): + def update_db(self, collection, new_file_id, file, representation, + site, error=None, progress=None): """ Update 'provider' portion of records in DB with success (file_id) or error (exception) Args: + collection (string): name of project - force to db connection as + each file might come from different collection new_file_id (string): file (dictionary): info about processed file (pulled from DB) representation (dictionary): parent repr of file (from DB) - provider_name (string): label ('gdrive', 'S3') + site (string): label ('gdrive', 'S3') error (string): exception message + progress (float): 0-1 of progress of upload/download Returns: None @@ -518,26 +559,33 @@ class SyncServer(PypeModule, ITrayService): file_index, _ = self._get_file_info(representation.get('files', []), file_id) site_index, _ = self._get_provider_rec(file.get('sites', []), - provider_name) + site) update = {} if new_file_id: update["$set"] = self._get_success_dict(file_index, site_index, new_file_id) # reset previous errors if any update["$unset"] = self._get_error_dict(file_index, site_index, - "", "") + "", "", "") + elif progress is not None: + update["$set"] = self._get_progress_dict(file_index, site_index, + progress) else: - tries = self._get_tries_count(file, provider_name) + tries = self._get_tries_count(file, site) tries += 1 update["$set"] = self._get_error_dict(file_index, site_index, error, tries) + self.connection.Session["AVALON_PROJECT"] = collection self.connection.update_one( query, update ) + if progress is not None: + return + status = 'failed' error_str = 'with error {}'.format(error) if new_file_id: @@ -553,7 +601,7 @@ class SyncServer(PypeModule, ITrayService): def _get_file_info(self, files, _id): """ Return record from list of records which name matches to 'provider' - Could be possibly refactored with '_get_file_info' together. + Could be possibly refactored with '_get_provider_rec' together. Args: files (list): of dictionaries with info about published files @@ -590,7 +638,7 @@ class SyncServer(PypeModule, ITrayService): return -1, None def reset_provider_for_file(self, collection, representation_id, - file_id, site_name): + file_id, side): """ Reset information about synchronization for particular 'file_id' and provider. @@ -599,7 +647,7 @@ class SyncServer(PypeModule, ITrayService): collection (string): name of project (eg. collection) in DB representation_id(string): _id of representation file_id (string): file _id in representation - site_name (string): 'gdrive', 'S3' etc + side (string): local or remote side Returns: None """ @@ -607,19 +655,25 @@ class SyncServer(PypeModule, ITrayService): query = { "_id": ObjectId(representation_id) } - self.connection.Session["AVALON_PROJECT"] = collection - representation = list(self.connection.find(query)) + + representation = list(self.connection.database[collection].find(query)) if not representation: raise ValueError("Representation {} not found in {}". format(representation_id, collection)) + local_site, remote_site = self.get_sites_for_project(collection) + if side == 'local': + site_name = local_site + else: + site_name = remote_site + files = representation[0].get('files', []) file_index, _ = self._get_file_info(files, file_id) site_index, _ = self._get_provider_rec(files[file_index]. get('sites', []), site_name) - if file_index > 0 and site_index > 0: + if file_index >= 0 and site_index >= 0: elem = {"name": site_name} update = { "$set": {"files.{}.sites.{}".format(file_index, site_index): @@ -627,7 +681,7 @@ class SyncServer(PypeModule, ITrayService): } } - self.connection.update_one( + self.connection.database[collection].update_one( query, update ) @@ -641,6 +695,10 @@ class SyncServer(PypeModule, ITrayService): """ return int(self.presets[project_name]["config"]["loop_delay"]) + def show_widget(self): + """Show dialog to enter credentials""" + self.widget.show() + def _get_success_dict(self, file_index, site_index, new_file_id): """ Provide success metadata ("id", "created_dt") to be stored in Db. @@ -660,7 +718,8 @@ class SyncServer(PypeModule, ITrayService): datetime.utcnow()} return val - def _get_error_dict(self, file_index, site_index, error="", tries=""): + def _get_error_dict(self, file_index, site_index, + error="", tries="", progress=""): """ Provide error metadata to be stored in Db. Used for set (error and tries provided) or unset mode. @@ -675,7 +734,9 @@ class SyncServer(PypeModule, ITrayService): val = {"files.{}.sites.{}.last_failed_dt". format(file_index, site_index): datetime.utcnow(), "files.{}.sites.{}.error".format(file_index, site_index): error, - "files.{}.sites.{}.tries".format(file_index, site_index): tries + "files.{}.sites.{}.tries".format(file_index, site_index): tries, + "files.{}.sites.{}.progress".format(file_index, site_index): + progress } return val @@ -703,6 +764,22 @@ class SyncServer(PypeModule, ITrayService): _, rec = self._get_provider_rec(file.get("sites", []), provider) return rec.get("tries", 0) + def _get_progress_dict(self, file_index, site_index, progress): + """ + Provide progress metadata to be stored in Db. + Used during upload/download for GUI to show. + Args: + file_index: (int) - index of modified file + site_index: (int) - index of modified site of modified file + progress: (float) - 0-1 progress of upload/download + Returns: + (dictionary) + """ + val = {"files.{}.sites.{}.progress". + format(file_index, site_index): progress + } + return val + def _get_local_file_path(self, file, local_root): """ Auxiliary function for replacing rootless path with real path @@ -848,23 +925,27 @@ class SyncServerThread(threading.Thread): tree = handler.get_tree() limit -= 1 task = asyncio.create_task( - self.module.upload(file, + self.module.upload(collection, + file, sync, provider, site, tree, site_preset)) task_files_to_process.append(task) - # store info for exception handling + # store info for exception handlingy files_processed_info.append((file, sync, - site)) + site, + collection + )) processed_file_path.add(file_path) if status == SyncStatus.DO_DOWNLOAD: tree = handler.get_tree() limit -= 1 task = asyncio.create_task( - self.module.download(file, + self.module.download(collection, + file, sync, provider, site, @@ -874,7 +955,9 @@ class SyncServerThread(threading.Thread): files_processed_info.append((file, sync, - local)) + local, + collection + )) processed_file_path.add(file_path) log.debug("Sync tasks count {}". @@ -884,12 +967,13 @@ class SyncServerThread(threading.Thread): return_exceptions=True) for file_id, info in zip(files_created, files_processed_info): - file, representation, site = info + file, representation, site, collection = info error = None if isinstance(file_id, BaseException): error = str(file_id) file_id = None - self.module.update_db(file_id, + self.module.update_db(collection, + file_id, file, representation, site, diff --git a/pype/modules/sync_server/tray/app.py b/pype/modules/sync_server/tray/app.py new file mode 100644 index 0000000000..afd103f9d5 --- /dev/null +++ b/pype/modules/sync_server/tray/app.py @@ -0,0 +1,1509 @@ +from Qt import QtWidgets, QtCore, QtGui +from Qt.QtCore import Qt +from pype.tools.settings.settings.widgets.base import ProjectListWidget +import attr +import os +from pype.tools.settings.settings import style +from avalon.tools.delegates import PrettyTimeDelegate, pretty_timestamp + +from pype.lib import PypeLogger + +import json + +log = PypeLogger().get_logger("SyncServer") + +STATUS = { + 0: 'In Progress', + 1: 'Failed', + 2: 'Queued', + 3: 'Paused', + 4: 'Synced OK', + -1: 'Not available' +} + + +class SyncServerWindow(QtWidgets.QDialog): + """ + Main window that contains list of synchronizable projects and summary + view with all synchronizable representations for first project + """ + + def __init__(self, sync_server, parent=None): + super(SyncServerWindow, self).__init__(parent) + self.setWindowFlags(QtCore.Qt.Window) + self.setFocusPolicy(QtCore.Qt.StrongFocus) + + self.setStyleSheet(style.load_stylesheet()) + self.setWindowIcon(QtGui.QIcon(style.app_icon_path())) + self.resize(1400, 800) + + body = QtWidgets.QWidget(self) + footer = QtWidgets.QWidget(self) + footer.setFixedHeight(20) + + container = QtWidgets.QWidget() + projects = SyncProjectListWidget(sync_server, self) + projects.refresh() # force selection of default + repres = SyncRepresentationWidget(sync_server, + project=projects.current_project, + parent=self) + + container_layout = QtWidgets.QHBoxLayout(container) + container_layout.setContentsMargins(0, 0, 0, 0) + split = QtWidgets.QSplitter() + split.addWidget(projects) + split.addWidget(repres) + split.setSizes([180, 950, 200]) + container_layout.addWidget(split) + + container.setLayout(container_layout) + + body_layout = QtWidgets.QHBoxLayout(body) + body_layout.addWidget(container) + body_layout.setContentsMargins(0, 0, 0, 0) + + message = QtWidgets.QLabel(footer) + message.hide() + + footer_layout = QtWidgets.QVBoxLayout(footer) + footer_layout.addWidget(message) + footer_layout.setContentsMargins(0, 0, 0, 0) + + layout = QtWidgets.QVBoxLayout(self) + layout.addWidget(body) + layout.addWidget(footer) + + self.setLayout(body_layout) + self.setWindowTitle("Sync Server") + + projects.project_changed.connect( + lambda: repres.table_view.model().set_project( + projects.current_project)) + + +class SyncProjectListWidget(ProjectListWidget): + """ + Lists all projects that are synchronized to choose from + """ + + def __init__(self, sync_server, parent): + super(SyncProjectListWidget, self).__init__(parent) + self.sync_server = sync_server + + def validate_context_change(self): + return True + + def refresh(self): + model = self.project_list.model() + model.clear() + + for project_name in self.sync_server.get_synced_presets().keys(): + model.appendRow(QtGui.QStandardItem(project_name)) + + if len(self.sync_server.get_synced_presets().keys()) == 0: + model.appendRow(QtGui.QStandardItem("No project configured")) + + self.current_project = self.project_list.currentIndex().data( + QtCore.Qt.DisplayRole + ) + if not self.current_project: + self.current_project = self.project_list.model().item(0). \ + data(QtCore.Qt.DisplayRole) + + +class SyncRepresentationWidget(QtWidgets.QWidget): + """ + Summary dialog with list of representations that matches current + settings 'local_site' and 'remote_site'. + """ + active_changed = QtCore.Signal() # active index changed + + default_widths = ( + ("asset", 210), + ("subset", 190), + ("version", 10), + ("representation", 90), + ("created_dt", 100), + ("sync_dt", 100), + ("local_site", 60), + ("remote_site", 70), + ("files_count", 70), + ("files_size", 70), + ("priority", 20), + ("state", 50) + ) + + def __init__(self, sync_server, project=None, parent=None): + super(SyncRepresentationWidget, self).__init__(parent) + + self.sync_server = sync_server + + self._selected_id = None # keep last selected _id + + self.filter = QtWidgets.QLineEdit() + self.filter.setPlaceholderText("Filter representations..") + + top_bar_layout = QtWidgets.QHBoxLayout() + top_bar_layout.addWidget(self.filter) + + self.table_view = QtWidgets.QTableView() + headers = [item[0] for item in self.default_widths] + + model = SyncRepresentationModel(sync_server, headers, project) + self.table_view.setModel(model) + self.table_view.setContextMenuPolicy(QtCore.Qt.CustomContextMenu) + self.table_view.setSelectionMode( + QtWidgets.QAbstractItemView.SingleSelection) + self.table_view.setSelectionBehavior( + QtWidgets.QAbstractItemView.SelectRows) + self.table_view.horizontalHeader().setSortIndicator( + -1, Qt.AscendingOrder) + self.table_view.setSortingEnabled(True) + self.table_view.setAlternatingRowColors(True) + self.table_view.verticalHeader().hide() + + time_delegate = PrettyTimeDelegate(self) + column = self.table_view.model().get_header_index("created_dt") + self.table_view.setItemDelegateForColumn(column, time_delegate) + column = self.table_view.model().get_header_index("sync_dt") + self.table_view.setItemDelegateForColumn(column, time_delegate) + + column = self.table_view.model().get_header_index("local_site") + delegate = ImageDelegate(self) + self.table_view.setItemDelegateForColumn(column, delegate) + + column = self.table_view.model().get_header_index("remote_site") + delegate = ImageDelegate(self) + self.table_view.setItemDelegateForColumn(column, delegate) + + column = self.table_view.model().get_header_index("files_size") + delegate = SizeDelegate(self) + self.table_view.setItemDelegateForColumn(column, delegate) + + for column_name, width in self.default_widths: + idx = model.get_header_index(column_name) + self.table_view.setColumnWidth(idx, width) + + layout = QtWidgets.QVBoxLayout(self) + layout.setContentsMargins(0, 0, 0, 0) + layout.addLayout(top_bar_layout) + layout.addWidget(self.table_view) + + self.table_view.doubleClicked.connect(self._double_clicked) + self.filter.textChanged.connect(lambda: model.set_filter( + self.filter.text())) + self.table_view.customContextMenuRequested.connect( + self._on_context_menu) + + self.table_view.model().modelReset.connect(self._set_selection) + + self.selection_model = self.table_view.selectionModel() + self.selection_model.selectionChanged.connect(self._selection_changed) + + def _selection_changed(self, new_selection): + index = self.selection_model.currentIndex() + self._selected_id = self.table_view.model().data(index, Qt.UserRole) + + def _set_selection(self): + """ + Sets selection to 'self._selected_id' if exists. + + Keep selection during model refresh. + """ + if self._selected_id: + index = self.table_view.model().get_index(self._selected_id) + if index and index.isValid(): + mode = QtCore.QItemSelectionModel.Select | \ + QtCore.QItemSelectionModel.Rows + self.selection_model.setCurrentIndex(index, mode) + else: + self._selected_id = None + + def _double_clicked(self, index): + """ + Opens representation dialog with all files after doubleclick + """ + _id = self.table_view.model().data(index, Qt.UserRole) + detail_window = SyncServerDetailWindow( + self.sync_server, _id, self.table_view.model()._project) + detail_window.exec() + + def _on_context_menu(self, point): + """ + Shows menu with loader actions on Right-click. + """ + point_index = self.table_view.indexAt(point) + if not point_index.isValid(): + return + + +class SyncRepresentationModel(QtCore.QAbstractTableModel): + PAGE_SIZE = 19 + REFRESH_SEC = 5000 + DEFAULT_SORT = { + "updated_dt_remote": -1, + "_id": 1 + } + SORT_BY_COLUMN = [ + "context.asset", # asset + "context.subset", # subset + "context.version", # version + "context.representation", # representation + "updated_dt_local", # local created_dt + "updated_dt_remote", # remote created_dt + "avg_progress_local", # local progress + "avg_progress_remote", # remote progress + "files_count", # count of files + "files_size", # file size of all files + "context.asset", # priority TODO + "status" # state + ] + + numberPopulated = QtCore.Signal(int) + + @attr.s + class SyncRepresentation: + """ + Auxiliary object for easier handling. + + Fields must contain all header values (+ any arbitrary values). + """ + _id = attr.ib() + asset = attr.ib() + subset = attr.ib() + version = attr.ib() + representation = attr.ib() + created_dt = attr.ib(default=None) + sync_dt = attr.ib(default=None) + local_site = attr.ib(default=None) + remote_site = attr.ib(default=None) + files_count = attr.ib(default=None) + files_size = attr.ib(default=None) + priority = attr.ib(default=None) + state = attr.ib(default=None) + + def __init__(self, sync_server, header, project=None): + super(SyncRepresentationModel, self).__init__() + self._header = header + self._data = [] + self._project = project + self._rec_loaded = 0 + self._buffer = [] # stash one page worth of records (actually cursor) + self.filter = None + + self._initialized = False + + self.sync_server = sync_server + # TODO think about admin mode + # this is for regular user, always only single local and single remote + self.local_site, self.remote_site = \ + self.sync_server.get_sites_for_project(self._project) + + self.projection = self.get_default_projection() + + self.sort = self.DEFAULT_SORT + + self.query = self.get_default_query() + self.default_query = list(self.get_default_query()) + log.debug("!!! init query: {}".format(json.dumps(self.query, + indent=4))) + representations = self.dbcon.aggregate(self.query) + self.refresh(representations) + + self.timer = QtCore.QTimer() + self.timer.timeout.connect(self.tick) + self.timer.start(self.REFRESH_SEC) + + @property + def dbcon(self): + return self.sync_server.connection.database[self._project] + + def data(self, index, role): + item = self._data[index.row()] + + if role == Qt.DisplayRole: + return attr.asdict(item)[self._header[index.column()]] + if role == Qt.UserRole: + return item._id + + def rowCount(self, index): + return len(self._data) + + def columnCount(self, index): + return len(self._header) + + def headerData(self, section, orientation, role): + if role == Qt.DisplayRole: + if orientation == Qt.Horizontal: + return str(self._header[section]) + + def tick(self): + self.refresh(representations=None, load_records=self._rec_loaded) + self.timer.start(self.REFRESH_SEC) + + def get_header_index(self, value): + """ + Returns index of 'value' in headers + + Args: + value (str): header name value + Returns: + (int) + """ + return self._header.index(value) + + def refresh(self, representations=None, load_records=0): + self.beginResetModel() + self._data = [] + self._rec_loaded = 0 + + if not representations: + self.query = self.get_default_query(load_records) + representations = self.dbcon.aggregate(self.query) + + self._add_page_records(self.local_site, self.remote_site, + representations) + self.endResetModel() + + def _add_page_records(self, local_site, remote_site, representations): + for repre in representations: + context = repre.get("context").pop() + files = repre.get("files", []) + if isinstance(files, dict): # aggregate returns dictionary + files = [files] + + # representation without files doesnt concern us + if not files: + continue + + local_updated = remote_updated = None + if repre.get('updated_dt_local'): + local_updated = \ + repre.get('updated_dt_local').strftime("%Y%m%dT%H%M%SZ") + + if repre.get('updated_dt_remote'): + remote_updated = \ + repre.get('updated_dt_remote').strftime("%Y%m%dT%H%M%SZ") + + avg_progress_remote = repre.get('avg_progress_remote', '') + avg_progress_local = repre.get('avg_progress_local', '') + + item = self.SyncRepresentation( + repre.get("_id"), + context.get("asset"), + context.get("subset"), + "v{:0>3d}".format(context.get("version", 1)), + context.get("representation"), + local_updated, + remote_updated, + '{} {}'.format(local_site, avg_progress_local), + '{} {}'.format(remote_site, avg_progress_remote), + repre.get("files_count", 1), + repre.get("files_size", 0), + 1, + STATUS[repre.get("status", -1)] + ) + + self._data.append(item) + self._rec_loaded += 1 + + def canFetchMore(self, index): + """ + Check if there are more records than currently loaded + """ + # 'skip' might be suboptimal when representation hits 500k+ + self._buffer = list(self.dbcon.aggregate(self.query)) + # log.info("!!! canFetchMore _rec_loaded::{} - {}".format( + # self._rec_loaded, len(self._buffer))) + return len(self._buffer) > self._rec_loaded + + def fetchMore(self, index): + """ + Add more record to model. + + Called when 'canFetchMore' returns true, which means there are + more records in DB than loaded. + 'self._buffer' is used to stash cursor to limit requery + """ + log.debug("fetchMore") + # cursor.count() returns always total number, not only skipped + limit + remainder = len(self._buffer) - self._rec_loaded + items_to_fetch = min(self.PAGE_SIZE, remainder) + self.beginInsertRows(index, + self._rec_loaded, + self._rec_loaded + items_to_fetch - 1) + + self._add_page_records(self.local_site, self.remote_site, self._buffer) + + self.endInsertRows() + + self.numberPopulated.emit(items_to_fetch) # ?? + + def sort(self, index, order): + """ + Summary sort per representation. + + Sort is happening on a DB side, model is reset, db queried + again. + + Args: + index (int): column index + order (int): 0| + """ + # limit unwanted first re-sorting by view + if index < 0: + return + + self._rec_loaded = 0 + if order == 0: + order = 1 + else: + order = -1 + + self.sort = {self.SORT_BY_COLUMN[index]: order, '_id': 1} + self.query = self.get_default_query() + + representations = self.dbcon.aggregate(self.query) + self.refresh(representations) + + def set_filter(self, filter): + """ + Adds text value filtering + + Args: + filter (str): string inputted by user + """ + self.filter = filter + self.refresh() + + def set_project(self, project): + """ + Changes project, called after project selection is changed + + Args: + project (str): name of project + """ + self._project = project + self.refresh() + + def get_index(self, id): + """ + Get index of 'id' value. + + Used for keeping selection after refresh. + + Args: + id (str): MongoDB _id + Returns: + (QModelIndex) + """ + index = None + for i in range(self.rowCount(None)): + index = self.index(i, 0) + value = self.data(index, Qt.UserRole) + if value == id: + return index + return index + + def get_default_query(self, limit=0): + """ + Returns basic aggregate query for main table. + + Main table provides summary information about representation, + which could have multiple files. Details are accessible after + double click on representation row. + Columns: + 'created_dt' - max of created or updated (when failed) per repr + 'sync_dt' - same for remote side + 'local_site' - progress of repr on local side, 1 = finished + 'remote_site' - progress on remote side, calculates from files + 'state' - + 0 - in progress + 1 - failed + 2 - queued + 3 - paused (not implemented yet) + 4 - finished on both sides + + are calculated and must be calculated in DB because of + pagination + + Args: + limit (int): how many records should be returned, by default + it 'PAGE_SIZE' for performance. + Should be overridden by value of loaded records for refresh + functionality (got more records by scrolling, refresh + shouldn't reset that) + """ + if limit == 0: + limit = SyncRepresentationModel.PAGE_SIZE + + return [ + {"$match": self._get_match_part()}, + {'$unwind': '$files'}, + # merge potentially unwinded records back to single per repre + {'$addFields': { + 'order_remote': { + '$filter': {'input': '$files.sites', 'as': 'p', + 'cond': {'$eq': ['$$p.name', self.remote_site]} + }}, + 'order_local': { + '$filter': {'input': '$files.sites', 'as': 'p', + 'cond': {'$eq': ['$$p.name', self.local_site]} + }} + }}, + {'$addFields': { + # prepare progress per file, presence of 'created_dt' denotes + # successfully finished load/download + 'progress_remote': {'$first': { + '$cond': [{'$size': "$order_remote.progress"}, + "$order_remote.progress", + {'$cond': [ + {'$size': "$order_remote.created_dt"}, + [1], + [0] + ]} + ]}}, + 'progress_local': {'$first': { + '$cond': [{'$size': "$order_local.progress"}, + "$order_local.progress", + {'$cond': [ + {'$size': "$order_local.created_dt"}, + [1], + [0] + ]} + ]}}, + # file might be successfully created or failed, not both + 'updated_dt_remote': {'$first': { + '$cond': [{'$size': "$order_remote.created_dt"}, + "$order_remote.created_dt", + {'$cond': [ + {'$size': "$order_remote.last_failed_dt"}, + "$order_remote.last_failed_dt", + [] + ]} + ]}}, + 'updated_dt_local': {'$first': { + '$cond': [{'$size': "$order_local.created_dt"}, + "$order_local.created_dt", + {'$cond': [ + {'$size': "$order_local.last_failed_dt"}, + "$order_local.last_failed_dt", + [] + ]} + ]}}, + 'files_size': {'$ifNull': ["$files.size", 0]}, + 'failed_remote': { + '$cond': [{'$size': "$order_remote.last_failed_dt"}, + 1, + 0]}, + 'failed_local': { + '$cond': [{'$size': "$order_local.last_failed_dt"}, + 1, + 0]} + }}, + {'$group': { + '_id': '$_id', + # pass through context - same for representation + 'context': {'$addToSet': '$context'}, + # pass through files as a list + 'files': {'$addToSet': '$files'}, + # count how many files + 'files_count': {'$sum': 1}, + 'files_size': {'$sum': '$files_size'}, + # sum avg progress, finished = 1 + 'avg_progress_remote': {'$avg': "$progress_remote"}, + 'avg_progress_local': {'$avg': "$progress_local"}, + # select last touch of file + 'updated_dt_remote': {'$max': "$updated_dt_remote"}, + 'failed_remote': {'$sum': '$failed_remote'}, + 'failed_local': {'$sum': '$failed_local'}, + 'updated_dt_local': {'$max': "$updated_dt_local"} + }}, + {"$limit": limit}, + {"$skip": self._rec_loaded}, + {"$project": self.projection}, + {"$sort": self.sort} + ] + + def _get_match_part(self): + """ + Extend match part with filter if present. + + Filter is set by user input. Each model has different fields to be + checked. + If performance issues are found, '$text' and text indexes should + be investigated. + """ + if not self.filter: + return { + "type": "representation", + 'files.sites': { + '$elemMatch': { + '$or': [ + {'name': self.local_site}, + {'name': self.remote_site} + ] + } + } + } + else: + regex_str = '.*{}.*'.format(self.filter) + return { + "type": "representation", + '$or': [ + {'context.subset': {'$regex': regex_str, '$options': 'i'}}, + {'context.asset': {'$regex': regex_str, '$options': 'i'}}, + {'context.representation': {'$regex': regex_str, + '$options': 'i'}}], + 'files.sites': { + '$elemMatch': { + '$or': [ + {'name': self.local_site}, + {'name': self.remote_site} + ] + } + } + } + + def get_default_projection(self): + """ + Projection part for aggregate query. + + All fields with '1' will be returned, no others. + + Returns: + (dict) + """ + return { + "context.subset": 1, + "context.asset": 1, + "context.version": 1, + "context.representation": 1, + "files": 1, + 'files_count': 1, + "files_size": 1, + 'avg_progress_remote': 1, + 'avg_progress_local': 1, + 'updated_dt_remote': 1, + 'updated_dt_local': 1, + 'status': { + '$switch': { + 'branches': [ + { + 'case': { + '$or': [{'$eq': ['$avg_progress_remote', 0]}, + {'$eq': ['$avg_progress_local', 0]}]}, + 'then': 2 # Queued + }, + { + 'case': { + '$or': ['$failed_remote', '$failed_local']}, + 'then': 1 # Failed + }, + { + 'case': {'$or': [{'$and': [ + {'$gt': ['$avg_progress_remote', 0]}, + {'$lt': ['$avg_progress_remote', 1]} + ]}, + {'$and': [ + {'$gt': ['$avg_progress_local', 0]}, + {'$lt': ['$avg_progress_local', 1]} + ]} + ]}, + 'then': 0 # In progress + }, + { + 'case': {'$eq': ['dummy_placeholder', 'paused']}, + 'then': 3 # Paused + }, + { + 'case': {'$and': [ + {'$eq': ['$avg_progress_remote', 1]}, + {'$eq': ['$avg_progress_local', 1]} + ]}, + 'then': 4 # Synced OK + }, + ], + 'default': -1 + } + } + } + + +class SyncServerDetailWindow(QtWidgets.QDialog): + def __init__(self, sync_server, _id, project, parent=None): + log.debug( + "!!! SyncServerDetailWindow _id:: {}".format(_id)) + super(SyncServerDetailWindow, self).__init__(parent) + self.setWindowFlags(QtCore.Qt.Window) + self.setFocusPolicy(QtCore.Qt.StrongFocus) + + self.setStyleSheet(style.load_stylesheet()) + self.setWindowIcon(QtGui.QIcon(style.app_icon_path())) + self.resize(1000, 400) + + body = QtWidgets.QWidget() + footer = QtWidgets.QWidget() + footer.setFixedHeight(20) + + container = SyncRepresentationDetailWidget(sync_server, _id, project, + parent=self) + body_layout = QtWidgets.QHBoxLayout(body) + body_layout.addWidget(container) + body_layout.setContentsMargins(0, 0, 0, 0) + + message = QtWidgets.QLabel() + message.hide() + + footer_layout = QtWidgets.QVBoxLayout(footer) + footer_layout.addWidget(message) + footer_layout.setContentsMargins(0, 0, 0, 0) + + layout = QtWidgets.QVBoxLayout(self) + layout.addWidget(body) + layout.addWidget(footer) + + self.setLayout(body_layout) + self.setWindowTitle("Sync Representation Detail") + + +class SyncRepresentationDetailWidget(QtWidgets.QWidget): + """ + Widget to display list of synchronizable files for single repre. + + Args: + _id (str): representation _id + project (str): name of project with repre + parent (QDialog): SyncServerDetailWindow + """ + active_changed = QtCore.Signal() # active index changed + + default_widths = ( + ("file", 290), + ("created_dt", 120), + ("sync_dt", 120), + ("local_site", 60), + ("remote_site", 60), + ("size", 60), + ("priority", 20), + ("state", 90) + ) + + def __init__(self, sync_server, _id=None, project=None, parent=None): + super(SyncRepresentationDetailWidget, self).__init__(parent) + + self.representation_id = _id + self.item = None # set to item that mouse was clicked over + + self.sync_server = sync_server + + self._selected_id = None + + self.filter = QtWidgets.QLineEdit() + self.filter.setPlaceholderText("Filter representation..") + + top_bar_layout = QtWidgets.QHBoxLayout() + top_bar_layout.addWidget(self.filter) + + self.table_view = QtWidgets.QTableView() + headers = [item[0] for item in self.default_widths] + + model = SyncRepresentationDetailModel(sync_server, headers, _id, + project) + self.table_view.setModel(model) + self.table_view.setContextMenuPolicy(QtCore.Qt.CustomContextMenu) + self.table_view.setSelectionMode( + QtWidgets.QAbstractItemView.SingleSelection) + self.table_view.setSelectionBehavior( + QtWidgets.QTableView.SelectRows) + self.table_view.horizontalHeader().setSortIndicator(-1, + Qt.AscendingOrder) + self.table_view.setSortingEnabled(True) + self.table_view.setAlternatingRowColors(True) + self.table_view.verticalHeader().hide() + + time_delegate = PrettyTimeDelegate(self) + column = self.table_view.model().get_header_index("created_dt") + self.table_view.setItemDelegateForColumn(column, time_delegate) + column = self.table_view.model().get_header_index("sync_dt") + self.table_view.setItemDelegateForColumn(column, time_delegate) + + column = self.table_view.model().get_header_index("local_site") + delegate = ImageDelegate(self) + self.table_view.setItemDelegateForColumn(column, delegate) + + column = self.table_view.model().get_header_index("remote_site") + delegate = ImageDelegate(self) + self.table_view.setItemDelegateForColumn(column, delegate) + + column = self.table_view.model().get_header_index("size") + delegate = SizeDelegate(self) + self.table_view.setItemDelegateForColumn(column, delegate) + + for column_name, width in self.default_widths: + idx = model.get_header_index(column_name) + self.table_view.setColumnWidth(idx, width) + + layout = QtWidgets.QVBoxLayout(self) + layout.setContentsMargins(0, 0, 0, 0) + layout.addLayout(top_bar_layout) + layout.addWidget(self.table_view) + + self.filter.textChanged.connect(lambda: model.set_filter( + self.filter.text())) + self.table_view.customContextMenuRequested.connect( + self._on_context_menu) + + self.table_view.model().modelReset.connect(self._set_selection) + + self.selection_model = self.table_view.selectionModel() + self.selection_model.selectionChanged.connect(self._selection_changed) + + def _selection_changed(self): + index = self.selection_model.currentIndex() + self._selected_id = self.table_view.model().data(index, Qt.UserRole) + + def _set_selection(self): + """ + Sets selection to 'self._selected_id' if exists. + + Keep selection during model refresh. + """ + if self._selected_id: + index = self.table_view.model().get_index(self._selected_id) + if index.isValid(): + mode = QtCore.QItemSelectionModel.Select | \ + QtCore.QItemSelectionModel.Rows + self.selection_model.setCurrentIndex(index, mode) + else: + self._selected_id = None + + def _show_detail(self): + """ + Shows windows with error message for failed sync of a file. + """ + dt = max(self.item.created_dt, self.item.sync_dt) + detail_window = SyncRepresentationErrorWindow(self.item._id, + self.project, + dt, + self.item.tries, + self.item.error) + detail_window.exec() + + def _on_context_menu(self, point): + """ + Shows menu with loader actions on Right-click. + """ + point_index = self.table_view.indexAt(point) + if not point_index.isValid(): + return + + self.item = self.table_view.model()._data[point_index.row()] + + menu = QtWidgets.QMenu() + actions_mapping = {} + + if self.item.state == STATUS[1]: + action = QtWidgets.QAction("Open error detail") + actions_mapping[action] = self._show_detail + menu.addAction(action) + + remote_site, remote_progress = self.item.remote_site.split() + if remote_progress == '1': + action = QtWidgets.QAction("Reset local site") + actions_mapping[action] = self._reset_local_site + menu.addAction(action) + + local_site, local_progress = self.item.local_site.split() + if local_progress == '1': + action = QtWidgets.QAction("Reset remote site") + actions_mapping[action] = self._reset_remote_site + menu.addAction(action) + + if not actions_mapping: + action = QtWidgets.QAction("< No action >") + actions_mapping[action] = None + menu.addAction(action) + + result = menu.exec_(QtGui.QCursor.pos()) + if result: + to_run = actions_mapping[result] + if to_run: + to_run() + + def _reset_local_site(self): + """ + Removes errors or success metadata for particular file >> forces + redo of upload/download + """ + self.sync_server.reset_provider_for_file( + self.table_view.model()._project, + self.representation_id, + self.item._id, + 'local') + + def _reset_remote_site(self): + """ + Removes errors or success metadata for particular file >> forces + redo of upload/download + """ + self.sync_server.reset_provider_for_file( + self.table_view.model()._project, + self.representation_id, + self.item._id, + 'remote') + + +class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): + """ + List of all syncronizable files per single representation. + """ + PAGE_SIZE = 30 + # TODO add filter filename + DEFAULT_SORT = { + "files.path": 1 + } + SORT_BY_COLUMN = [ + "files.path", + "updated_dt_local", # local created_dt + "updated_dt_remote", # remote created_dt + "progress_local", # local progress + "progress_remote", # remote progress + "size", # remote progress + "context.asset", # priority TODO + "status" # state + ] + + @attr.s + class SyncRepresentationDetail: + """ + Auxiliary object for easier handling. + + Fields must contain all header values (+ any arbitrary values). + """ + _id = attr.ib() + file = attr.ib() + created_dt = attr.ib(default=None) + sync_dt = attr.ib(default=None) + local_site = attr.ib(default=None) + remote_site = attr.ib(default=None) + size = attr.ib(default=None) + priority = attr.ib(default=None) + state = attr.ib(default=None) + tries = attr.ib(default=None) + error = attr.ib(default=None) + + def __init__(self, sync_server, header, _id, project=None): + super(SyncRepresentationDetailModel, self).__init__() + self._header = header + self._data = [] + self._project = project + self._rec_loaded = 0 + self.filter = None + self._buffer = [] # stash one page worth of records (actually cursor) + self._id = _id + self._initialized = False + + self.sync_server = sync_server + # TODO think about admin mode + # this is for regular user, always only single local and single remote + self.local_site, self.remote_site = \ + self.sync_server.get_sites_for_project(self._project) + + self.sort = self.DEFAULT_SORT + + # in case we would like to hide/show some columns + self.projection = self.get_default_projection() + + self.query = self.get_default_query() + representations = self.dbcon.aggregate(self.query) + self.refresh(representations) + + self.timer = QtCore.QTimer() + self.timer.timeout.connect(self.tick) + self.timer.start(SyncRepresentationModel.REFRESH_SEC) + + @property + def dbcon(self): + return self.sync_server.connection.database[self._project] + + def tick(self): + self.refresh(representations=None, load_records=self._rec_loaded) + self.timer.start(SyncRepresentationModel.REFRESH_SEC) + + def get_header_index(self, value): + """ + Returns index of 'value' in headers + + Args: + value (str): header name value + Returns: + (int) + """ + return self._header.index(value) + + def data(self, index, role): + item = self._data[index.row()] + if role == Qt.DisplayRole: + return attr.asdict(item)[self._header[index.column()]] + if role == Qt.UserRole: + return item._id + + def rowCount(self, index): + return len(self._data) + + def columnCount(self, index): + return len(self._header) + + def headerData(self, section, orientation, role): + if role == Qt.DisplayRole: + if orientation == Qt.Horizontal: + return str(self._header[section]) + + def refresh(self, representations=None, load_records=0): + self.beginResetModel() + self._data = [] + self._rec_loaded = 0 + + if not representations: + self.query = self.get_default_query(load_records) + representations = self.dbcon.aggregate(self.query) + + self._add_page_records(self.local_site, self.remote_site, + representations) + self.endResetModel() + + def _add_page_records(self, local_site, remote_site, representations): + """ + Process all records from 'representation' and add them to storage. + + Args: + local_site (str): name of local site (mine) + remote_site (str): name of cloud provider (theirs) + representations (Mongo Cursor) + """ + for repre in representations: + # log.info("!!! repre:: {}".format(repre)) + files = repre.get("files", []) + if isinstance(files, dict): # aggregate returns dictionary + files = [files] + + for file in files: + local_updated = remote_updated = None + if repre.get('updated_dt_local'): + local_updated = \ + repre.get('updated_dt_local').strftime( + "%Y%m%dT%H%M%SZ") + + if repre.get('updated_dt_remote'): + remote_updated = \ + repre.get('updated_dt_remote').strftime( + "%Y%m%dT%H%M%SZ") + + progress_remote = repre.get('progress_remote', '') + progress_local = repre.get('progress_local', '') + + errors = [] + if repre.get('failed_remote_error'): + errors.append(repre.get('failed_remote_error')) + if repre.get('failed_local_error'): + errors.append(repre.get('failed_local_error')) + + item = self.SyncRepresentationDetail( + file.get("_id"), + os.path.basename(file["path"]), + local_updated, + remote_updated, + '{} {}'.format(local_site, progress_local), + '{} {}'.format(remote_site, progress_remote), + file.get('size', 0), + 1, + STATUS[repre.get("status", -1)], + repre.get("tries"), + '\n'.join(errors) + ) + self._data.append(item) + self._rec_loaded += 1 + + def canFetchMore(self, index): + """ + Check if there are more records than currently loaded + """ + # 'skip' might be suboptimal when representation hits 500k+ + self._buffer = list(self.dbcon.aggregate(self.query)) + return len(self._buffer) > self._rec_loaded + + def fetchMore(self, index): + """ + Add more record to model. + + Called when 'canFetchMore' returns true, which means there are + more records in DB than loaded. + 'self._buffer' is used to stash cursor to limit requery + """ + log.debug("fetchMore") + # cursor.count() returns always total number, not only skipped + limit + remainder = len(self._buffer) - self._rec_loaded + items_to_fetch = min(self.PAGE_SIZE, remainder) + + self.beginInsertRows(index, + self._rec_loaded, + self._rec_loaded + items_to_fetch - 1) + self._add_page_records(self.local_site, self.remote_site, self._buffer) + + self.endInsertRows() + + def sort(self, index, order): + # limit unwanted first re-sorting by view + if index < 0: + return + + self._rec_loaded = 0 # change sort - reset from start + + if order == 0: + order = 1 + else: + order = -1 + + self.sort = {self.SORT_BY_COLUMN[index]: order} + self.query = self.get_default_query() + + representations = self.dbcon.aggregate(self.query) + self.refresh(representations) + + def set_filter(self, filter): + self.filter = filter + self.refresh() + + def get_index(self, id): + """ + Get index of 'id' value. + + Used for keeping selection after refresh. + + Args: + id (str): MongoDB _id + Returns: + (QModelIndex) + """ + index = None + for i in range(self.rowCount(None)): + index = self.index(i, 0) + value = self.data(index, Qt.UserRole) + if value == id: + return index + return index + + def get_default_query(self, limit=0): + """ + Gets query that gets used when no extra sorting, filtering or + projecting is needed. + + Called for basic table view. + + Returns: + [(dict)] - list with single dict - appropriate for aggregate + function for MongoDB + """ + if limit == 0: + limit = SyncRepresentationModel.PAGE_SIZE + + return [ + {"$match": self._get_match_part()}, + {"$unwind": "$files"}, + {'$addFields': { + 'order_remote': { + '$filter': {'input': '$files.sites', 'as': 'p', + 'cond': {'$eq': ['$$p.name', self.remote_site]} + }}, + 'order_local': { + '$filter': {'input': '$files.sites', 'as': 'p', + 'cond': {'$eq': ['$$p.name', self.local_site]} + }} + }}, + {'$addFields': { + # prepare progress per file, presence of 'created_dt' denotes + # successfully finished load/download + 'progress_remote': {'$first': { + '$cond': [{'$size': "$order_remote.progress"}, + "$order_remote.progress", + {'$cond': [ + {'$size': "$order_remote.created_dt"}, + [1], + [0] + ]} + ]}}, + 'progress_local': {'$first': { + '$cond': [{'$size': "$order_local.progress"}, + "$order_local.progress", + {'$cond': [ + {'$size': "$order_local.created_dt"}, + [1], + [0] + ]} + ]}}, + # file might be successfully created or failed, not both + 'updated_dt_remote': {'$first': { + '$cond': [ + {'$size': "$order_remote.created_dt"}, + "$order_remote.created_dt", + { + '$cond': [ + {'$size': "$order_remote.last_failed_dt"}, + "$order_remote.last_failed_dt", + [] + ] + } + ] + }}, + 'updated_dt_local': {'$first': { + '$cond': [ + {'$size': "$order_local.created_dt"}, + "$order_local.created_dt", + { + '$cond': [ + {'$size': "$order_local.last_failed_dt"}, + "$order_local.last_failed_dt", + [] + ] + } + ] + }}, + 'failed_remote': { + '$cond': [{'$size': "$order_remote.last_failed_dt"}, + 1, + 0]}, + 'failed_local': { + '$cond': [{'$size': "$order_local.last_failed_dt"}, + 1, + 0]}, + 'failed_remote_error': {'$first': { + '$cond': [{'$size': "$order_remote.error"}, + "$order_remote.error", + [""]]}}, + 'failed_local_error': {'$first': { + '$cond': [{'$size': "$order_local.error"}, + "$order_local.error", + [""]]}}, + 'tries': {'$first': { + '$cond': [{'$size': "$order_local.tries"}, + "$order_local.tries", + {'$cond': [ + {'$size': "$order_remote.tries"}, + "$order_remote.tries", + [] + ]} + ]}} + }}, + {"$limit": limit}, + {"$skip": self._rec_loaded}, + {"$project": self.projection}, + {"$sort": self.sort} + ] + + def _get_match_part(self): + """ + Returns different content for 'match' portion if filtering by + name is present + + Returns: + (dict) + """ + if not self.filter: + return { + "type": "representation", + "_id": self._id + } + else: + regex_str = '.*{}.*'.format(self.filter) + return { + "type": "representation", + "_id": self._id, + '$or': [{'files.path': {'$regex': regex_str, '$options': 'i'}}] + } + + def get_default_projection(self): + """ + Projection part for aggregate query. + + All fields with '1' will be returned, no others. + + Returns: + (dict) + """ + return { + "files": 1, + 'progress_remote': 1, + 'progress_local': 1, + 'updated_dt_remote': 1, + 'updated_dt_local': 1, + 'failed_remote_error': 1, + 'failed_local_error': 1, + 'tries': 1, + 'status': { + '$switch': { + 'branches': [ + { + 'case': { + '$or': [{'$eq': ['$progress_remote', 0]}, + {'$eq': ['$progress_local', 0]}]}, + 'then': 2 # Queued + }, + { + 'case': { + '$or': ['$failed_remote', '$failed_local']}, + 'then': 1 # Failed + }, + { + 'case': {'$or': [{'$and': [ + {'$gt': ['$progress_remote', 0]}, + {'$lt': ['$progress_remote', 1]} + ]}, + {'$and': [ + {'$gt': ['$progress_local', 0]}, + {'$lt': ['$progress_local', 1]} + ]} + ]}, + 'then': 0 # In Progress + }, + { + 'case': {'$eq': ['dummy_placeholder', 'paused']}, + 'then': 3 + }, + { + 'case': {'$and': [ + {'$eq': ['$progress_remote', 1]}, + {'$eq': ['$progress_local', 1]} + ]}, + 'then': 4 # Synced OK + }, + ], + 'default': -1 + } + } + } + + +class ImageDelegate(QtWidgets.QStyledItemDelegate): + """ + Prints icon of site and progress of synchronization + """ + + def __init__(self, parent=None): + super(ImageDelegate, self).__init__(parent) + self.icons = {} + + def paint(self, painter, option, index): + option = QtWidgets.QStyleOptionViewItem(option) + option.showDecorationSelected = True + + if (option.showDecorationSelected and + (option.state & QtWidgets.QStyle.State_Selected)): + painter.setOpacity(0.20) # highlight color is a bit off + painter.fillRect(option.rect, + option.palette.highlight()) + painter.setOpacity(1) + + d = index.data(QtCore.Qt.DisplayRole) + if d: + provider, value = d.split() + else: + return + + if not self.icons.get(provider): + resource_path = os.path.dirname(__file__) + resource_path = os.path.join(resource_path, "..", + "providers", "resources") + pix_url = "{}/{}.png".format(resource_path, provider) + pixmap = QtGui.QPixmap(pix_url) + self.icons[provider] = pixmap + else: + pixmap = self.icons[provider] + + point = QtCore.QPoint(option.rect.x() + + (option.rect.width() - pixmap.width()) / 2, + option.rect.y() + + (option.rect.height() - pixmap.height()) / 2) + painter.drawPixmap(point, pixmap) + + painter.setOpacity(0.5) + overlay_rect = option.rect + overlay_rect.setHeight(overlay_rect.height() * (1.0 - float(value))) + painter.fillRect(overlay_rect, + QtGui.QBrush(QtGui.QColor(0, 0, 0, 200))) + painter.setOpacity(1) + + +class SyncRepresentationErrorWindow(QtWidgets.QDialog): + def __init__(self, _id, project, dt, tries, msg, parent=None): + super(SyncRepresentationErrorWindow, self).__init__(parent) + self.setWindowFlags(QtCore.Qt.Window) + self.setFocusPolicy(QtCore.Qt.StrongFocus) + + self.setStyleSheet(style.load_stylesheet()) + self.setWindowIcon(QtGui.QIcon(style.app_icon_path())) + self.resize(250, 200) + + body = QtWidgets.QWidget() + footer = QtWidgets.QWidget() + footer.setFixedHeight(20) + + container = SyncRepresentationErrorWidget(_id, project, dt, tries, msg, + parent=self) + body_layout = QtWidgets.QHBoxLayout(body) + body_layout.addWidget(container) + body_layout.setContentsMargins(0, 0, 0, 0) + + message = QtWidgets.QLabel() + message.hide() + + footer_layout = QtWidgets.QVBoxLayout(footer) + footer_layout.addWidget(message) + footer_layout.setContentsMargins(0, 0, 0, 0) + + layout = QtWidgets.QVBoxLayout(self) + layout.addWidget(body) + layout.addWidget(footer) + + self.setLayout(body_layout) + self.setWindowTitle("Sync Representation Error Detail") + + +class SyncRepresentationErrorWidget(QtWidgets.QWidget): + """ + Dialog to show when sync error happened, prints error message + """ + + def __init__(self, _id, project, dt, tries, msg, parent=None): + super(SyncRepresentationErrorWidget, self).__init__(parent) + + layout = QtWidgets.QFormLayout(self) + layout.addRow(QtWidgets.QLabel("Last update date"), + QtWidgets.QLabel(pretty_timestamp(dt))) + layout.addRow(QtWidgets.QLabel("Retries"), + QtWidgets.QLabel(str(tries))) + layout.addRow(QtWidgets.QLabel("Error message"), + QtWidgets.QLabel(msg)) + + +class SizeDelegate(QtWidgets.QStyledItemDelegate): + """ + Pretty print for file size + """ + + def __init__(self, parent=None): + super(SizeDelegate, self).__init__(parent) + + def displayText(self, value, locale): + if value is None: + # Ignore None value + return + + return self._pretty_size(value) + + def _pretty_size(self, value, suffix='B'): + for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']: + if abs(value) < 1024.0: + return "%3.1f%s%s" % (value, unit, suffix) + value /= 1024.0 + return "%.1f%s%s" % (value, 'Yi', suffix) diff --git a/pype/plugins/global/publish/extract_burnin.py b/pype/plugins/global/publish/extract_burnin.py index c14dfba50a..8d1b9c81e3 100644 --- a/pype/plugins/global/publish/extract_burnin.py +++ b/pype/plugins/global/publish/extract_burnin.py @@ -814,9 +814,9 @@ class ExtractBurnin(pype.api.Extractor): """Return path to python script for burnin processing.""" # TODO maybe convert to Plugin's attribute # Get script path. - module_path = os.environ["PYPE_MODULE_ROOT"] + module_path = os.environ["PYPE_ROOT"] - # There can be multiple paths in PYPE_MODULE_ROOT, in which case + # There can be multiple paths in PYPE_ROOT, in which case # we just take first one. if os.pathsep in module_path: module_path = module_path.split(os.pathsep)[0] diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index 2e5a9e5b94..5ba92435fd 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -908,7 +908,8 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): file_info = self.prepare_file_info(path, integrated_file_sizes[dest], - file_hash) + file_hash, + instance=instance) output_resources.append(file_info) return output_resources @@ -928,7 +929,8 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): dest += '.{}'.format(self.TMP_FILE_EXT) return dest - def prepare_file_info(self, path, size=None, file_hash=None, sites=None): + def prepare_file_info(self, path, size=None, file_hash=None, + sites=None, instance=None): """ Prepare information for one file (asset or resource) Arguments: @@ -938,6 +940,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): sites(optional): array of published locations, [ {'name':'studio', 'created_dt':date} by default keys expected ['studio', 'site1', 'gdrive1'] + instance(dict, optional): to get collected settings Returns: rec: dictionary with filled info """ @@ -945,15 +948,18 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): remote_site = None sync_server_presets = None - # manager = ModulesManager() - # sync_server = manager.modules_by_name["sync_server"] - # try: - # if sync_server.enabled: - # local_site, remote_site = sync_server.get_sites_for_project() - # except ValueError: - # log.debug(("There are not set presets for SyncServer." - # " No credentials provided, no synching possible"). - # format(str(sync_server_presets))) + if (instance.context.data["system_settings"] + ["modules"] + ["sync_server"] + ["enabled"]): + sync_server_presets = (instance.context.data["project_settings"] + ["global"] + ["sync_server"]) + + if sync_server_presets["enabled"]: + local_site = sync_server_presets["config"].\ + get("active_site", "studio").strip() + remote_site = sync_server_presets["config"].get("remote_site") rec = { "_id": io.ObjectId(), diff --git a/pype/pype_commands.py b/pype/pype_commands.py index 36e5ac639f..1ec4d2c553 100644 --- a/pype/pype_commands.py +++ b/pype/pype_commands.py @@ -1,8 +1,9 @@ # -*- coding: utf-8 -*- """Implementation of Pype commands.""" import os -import subprocess import sys +from pathlib import Path + from pype.lib import PypeLogger @@ -61,3 +62,27 @@ class PypeCommands: def validate_jsons(self): pass + + @staticmethod + def generate_zip(out_path: str): + """Generate zip file from current sources. + + Args: + out_path (str): Path to generated zip file. + + """ + from igniter import bootstrap_repos + + # create zip file + bs = bootstrap_repos.BootstrapRepos() + if out_path: + out_path = Path(out_path) + bs.data_dir = out_path.parent + + print(f">>> Creating zip in {bs.data_dir} ...") + repo_file = bs.install_live_repos() + if not repo_file: + print("!!! Error while creating zip file.") + exit(1) + + print(f">>> Created {repo_file}") diff --git a/pype/settings/defaults/system_settings/applications.json b/pype/settings/defaults/system_settings/applications.json index 641c9f25da..ab2fdcbb57 100644 --- a/pype/settings/defaults/system_settings/applications.json +++ b/pype/settings/defaults/system_settings/applications.json @@ -13,14 +13,13 @@ "MAYA_DISABLE_CER", "PYMEL_SKIP_MEL_INIT", "LC_ALL", - "PYPE_LOG_NO_COLORS", - "MAYA_TEST" + "PYPE_LOG_NO_COLORS" ] }, "PYTHONPATH": [ - "{PYPE_MODULE_ROOT}/repos/avalon-core/setup/maya", - "{PYPE_MODULE_ROOT}/repos/maya-look-assigner", - "{PYTHON_ENV}/python2/Lib/site-packages", + "{PYPE_ROOT}/pype/setup/maya", + "{PYPE_REPOS_ROOT}/avalon-core/setup/maya", + "{PYPE_REPOS_ROOT}/maya-look-assigner", "{PYTHONPATH}" ], "MAYA_DISABLE_CLIC_IPM": "Yes", @@ -28,8 +27,7 @@ "MAYA_DISABLE_CER": "Yes", "PYMEL_SKIP_MEL_INIT": "Yes", "LC_ALL": "C", - "PYPE_LOG_NO_COLORS": "Yes", - "MAYA_TEST": "{MAYA_VERSION}" + "PYPE_LOG_NO_COLORS": "Yes" }, "variants": { "maya_2020": { @@ -140,8 +138,8 @@ ] }, "PYTHONPATH": [ - "{PYPE_MODULE_ROOT}/repos/avalon-core/setup/maya", - "{PYPE_MODULE_ROOT}/repos/maya-look-assigner", + "{PYPE_REPOS_ROOT}/avalon-core/setup/maya", + "{PYPE_REPOS_ROOT}/maya-look-assigner", "{PYTHON_ENV}/python2/Lib/site-packages", "{PYTHONPATH}" ], @@ -242,7 +240,7 @@ ] }, "NUKE_PATH": [ - "{PYPE_ROOT}/repos/avalon-core/setup/nuke/nuke_path", + "{PYPE_REPOS_ROOT}/avalon-core/setup/nuke/nuke_path", "{PYPE_ROOT}/pype/hosts/nuke/startup", "{PYPE_STUDIO_PLUGINS}/nuke" ], @@ -367,7 +365,7 @@ ] }, "NUKE_PATH": [ - "{PYPE_ROOT}/repos/avalon-core/setup/nuke/nuke_path", + "{PYPE_REPOS_ROOT}/avalon-core/setup/nuke/nuke_path", "{PYPE_ROOT}/pype/hosts/nuke/startup", "{PYPE_STUDIO_PLUGINS}/nuke" ], @@ -494,7 +492,7 @@ ] }, "HIERO_PLUGIN_PATH": [ - "{PYPE_MODULE_ROOT}/setup/hiero/hiero_plugin_path" + "{PYPE_ROOT}/setup/hiero/hiero_plugin_path" ], "PATH": { "windows": "C:/Program Files (x86)/QuickTime/QTSystem/;{PATH}" @@ -616,7 +614,7 @@ ] }, "HIERO_PLUGIN_PATH": [ - "{PYPE_MODULE_ROOT}/setup/hiero/hiero_plugin_path" + "{PYPE_ROOT}/setup/hiero/hiero_plugin_path" ], "PATH": { "windows": "C:/Program Files (x86)/QuickTime/QTSystem/;{PATH}" @@ -815,8 +813,6 @@ }, "PYTHONPATH": [ "{PYTHON36_RESOLVE}/Lib/site-packages", - "{VIRTUAL_ENV}/Lib/site-packages", - "{PYTHONPATH}", "{RESOLVE_SCRIPT_API}/Modules", "{PYTHONPATH}" ], @@ -825,7 +821,7 @@ "{PYTHON36_RESOLVE}/Scripts", "{PATH}" ], - "PRE_PYTHON_SCRIPT": "{PYPE_MODULE_ROOT}/pype/resolve/preload_console.py", + "PRE_PYTHON_SCRIPT": "{PYPE_ROOT}/pype/resolve/preload_console.py", "PYPE_LOG_NO_COLORS": "True", "RESOLVE_DEV": "True" }, @@ -866,14 +862,14 @@ ] }, "HOUDINI_PATH": { - "darwin": "{PYPE_MODULE_ROOT}/setup/houdini:&", - "linux": "{PYPE_MODULE_ROOT}/setup/houdini:&", - "windows": "{PYPE_MODULE_ROOT}/setup/houdini;&" + "darwin": "{PYPE_ROOT}/setup/houdini:&", + "linux": "{PYPE_ROOT}/setup/houdini:&", + "windows": "{PYPE_ROOT}/setup/houdini;&" }, "HOUDINI_MENU_PATH": { - "darwin": "{PYPE_MODULE_ROOT}/setup/houdini:&", - "linux": "{PYPE_MODULE_ROOT}/setup/houdini:&", - "windows": "{PYPE_MODULE_ROOT}/setup/houdini;&" + "darwin": "{PYPE_ROOT}/setup/houdini:&", + "linux": "{PYPE_ROOT}/setup/houdini:&", + "windows": "{PYPE_ROOT}/setup/houdini;&" } }, "variants": { @@ -924,9 +920,9 @@ "CREATE_NEW_CONSOLE" ] }, - "BLENDER_USER_SCRIPTS": "{PYPE_MODULE_ROOT}/repos/avalon-core/setup/blender", + "BLENDER_USER_SCRIPTS": "{PYPE_REPOS_ROOT}/avalon-core/setup/blender", "PYTHONPATH": [ - "{PYPE_MODULE_ROOT}/repos/avalon-core/setup/blender", + "{PYPE_REPOS_ROOT}/avalon-core/setup/blender", "{PYTHONPATH}" ], "CREATE_NEW_CONSOLE": "yes" @@ -1102,14 +1098,12 @@ "__environment_keys__": { "photoshop": [ "AVALON_PHOTOSHOP_WORKFILES_ON_LAUNCH", - "PYTHONPATH", "PYPE_LOG_NO_COLORS", "WEBSOCKET_URL", "WORKFILES_SAVE_AS" ] }, "AVALON_PHOTOSHOP_WORKFILES_ON_LAUNCH": "1", - "PYTHONPATH": "{PYTHONPATH}", "PYPE_LOG_NO_COLORS": "Yes", "WEBSOCKET_URL": "ws://localhost:8099/ws/", "WORKFILES_SAVE_AS": "Yes" @@ -1168,14 +1162,12 @@ "__environment_keys__": { "aftereffects": [ "AVALON_AFTEREFFECTS_WORKFILES_ON_LAUNCH", - "PYTHONPATH", "PYPE_LOG_NO_COLORS", "WEBSOCKET_URL", "WORKFILES_SAVE_AS" ] }, "AVALON_AFTEREFFECTS_WORKFILES_ON_LAUNCH": "1", - "PYTHONPATH": "{PYTHONPATH}", "PYPE_LOG_NO_COLORS": "Yes", "WEBSOCKET_URL": "ws://localhost:8097/ws/", "WORKFILES_SAVE_AS": "Yes" @@ -1236,7 +1228,7 @@ "CELACTION_TEMPLATE" ] }, - "CELACTION_TEMPLATE": "{PYPE_MODULE_ROOT}/pype/hosts/celaction/celaction_template_scene.scn" + "CELACTION_TEMPLATE": "{PYPE_ROOT}/pype/hosts/celaction/celaction_template_scene.scn" }, "variants": { "celation_Local": { @@ -1284,7 +1276,7 @@ "QT_PREFERRED_BINDING" ] }, - "AVALON_UNREAL_PLUGIN": "{PYPE_MODULE_ROOT}/repos/avalon-unreal-integration", + "AVALON_UNREAL_PLUGIN": "{PYPE_REPOS_ROOT}/avalon-unreal-integration", "PYPE_LOG_NO_COLORS": "True", "QT_PREFERRED_BINDING": "PySide" }, diff --git a/pype/settings/defaults/system_settings/general.json b/pype/settings/defaults/system_settings/general.json index 1e7d4b697f..d1810148cd 100644 --- a/pype/settings/defaults/system_settings/general.json +++ b/pype/settings/defaults/system_settings/general.json @@ -15,9 +15,6 @@ "__environment_keys__": { "global": [ "FFMPEG_PATH", - "PATH", - "PYTHONPATH", - "PYPE_PROJECT_CONFIGS", "PYPE_PYTHON_EXE", "PYPE_OCIO_CONFIG", "PYBLISH_GUI", @@ -26,19 +23,9 @@ }, "FFMPEG_PATH": { "windows": "{PYPE_ROOT}/vendor/bin/ffmpeg_exec/windows/bin", - "darwin": "{VIRTUAL_ENV}/localized/ffmpeg_exec/darwin/bin:{PYPE_SETUP_PATH}/vendor/bin/ffmpeg_exec/darwin/bin", - "linux": "{VIRTUAL_ENV}/localized/ffmpeg_exec/linux:{PYPE_SETUP_PATH}/vendor/bin/ffmpeg_exec/linux" + "darwin": "{PYPE_ROOT}/vendor/bin/ffmpeg_exec/darwin/bin", + "linux": ":{PYPE_ROOT}/vendor/bin/ffmpeg_exec/linux" }, - "PATH": [ - "{FFMPEG_PATH}", - "{PATH}" - ], - "PYTHONPATH": { - "windows": "{VIRTUAL_ENV}/Lib/site-packages;{PYPE_MODULE_ROOT}/pype/tools;{PYPE_MODULE_ROOT}/pype/vendor;{PYTHONPATH}", - "linux": "{VIRTUAL_ENV}/lib/python{PYTHON_VERSION}/site-packages:{PYPE_MODULE_ROOT}/pype/tools:{PYPE_MODULE_ROOT}/pype/vendor:{PYTHONPATH}", - "darwin": "{VIRTUAL_ENV}/lib/python{PYTHON_VERSION}/site-packages:{PYPE_MODULE_ROOT}/pype/tools:{PYPE_MODULE_ROOT}/pype/vendor:{PYTHONPATH}" - }, - "PYPE_PROJECT_CONFIGS": "{PYPE_SETUP_PATH}/../studio-project-configs", "PYPE_PYTHON_EXE": { "windows": "{VIRTUAL_ENV}/Scripts/python.exe", "linux": "{VIRTUAL_ENV}/Scripts/python", diff --git a/pype/settings/defaults/system_settings/modules.json b/pype/settings/defaults/system_settings/modules.json index 6493901bac..eb794b5f9d 100644 --- a/pype/settings/defaults/system_settings/modules.json +++ b/pype/settings/defaults/system_settings/modules.json @@ -61,7 +61,7 @@ } }, "is_hierarchical": { - "avalon_mongo_id": { + "tools_env": { "write_security_roles": [ "API", "Administrator", @@ -73,7 +73,7 @@ "Pypeclub" ] }, - "tools_env": { + "avalon_mongo_id": { "write_security_roles": [ "API", "Administrator", diff --git a/setup/blender/init.py b/pype/setup/blender/init.py similarity index 100% rename from setup/blender/init.py rename to pype/setup/blender/init.py diff --git a/setup/fusion/scripts/Comp/colorbleed/32bit/backgrounds_selected_to32bit.py b/pype/setup/fusion/scripts/Comp/colorbleed/32bit/backgrounds_selected_to32bit.py similarity index 100% rename from setup/fusion/scripts/Comp/colorbleed/32bit/backgrounds_selected_to32bit.py rename to pype/setup/fusion/scripts/Comp/colorbleed/32bit/backgrounds_selected_to32bit.py diff --git a/setup/fusion/scripts/Comp/colorbleed/32bit/backgrounds_to32bit.py b/pype/setup/fusion/scripts/Comp/colorbleed/32bit/backgrounds_to32bit.py similarity index 100% rename from setup/fusion/scripts/Comp/colorbleed/32bit/backgrounds_to32bit.py rename to pype/setup/fusion/scripts/Comp/colorbleed/32bit/backgrounds_to32bit.py diff --git a/setup/fusion/scripts/Comp/colorbleed/32bit/loaders_selected_to32bit.py b/pype/setup/fusion/scripts/Comp/colorbleed/32bit/loaders_selected_to32bit.py similarity index 100% rename from setup/fusion/scripts/Comp/colorbleed/32bit/loaders_selected_to32bit.py rename to pype/setup/fusion/scripts/Comp/colorbleed/32bit/loaders_selected_to32bit.py diff --git a/setup/fusion/scripts/Comp/colorbleed/32bit/loaders_to32bit.py b/pype/setup/fusion/scripts/Comp/colorbleed/32bit/loaders_to32bit.py similarity index 100% rename from setup/fusion/scripts/Comp/colorbleed/32bit/loaders_to32bit.py rename to pype/setup/fusion/scripts/Comp/colorbleed/32bit/loaders_to32bit.py diff --git a/setup/fusion/scripts/Comp/colorbleed/duplicate_with_input_connections.py b/pype/setup/fusion/scripts/Comp/colorbleed/duplicate_with_input_connections.py similarity index 100% rename from setup/fusion/scripts/Comp/colorbleed/duplicate_with_input_connections.py rename to pype/setup/fusion/scripts/Comp/colorbleed/duplicate_with_input_connections.py diff --git a/setup/fusion/scripts/Comp/colorbleed/set_rendermode.py b/pype/setup/fusion/scripts/Comp/colorbleed/set_rendermode.py similarity index 100% rename from setup/fusion/scripts/Comp/colorbleed/set_rendermode.py rename to pype/setup/fusion/scripts/Comp/colorbleed/set_rendermode.py diff --git a/setup/fusion/scripts/Comp/colorbleed/switch_ui.py b/pype/setup/fusion/scripts/Comp/colorbleed/switch_ui.py similarity index 100% rename from setup/fusion/scripts/Comp/colorbleed/switch_ui.py rename to pype/setup/fusion/scripts/Comp/colorbleed/switch_ui.py diff --git a/setup/fusion/scripts/Comp/colorbleed/update_selected_loader_ranges.py b/pype/setup/fusion/scripts/Comp/colorbleed/update_selected_loader_ranges.py similarity index 100% rename from setup/fusion/scripts/Comp/colorbleed/update_selected_loader_ranges.py rename to pype/setup/fusion/scripts/Comp/colorbleed/update_selected_loader_ranges.py diff --git a/setup/hiero/hiero_plugin_path/HieroPlayer/PlayerPresets.hrox b/pype/setup/hiero/hiero_plugin_path/HieroPlayer/PlayerPresets.hrox similarity index 100% rename from setup/hiero/hiero_plugin_path/HieroPlayer/PlayerPresets.hrox rename to pype/setup/hiero/hiero_plugin_path/HieroPlayer/PlayerPresets.hrox diff --git a/setup/hiero/hiero_plugin_path/Icons/1_add_handles_end.png b/pype/setup/hiero/hiero_plugin_path/Icons/1_add_handles_end.png similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/1_add_handles_end.png rename to pype/setup/hiero/hiero_plugin_path/Icons/1_add_handles_end.png diff --git a/setup/hiero/hiero_plugin_path/Icons/2_add_handles.png b/pype/setup/hiero/hiero_plugin_path/Icons/2_add_handles.png similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/2_add_handles.png rename to pype/setup/hiero/hiero_plugin_path/Icons/2_add_handles.png diff --git a/setup/hiero/hiero_plugin_path/Icons/3D.png b/pype/setup/hiero/hiero_plugin_path/Icons/3D.png similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/3D.png rename to pype/setup/hiero/hiero_plugin_path/Icons/3D.png diff --git a/setup/hiero/hiero_plugin_path/Icons/3_add_handles_start.png b/pype/setup/hiero/hiero_plugin_path/Icons/3_add_handles_start.png similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/3_add_handles_start.png rename to pype/setup/hiero/hiero_plugin_path/Icons/3_add_handles_start.png diff --git a/setup/hiero/hiero_plugin_path/Icons/4_2D.png b/pype/setup/hiero/hiero_plugin_path/Icons/4_2D.png similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/4_2D.png rename to pype/setup/hiero/hiero_plugin_path/Icons/4_2D.png diff --git a/setup/hiero/hiero_plugin_path/Icons/edit.png b/pype/setup/hiero/hiero_plugin_path/Icons/edit.png similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/edit.png rename to pype/setup/hiero/hiero_plugin_path/Icons/edit.png diff --git a/setup/hiero/hiero_plugin_path/Icons/fusion.png b/pype/setup/hiero/hiero_plugin_path/Icons/fusion.png similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/fusion.png rename to pype/setup/hiero/hiero_plugin_path/Icons/fusion.png diff --git a/setup/hiero/hiero_plugin_path/Icons/hierarchy.png b/pype/setup/hiero/hiero_plugin_path/Icons/hierarchy.png similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/hierarchy.png rename to pype/setup/hiero/hiero_plugin_path/Icons/hierarchy.png diff --git a/setup/hiero/hiero_plugin_path/Icons/houdini.png b/pype/setup/hiero/hiero_plugin_path/Icons/houdini.png similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/houdini.png rename to pype/setup/hiero/hiero_plugin_path/Icons/houdini.png diff --git a/setup/hiero/hiero_plugin_path/Icons/layers.psd b/pype/setup/hiero/hiero_plugin_path/Icons/layers.psd similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/layers.psd rename to pype/setup/hiero/hiero_plugin_path/Icons/layers.psd diff --git a/setup/hiero/hiero_plugin_path/Icons/lense.png b/pype/setup/hiero/hiero_plugin_path/Icons/lense.png similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/lense.png rename to pype/setup/hiero/hiero_plugin_path/Icons/lense.png diff --git a/setup/hiero/hiero_plugin_path/Icons/lense1.png b/pype/setup/hiero/hiero_plugin_path/Icons/lense1.png similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/lense1.png rename to pype/setup/hiero/hiero_plugin_path/Icons/lense1.png diff --git a/setup/hiero/hiero_plugin_path/Icons/maya.png b/pype/setup/hiero/hiero_plugin_path/Icons/maya.png similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/maya.png rename to pype/setup/hiero/hiero_plugin_path/Icons/maya.png diff --git a/setup/hiero/hiero_plugin_path/Icons/nuke.png b/pype/setup/hiero/hiero_plugin_path/Icons/nuke.png similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/nuke.png rename to pype/setup/hiero/hiero_plugin_path/Icons/nuke.png diff --git a/setup/hiero/hiero_plugin_path/Icons/resolution.png b/pype/setup/hiero/hiero_plugin_path/Icons/resolution.png similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/resolution.png rename to pype/setup/hiero/hiero_plugin_path/Icons/resolution.png diff --git a/setup/hiero/hiero_plugin_path/Icons/resolution.psd b/pype/setup/hiero/hiero_plugin_path/Icons/resolution.psd similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/resolution.psd rename to pype/setup/hiero/hiero_plugin_path/Icons/resolution.psd diff --git a/setup/hiero/hiero_plugin_path/Icons/retiming.png b/pype/setup/hiero/hiero_plugin_path/Icons/retiming.png similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/retiming.png rename to pype/setup/hiero/hiero_plugin_path/Icons/retiming.png diff --git a/setup/hiero/hiero_plugin_path/Icons/retiming.psd b/pype/setup/hiero/hiero_plugin_path/Icons/retiming.psd similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/retiming.psd rename to pype/setup/hiero/hiero_plugin_path/Icons/retiming.psd diff --git a/setup/hiero/hiero_plugin_path/Icons/review.png b/pype/setup/hiero/hiero_plugin_path/Icons/review.png similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/review.png rename to pype/setup/hiero/hiero_plugin_path/Icons/review.png diff --git a/setup/hiero/hiero_plugin_path/Icons/review.psd b/pype/setup/hiero/hiero_plugin_path/Icons/review.psd similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/review.psd rename to pype/setup/hiero/hiero_plugin_path/Icons/review.psd diff --git a/setup/hiero/hiero_plugin_path/Icons/volume.png b/pype/setup/hiero/hiero_plugin_path/Icons/volume.png similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/volume.png rename to pype/setup/hiero/hiero_plugin_path/Icons/volume.png diff --git a/setup/hiero/hiero_plugin_path/Icons/z_layer_bg.png b/pype/setup/hiero/hiero_plugin_path/Icons/z_layer_bg.png similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/z_layer_bg.png rename to pype/setup/hiero/hiero_plugin_path/Icons/z_layer_bg.png diff --git a/setup/hiero/hiero_plugin_path/Icons/z_layer_fg.png b/pype/setup/hiero/hiero_plugin_path/Icons/z_layer_fg.png similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/z_layer_fg.png rename to pype/setup/hiero/hiero_plugin_path/Icons/z_layer_fg.png diff --git a/setup/hiero/hiero_plugin_path/Icons/z_layer_main.png b/pype/setup/hiero/hiero_plugin_path/Icons/z_layer_main.png similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/z_layer_main.png rename to pype/setup/hiero/hiero_plugin_path/Icons/z_layer_main.png diff --git a/setup/hiero/hiero_plugin_path/Python/Startup/SpreadsheetExport.py b/pype/setup/hiero/hiero_plugin_path/Python/Startup/SpreadsheetExport.py similarity index 100% rename from setup/hiero/hiero_plugin_path/Python/Startup/SpreadsheetExport.py rename to pype/setup/hiero/hiero_plugin_path/Python/Startup/SpreadsheetExport.py diff --git a/setup/hiero/hiero_plugin_path/Python/Startup/Startup.py b/pype/setup/hiero/hiero_plugin_path/Python/Startup/Startup.py similarity index 100% rename from setup/hiero/hiero_plugin_path/Python/Startup/Startup.py rename to pype/setup/hiero/hiero_plugin_path/Python/Startup/Startup.py diff --git a/setup/hiero/hiero_plugin_path/Python/Startup/otioexporter/OTIOExportTask.py b/pype/setup/hiero/hiero_plugin_path/Python/Startup/otioexporter/OTIOExportTask.py similarity index 100% rename from setup/hiero/hiero_plugin_path/Python/Startup/otioexporter/OTIOExportTask.py rename to pype/setup/hiero/hiero_plugin_path/Python/Startup/otioexporter/OTIOExportTask.py diff --git a/setup/hiero/hiero_plugin_path/Python/Startup/otioexporter/OTIOExportUI.py b/pype/setup/hiero/hiero_plugin_path/Python/Startup/otioexporter/OTIOExportUI.py similarity index 100% rename from setup/hiero/hiero_plugin_path/Python/Startup/otioexporter/OTIOExportUI.py rename to pype/setup/hiero/hiero_plugin_path/Python/Startup/otioexporter/OTIOExportUI.py diff --git a/setup/hiero/hiero_plugin_path/Python/Startup/otioexporter/__init__.py b/pype/setup/hiero/hiero_plugin_path/Python/Startup/otioexporter/__init__.py similarity index 100% rename from setup/hiero/hiero_plugin_path/Python/Startup/otioexporter/__init__.py rename to pype/setup/hiero/hiero_plugin_path/Python/Startup/otioexporter/__init__.py diff --git a/setup/hiero/hiero_plugin_path/Python/Startup/project_helpers.py b/pype/setup/hiero/hiero_plugin_path/Python/Startup/project_helpers.py similarity index 100% rename from setup/hiero/hiero_plugin_path/Python/Startup/project_helpers.py rename to pype/setup/hiero/hiero_plugin_path/Python/Startup/project_helpers.py diff --git a/setup/hiero/hiero_plugin_path/Python/Startup/selection_tracker.py b/pype/setup/hiero/hiero_plugin_path/Python/Startup/selection_tracker.py similarity index 100% rename from setup/hiero/hiero_plugin_path/Python/Startup/selection_tracker.py rename to pype/setup/hiero/hiero_plugin_path/Python/Startup/selection_tracker.py diff --git a/setup/hiero/hiero_plugin_path/Python/Startup/setFrameRate.py b/pype/setup/hiero/hiero_plugin_path/Python/Startup/setFrameRate.py similarity index 100% rename from setup/hiero/hiero_plugin_path/Python/Startup/setFrameRate.py rename to pype/setup/hiero/hiero_plugin_path/Python/Startup/setFrameRate.py diff --git a/setup/hiero/hiero_plugin_path/Python/Startup/version_everywhere.py b/pype/setup/hiero/hiero_plugin_path/Python/Startup/version_everywhere.py similarity index 100% rename from setup/hiero/hiero_plugin_path/Python/Startup/version_everywhere.py rename to pype/setup/hiero/hiero_plugin_path/Python/Startup/version_everywhere.py diff --git a/setup/hiero/hiero_plugin_path/Python/StartupUI/PimpMySpreadsheet.py b/pype/setup/hiero/hiero_plugin_path/Python/StartupUI/PimpMySpreadsheet.py similarity index 100% rename from setup/hiero/hiero_plugin_path/Python/StartupUI/PimpMySpreadsheet.py rename to pype/setup/hiero/hiero_plugin_path/Python/StartupUI/PimpMySpreadsheet.py diff --git a/setup/hiero/hiero_plugin_path/Python/StartupUI/Purge.py b/pype/setup/hiero/hiero_plugin_path/Python/StartupUI/Purge.py similarity index 100% rename from setup/hiero/hiero_plugin_path/Python/StartupUI/Purge.py rename to pype/setup/hiero/hiero_plugin_path/Python/StartupUI/Purge.py diff --git a/setup/hiero/hiero_plugin_path/Python/StartupUI/nukeStyleKeyboardShortcuts.py b/pype/setup/hiero/hiero_plugin_path/Python/StartupUI/nukeStyleKeyboardShortcuts.py similarity index 100% rename from setup/hiero/hiero_plugin_path/Python/StartupUI/nukeStyleKeyboardShortcuts.py rename to pype/setup/hiero/hiero_plugin_path/Python/StartupUI/nukeStyleKeyboardShortcuts.py diff --git a/setup/hiero/hiero_plugin_path/Python/StartupUI/otioimporter/OTIOImport.py b/pype/setup/hiero/hiero_plugin_path/Python/StartupUI/otioimporter/OTIOImport.py similarity index 100% rename from setup/hiero/hiero_plugin_path/Python/StartupUI/otioimporter/OTIOImport.py rename to pype/setup/hiero/hiero_plugin_path/Python/StartupUI/otioimporter/OTIOImport.py diff --git a/setup/hiero/hiero_plugin_path/Python/StartupUI/otioimporter/__init__.py b/pype/setup/hiero/hiero_plugin_path/Python/StartupUI/otioimporter/__init__.py similarity index 100% rename from setup/hiero/hiero_plugin_path/Python/StartupUI/otioimporter/__init__.py rename to pype/setup/hiero/hiero_plugin_path/Python/StartupUI/otioimporter/__init__.py diff --git a/setup/hiero/hiero_plugin_path/Python/StartupUI/setPosterFrame.py b/pype/setup/hiero/hiero_plugin_path/Python/StartupUI/setPosterFrame.py similarity index 100% rename from setup/hiero/hiero_plugin_path/Python/StartupUI/setPosterFrame.py rename to pype/setup/hiero/hiero_plugin_path/Python/StartupUI/setPosterFrame.py diff --git a/setup/hiero/hiero_plugin_path/TaskPresets/10.5/Processors/hiero.exporters.FnShotProcessor.ShotProcessor/pipeline.xml b/pype/setup/hiero/hiero_plugin_path/TaskPresets/10.5/Processors/hiero.exporters.FnShotProcessor.ShotProcessor/pipeline.xml similarity index 100% rename from setup/hiero/hiero_plugin_path/TaskPresets/10.5/Processors/hiero.exporters.FnShotProcessor.ShotProcessor/pipeline.xml rename to pype/setup/hiero/hiero_plugin_path/TaskPresets/10.5/Processors/hiero.exporters.FnShotProcessor.ShotProcessor/pipeline.xml diff --git a/setup/hiero/hiero_plugin_path/TaskPresets/11.1/Processors/hiero.exporters.FnShotProcessor.ShotProcessor/pipeline.xml b/pype/setup/hiero/hiero_plugin_path/TaskPresets/11.1/Processors/hiero.exporters.FnShotProcessor.ShotProcessor/pipeline.xml similarity index 100% rename from setup/hiero/hiero_plugin_path/TaskPresets/11.1/Processors/hiero.exporters.FnShotProcessor.ShotProcessor/pipeline.xml rename to pype/setup/hiero/hiero_plugin_path/TaskPresets/11.1/Processors/hiero.exporters.FnShotProcessor.ShotProcessor/pipeline.xml diff --git a/setup/hiero/hiero_plugin_path/TaskPresets/11.2/hiero.exporters.FnShotProcessor.ShotProcessor/pipeline.xml b/pype/setup/hiero/hiero_plugin_path/TaskPresets/11.2/hiero.exporters.FnShotProcessor.ShotProcessor/pipeline.xml similarity index 100% rename from setup/hiero/hiero_plugin_path/TaskPresets/11.2/hiero.exporters.FnShotProcessor.ShotProcessor/pipeline.xml rename to pype/setup/hiero/hiero_plugin_path/TaskPresets/11.2/hiero.exporters.FnShotProcessor.ShotProcessor/pipeline.xml diff --git a/setup/houdini/MainMenuCommon.XML b/pype/setup/houdini/MainMenuCommon.XML similarity index 100% rename from setup/houdini/MainMenuCommon.XML rename to pype/setup/houdini/MainMenuCommon.XML diff --git a/setup/houdini/scripts/123.py b/pype/setup/houdini/scripts/123.py similarity index 100% rename from setup/houdini/scripts/123.py rename to pype/setup/houdini/scripts/123.py diff --git a/setup/maya/userSetup.py b/pype/setup/maya/userSetup.py similarity index 100% rename from setup/maya/userSetup.py rename to pype/setup/maya/userSetup.py diff --git a/pype/tools/settings/settings/style/style.css b/pype/tools/settings/settings/style/style.css index f3eb3a258e..3ce9837a8b 100644 --- a/pype/tools/settings/settings/style/style.css +++ b/pype/tools/settings/settings/style/style.css @@ -353,3 +353,39 @@ QScrollBar::up-arrow:vertical, QScrollBar::down-arrow:vertical { QScrollBar::add-page:vertical, QScrollBar::sub-page:vertical { background: none; } + +QTableView +{ + border: 1px solid #444; + gridline-color: #6c6c6c; + background-color: #201F1F; + alternate-background-color:#21252B; +} + +QHeaderView +{ + border: 1px transparent; + border-radius: 2px; + margin: 0px; + padding: 0px; +} + +QHeaderView::section { + background-color: #21252B; + /*color: silver;*/ + padding: 4px; + border: 1px solid #6c6c6c; + border-radius: 0px; + text-align: center; + color: #969b9e; + font-weight: bold; +} + +QTableView::item:pressed, QListView::item:pressed, QTreeView::item:pressed { + background: #78879b; + color: #FFFFFF; +} + +QTableView::item:selected:active, QTreeView::item:selected:active, QListView::item:selected:active { + background: #3d8ec9; +} \ No newline at end of file diff --git a/pype/vendor/python/common/README.md b/pype/vendor/python/common/README.md new file mode 100644 index 0000000000..52effb5a63 --- /dev/null +++ b/pype/vendor/python/common/README.md @@ -0,0 +1,2 @@ +### Info +Here are modules that can run under both Python 2 and Python 3 hosts. \ No newline at end of file diff --git a/pype/vendor/capture.py b/pype/vendor/python/common/capture.py similarity index 100% rename from pype/vendor/capture.py rename to pype/vendor/python/common/capture.py diff --git a/pype/vendor/capture_gui/__init__.py b/pype/vendor/python/common/capture_gui/__init__.py similarity index 100% rename from pype/vendor/capture_gui/__init__.py rename to pype/vendor/python/common/capture_gui/__init__.py diff --git a/pype/vendor/capture_gui/accordion.py b/pype/vendor/python/common/capture_gui/accordion.py similarity index 100% rename from pype/vendor/capture_gui/accordion.py rename to pype/vendor/python/common/capture_gui/accordion.py diff --git a/pype/vendor/capture_gui/app.py b/pype/vendor/python/common/capture_gui/app.py similarity index 100% rename from pype/vendor/capture_gui/app.py rename to pype/vendor/python/common/capture_gui/app.py diff --git a/pype/vendor/capture_gui/colorpicker.py b/pype/vendor/python/common/capture_gui/colorpicker.py similarity index 100% rename from pype/vendor/capture_gui/colorpicker.py rename to pype/vendor/python/common/capture_gui/colorpicker.py diff --git a/pype/vendor/capture_gui/lib.py b/pype/vendor/python/common/capture_gui/lib.py similarity index 100% rename from pype/vendor/capture_gui/lib.py rename to pype/vendor/python/common/capture_gui/lib.py diff --git a/pype/vendor/capture_gui/plugin.py b/pype/vendor/python/common/capture_gui/plugin.py similarity index 100% rename from pype/vendor/capture_gui/plugin.py rename to pype/vendor/python/common/capture_gui/plugin.py diff --git a/pype/vendor/capture_gui/plugins/cameraplugin.py b/pype/vendor/python/common/capture_gui/plugins/cameraplugin.py similarity index 100% rename from pype/vendor/capture_gui/plugins/cameraplugin.py rename to pype/vendor/python/common/capture_gui/plugins/cameraplugin.py diff --git a/pype/vendor/capture_gui/plugins/codecplugin.py b/pype/vendor/python/common/capture_gui/plugins/codecplugin.py similarity index 100% rename from pype/vendor/capture_gui/plugins/codecplugin.py rename to pype/vendor/python/common/capture_gui/plugins/codecplugin.py diff --git a/pype/vendor/capture_gui/plugins/defaultoptionsplugin.py b/pype/vendor/python/common/capture_gui/plugins/defaultoptionsplugin.py similarity index 100% rename from pype/vendor/capture_gui/plugins/defaultoptionsplugin.py rename to pype/vendor/python/common/capture_gui/plugins/defaultoptionsplugin.py diff --git a/pype/vendor/capture_gui/plugins/displayplugin.py b/pype/vendor/python/common/capture_gui/plugins/displayplugin.py similarity index 100% rename from pype/vendor/capture_gui/plugins/displayplugin.py rename to pype/vendor/python/common/capture_gui/plugins/displayplugin.py diff --git a/pype/vendor/capture_gui/plugins/genericplugin.py b/pype/vendor/python/common/capture_gui/plugins/genericplugin.py similarity index 100% rename from pype/vendor/capture_gui/plugins/genericplugin.py rename to pype/vendor/python/common/capture_gui/plugins/genericplugin.py diff --git a/pype/vendor/capture_gui/plugins/ioplugin.py b/pype/vendor/python/common/capture_gui/plugins/ioplugin.py similarity index 100% rename from pype/vendor/capture_gui/plugins/ioplugin.py rename to pype/vendor/python/common/capture_gui/plugins/ioplugin.py diff --git a/pype/vendor/capture_gui/plugins/panzoomplugin.py b/pype/vendor/python/common/capture_gui/plugins/panzoomplugin.py similarity index 100% rename from pype/vendor/capture_gui/plugins/panzoomplugin.py rename to pype/vendor/python/common/capture_gui/plugins/panzoomplugin.py diff --git a/pype/vendor/capture_gui/plugins/rendererplugin.py b/pype/vendor/python/common/capture_gui/plugins/rendererplugin.py similarity index 100% rename from pype/vendor/capture_gui/plugins/rendererplugin.py rename to pype/vendor/python/common/capture_gui/plugins/rendererplugin.py diff --git a/pype/vendor/capture_gui/plugins/resolutionplugin.py b/pype/vendor/python/common/capture_gui/plugins/resolutionplugin.py similarity index 100% rename from pype/vendor/capture_gui/plugins/resolutionplugin.py rename to pype/vendor/python/common/capture_gui/plugins/resolutionplugin.py diff --git a/pype/vendor/capture_gui/plugins/timeplugin.py b/pype/vendor/python/common/capture_gui/plugins/timeplugin.py similarity index 100% rename from pype/vendor/capture_gui/plugins/timeplugin.py rename to pype/vendor/python/common/capture_gui/plugins/timeplugin.py diff --git a/pype/vendor/capture_gui/plugins/viewportplugin.py b/pype/vendor/python/common/capture_gui/plugins/viewportplugin.py similarity index 100% rename from pype/vendor/capture_gui/plugins/viewportplugin.py rename to pype/vendor/python/common/capture_gui/plugins/viewportplugin.py diff --git a/pype/vendor/capture_gui/presets.py b/pype/vendor/python/common/capture_gui/presets.py similarity index 100% rename from pype/vendor/capture_gui/presets.py rename to pype/vendor/python/common/capture_gui/presets.py diff --git a/pype/vendor/capture_gui/resources/config.png b/pype/vendor/python/common/capture_gui/resources/config.png similarity index 100% rename from pype/vendor/capture_gui/resources/config.png rename to pype/vendor/python/common/capture_gui/resources/config.png diff --git a/pype/vendor/capture_gui/resources/import.png b/pype/vendor/python/common/capture_gui/resources/import.png similarity index 100% rename from pype/vendor/capture_gui/resources/import.png rename to pype/vendor/python/common/capture_gui/resources/import.png diff --git a/pype/vendor/capture_gui/resources/reset.png b/pype/vendor/python/common/capture_gui/resources/reset.png similarity index 100% rename from pype/vendor/capture_gui/resources/reset.png rename to pype/vendor/python/common/capture_gui/resources/reset.png diff --git a/pype/vendor/capture_gui/resources/save.png b/pype/vendor/python/common/capture_gui/resources/save.png similarity index 100% rename from pype/vendor/capture_gui/resources/save.png rename to pype/vendor/python/common/capture_gui/resources/save.png diff --git a/pype/vendor/capture_gui/tokens.py b/pype/vendor/python/common/capture_gui/tokens.py similarity index 100% rename from pype/vendor/capture_gui/tokens.py rename to pype/vendor/python/common/capture_gui/tokens.py diff --git a/pype/vendor/capture_gui/vendor/Qt.py b/pype/vendor/python/common/capture_gui/vendor/Qt.py similarity index 100% rename from pype/vendor/capture_gui/vendor/Qt.py rename to pype/vendor/python/common/capture_gui/vendor/Qt.py diff --git a/pype/vendor/capture_gui/vendor/__init__.py b/pype/vendor/python/common/capture_gui/vendor/__init__.py similarity index 100% rename from pype/vendor/capture_gui/vendor/__init__.py rename to pype/vendor/python/common/capture_gui/vendor/__init__.py diff --git a/pype/vendor/capture_gui/version.py b/pype/vendor/python/common/capture_gui/version.py similarity index 100% rename from pype/vendor/capture_gui/version.py rename to pype/vendor/python/common/capture_gui/version.py diff --git a/pype/vendor/pysync.py b/pype/vendor/python/common/pysync.py similarity index 100% rename from pype/vendor/pysync.py rename to pype/vendor/python/common/pysync.py diff --git a/pype/vendor/python/python_2/README.md b/pype/vendor/python/python_2/README.md new file mode 100644 index 0000000000..f101ddbf54 --- /dev/null +++ b/pype/vendor/python/python_2/README.md @@ -0,0 +1,3 @@ +## Info + +Only **Python 2** specific modules are here. \ No newline at end of file diff --git a/pype/vendor/python/python_3/README.md b/pype/vendor/python/python_3/README.md new file mode 100644 index 0000000000..1b8ab4ce69 --- /dev/null +++ b/pype/vendor/python/python_3/README.md @@ -0,0 +1,2 @@ +## Info +Only **Python 3** modules are here. \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index a07cef35cd..b3dee18b26 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -aiohttp +git+https://github.com/pypeclub/acre.git aiohttp_json_rpc appdirs arrow @@ -18,6 +18,7 @@ log4mongo git+https://github.com/pypeclub/OpenTimelineIO.git@develop pathlib2 Pillow +pyblish-base pycodestyle pydocstyle pylint diff --git a/setup.py b/setup.py index 5e32a1afe1..3a782a8d72 100644 --- a/setup.py +++ b/setup.py @@ -1,15 +1,27 @@ # -*- coding: utf-8 -*- """Setup info for building Pype 3.0.""" -import sys import os +import sys +from pathlib import Path + from cx_Freeze import setup, Executable from sphinx.setup_command import BuildDoc version = {} -with open(os.path.join("pype", "version.py")) as fp: - exec(fp.read(), version) -__version__ = version['__version__'] +pype_root = Path(os.path.dirname(__file__)) + +with open(pype_root / "pype" / "version.py") as fp: + exec(fp.read(), version) +__version__ = version["__version__"] + +base = None +if sys.platform == "win32": + base = "Win32GUI" + +# ----------------------------------------------------------------------- +# build_exe +# Build options for cx_Freeze. Manually add/exclude packages and binaries install_requires = [ "appdirs", @@ -17,44 +29,54 @@ install_requires = [ "keyring", "clique", "jsonschema", - "OpenTimelineIO", + "opentimelineio", "pathlib2", + "pkg_resources", "PIL", "pymongo", + "pynput", + "jinxed", + "blessed", "Qt", "speedcopy", - "win32ctypes" + "googleapiclient", + "httplib2" ] -base = None -if sys.platform == "win32": - base = "Win32GUI" +includes = [] +excludes = [] +bin_includes = [] +include_files = [ + "igniter", + "pype", + "repos", + "schema", + "vendor", + "LICENSE", + "README.md", + "pype/version.py" +] -# Build options for cx_Freeze. Manually add/exclude packages and binaries -buildOptions = dict( +if sys.platform == "win32": + install_requires.append("win32ctypes") + +build_options = dict( packages=install_requires, - includes=[ - 'repos/acre/acre', - 'repos/avalon-core/avalon', - 'repos/pyblish-base/pyblish', - 'repos/maya-look-assigner/mayalookassigner' - ], - excludes=[], - bin_includes=[], - include_files=[ - "igniter", - "pype", - "repos", - "schema", - "setup", - "vendor", - "LICENSE", - "README.md", - "pype/version.py"] + includes=includes, + excludes=excludes, + bin_includes=bin_includes, + include_files=include_files, + optimize=0 ) +icon_path = pype_root / "igniter" / "pype.ico" -executables = [Executable("pype.py", base=None, targetName="pype")] +executables = [ + Executable("start.py", base=None, + target_name="pype_console", icon=icon_path.as_posix()), + Executable("start.py", base=base, + target_name="pype", icon=icon_path.as_posix()) +] setup( name="pype", @@ -62,13 +84,13 @@ setup( description="Ultimate pipeline", cmdclass={"build_sphinx": BuildDoc}, options={ - "build_exe": buildOptions, + "build_exe": build_options, "build_sphinx": { "project": "Pype", "version": __version__, "release": __version__, - "source_dir": "./docs/source", - "build_dir": "./docs/build" + "source_dir": (pype_root / "docs" / "source").as_posix(), + "build_dir": (pype_root / "docs" / "build").as_posix() } }, executables=executables diff --git a/start.py b/start.py new file mode 100644 index 0000000000..ad863481ff --- /dev/null +++ b/start.py @@ -0,0 +1,606 @@ +# -*- coding: utf-8 -*- +"""Main entry point for Pype command. + +Bootstrapping process of Pype is as follows: + +`PYPE_PATH` is checked for existence - either one from environment or +from user settings. Precedence takes the one set by environment. + +On this path we try to find pype in directories version string in their names. +For example: `pype-v3.0.1-foo` is valid name, or even `foo_3.0.2` - as long +as version can be determined from its name _AND_ file `pype/pype/version.py` +can be found inside, it is considered Pype installation. + +If no Pype repositories are found in `PYPE_PATH` (user data dir) +then **Igniter** (Pype setup tool) will launch its GUI. + +It can be used to specify `PYPE_PATH` or if it is _not_ specified, current +*"live"* repositories will be used to create zip file and copy it to +appdata dir in user home and extract it there. Version will be determined by +version specified in Pype module. + +If Pype repository directories are found in default install location +(user data dir) or in `PYPE_PATH`, it will get list of those dirs there and +use latest one or the one specified with optional `--use-version` command +line argument. If the one specified doesn't exist then latest available +version will be used. All repositories in that dir will be added +to `sys.path` and `PYTHONPATH`. + +If Pype is live (not frozen) then current version of Pype module will be +used. All directories under `repos` will be added to `sys.path` and +`PYTHONPATH`. + +Pype depends on connection to `MongoDB`_. You can specify MongoDB connection +string via `PYPE_MONGO` set in environment or it can be set in user +settings or via **Igniter** GUI. + +So, bootstrapping Pype looks like this:: + +.. code-block:: bash + ++-------------------------------------------------------+ +| Determine MongoDB connection: | +| Use `PYPE_MONGO`, system keyring `pypeMongo` | ++--------------------------|----------------------------+ + .--- Found? --. + YES NO + | | + | +------v--------------+ + | | Fire up Igniter GUI |<---------+ + | | and ask User | | + | +---------------------+ | + | | + | | ++-----------------v------------------------------------+ | +| Get location of Pype: | | +| 1) Test for `PYPE_PATH` environment variable | | +| 2) Test `pypePath` in registry setting | | +| 3) Test user data directory | | +| ................................................... | | +| If running from frozen code: | | +| - Use latest one found in user data dir | | +| If running from live code: | | +| - Use live code and install it to user data dir | | +| * can be overridden with `--use-version` argument | | ++-------------------------|----------------------------+ | + .-- Is Pype found? --. | + YES NO | + | | | + | +--------------v------------------+ | + | | Look in `PYPE_PATH`, find | | + | | latest version and install it | | + | | to user data dir. | | + | +--------------|------------------+ | + | .-- Is Pype found? --. | + | YES NO ---------+ + | | + |<---------+ + | ++-------------v------------+ +| Run Pype | ++--------------------------+ + + +Todo: + Move or remove bootstrapping environments out of the code. + +Attributes: + silent_commands (list): list of commands for which we won't print Pype + logo and info header. + +.. _MongoDB: + https://www.mongodb.com/ + +""" +import os +import re +import sys +import traceback +import subprocess +import site +from pathlib import Path + +# add dependencies folder to sys.pat for frozen code +if getattr(sys, 'frozen', False): + frozen_libs = os.path.normpath( + os.path.join(os.path.dirname(sys.executable), "dependencies")) + sys.path.append(frozen_libs) + # add stuff from `/dependencies` to PYTHONPATH. + pythonpath = os.getenv("PYTHONPATH", "") + paths = pythonpath.split(os.pathsep) + paths.append(frozen_libs) + os.environ["PYTHONPATH"] = os.pathsep.join(paths) + +from igniter import BootstrapRepos # noqa: E402 +from igniter.tools import load_environments # noqa: E402 +from igniter.bootstrap_repos import PypeVersion # noqa: E402 + +bootstrap = BootstrapRepos() +silent_commands = ["run", "igniter"] + + +def set_environments() -> None: + """Set loaded environments. + + .. todo: + better handling of environments + + """ + try: + import acre + except ImportError: + if getattr(sys, 'frozen', False): + sys.path.append(os.path.join( + os.path.dirname(sys.executable), + "dependencies" + )) + import acre + try: + env = load_environments(["global"]) + except OSError as e: + print(f"!!! {e}") + sys.exit(1) + + env = acre.merge(env, dict(os.environ)) + os.environ.clear() + os.environ.update(env) + + +def run(arguments: list, env: dict = None) -> int: + """Use correct executable to run stuff. + + This passing arguments to correct Pype executable. If Pype is run from + live sources, executable will be `python` in virtual environment. + If running from frozen code, executable will be `pype`. Its equivalent in + live code is `python start.py`. + + Args: + arguments (list): Argument list to pass Pype. + env (dict, optional): Dictionary containing environment. + + Returns: + int: Process return code. + + """ + if getattr(sys, 'frozen', False): + interpreter = [sys.executable] + else: + interpreter = [sys.executable, __file__] + + interpreter.extend(arguments) + + p = subprocess.Popen(interpreter, env=env) + p.wait() + print(f">>> done [{p.returncode}]") + return p.returncode + + +def set_modules_environments(): + """Set global environments for pype modules. + + This requires to have pype in `sys.path`. + """ + + from pype.modules import ModulesManager + import acre + + modules_manager = ModulesManager() + + module_envs = modules_manager.collect_global_environments() + publish_plugin_dirs = modules_manager.collect_plugin_paths()["publish"] + + # Set pyblish plugins paths if any module want to register them + if publish_plugin_dirs: + publish_paths_str = os.environ.get("PYBLISHPLUGINPATH") or "" + publish_paths = publish_paths_str.split(os.pathsep) + _publish_paths = { + os.path.normpath(path) for path in publish_paths if path + } + for path in publish_plugin_dirs: + _publish_paths.add(os.path.normpath(path)) + module_envs["PYBLISHPLUGINPATH"] = os.pathsep.join(_publish_paths) + + # Merge environments with current environments and update values + if module_envs: + parsed_envs = acre.parse(module_envs) + env = acre.merge(parsed_envs, dict(os.environ)) + os.environ.clear() + os.environ.update(env) + + +def _process_arguments() -> tuple: + """Process command line arguments. + + Returns: + tuple: Return tuple with specific version to use (if any) and flag + to prioritize staging (if set) + """ + # check for `--use-version=3.0.0` argument and `--use-staging` + use_version = None + use_staging = False + for arg in sys.argv: + if arg == "--use-version": + print("!!! Please use option --use-version like:") + print(" --use-version=3.0.0") + sys.exit(1) + + m = re.search(r"--use-version=(?P\d+\.\d+\.\d*.+?)", arg) + if m and m.group('version'): + use_version = m.group('version') + sys.argv.remove(arg) + break + if "--use-staging" in sys.argv: + use_staging = True + sys.argv.remove("--use-staging") + + # handle igniter + # this is helper to run igniter before anything else + if "igniter" in sys.argv: + import igniter + igniter.run() + + return use_version, use_staging + + +def _determine_mongodb() -> str: + """Determine mongodb connection string. + + First use ``PYPE_MONGO`` environment variable, then system keyring. + Then try to run **Igniter UI** to let user specify it. + + Returns: + str: mongodb connection URL + + Raises: + RuntimeError: if mongodb connection url cannot by determined. + + """ + + pype_mongo = os.getenv("PYPE_MONGO", None) + if not pype_mongo: + # try system keyring + try: + pype_mongo = bootstrap.registry.get_secure_item("pypeMongo") + except ValueError: + print("*** No DB connection string specified.") + print("--- launching setup UI ...") + run(["igniter"]) + try: + pype_mongo = bootstrap.registry.get_secure_item("pypeMongo") + except ValueError: + raise RuntimeError("missing mongodb url") + + return pype_mongo + + +def _initialize_environment(pype_version: PypeVersion) -> None: + version_path = pype_version.path + os.environ["PYPE_VERSION"] = pype_version.version + # inject version to Python environment (sys.path, ...) + print(">>> Injecting Pype version to running environment ...") + bootstrap.add_paths_from_directory(version_path) + + # add venv 'site-packages' to PYTHONPATH + python_path = os.getenv("PYTHONPATH", "") + split_paths = python_path.split(os.pathsep) + # add pype tools + split_paths.append(os.path.join(os.environ["PYPE_ROOT"], "pype", "tools")) + # add common pype vendor + # (common for multiple Python interpreter versions) + split_paths.append(os.path.join( + os.environ["PYPE_ROOT"], "pype", "vendor", "python", "common")) + os.environ["PYTHONPATH"] = os.pathsep.join(split_paths) + + # set PYPE_ROOT to point to currently used Pype version. + os.environ["PYPE_ROOT"] = os.path.normpath(version_path.as_posix()) + + +def _find_frozen_pype(use_version: str = None, + use_staging: bool = False) -> Path: + """Find Pype to run from frozen code. + + This will process and modify environment variables: + ``PYTHONPATH``, ``PYPE_VERSION``, ``PYPE_ROOT`` + + Args: + use_version (str, optional): Try to use specified version. + use_staging (bool, optional): Prefer *staging* flavor over production. + + Returns: + Path: Path to version to be used. + + Raises: + RuntimeError: If no Pype version are found or no staging version + (if requested). + + """ + pype_version = None + pype_versions = bootstrap.find_pype(include_zips=True, + staging=use_staging) + try: + # use latest one found (last in the list is latest) + pype_version = pype_versions[-1] + except IndexError: + # no pype version found, run Igniter and ask for them. + print('*** No Pype versions found.') + print("--- launching setup UI ...") + run(["igniter"]) + pype_versions = bootstrap.find_pype() + + if not pype_versions: + # no Pype versions found anyway, lets use then the one + # shipped with frozen Pype + version_path = _bootstrap_from_code(use_version) + pype_version = PypeVersion( + version=BootstrapRepos.get_version(version_path), + path=version_path) + _initialize_environment(pype_version) + return version_path + + # get path of version specified in `--use-version` + version_path = BootstrapRepos.get_version_path_from_list( + use_version, pype_versions) + + if not version_path: + if use_version is not None: + if not pype_version: + ... + else: + print(("!!! Specified version was not found, using " + "latest available")) + # specified version was not found so use latest detected. + version_path = pype_version.path + print(f">>> Using version [ {pype_version} ]") + print(f" From {version_path}") + + # test if latest detected is installed (in user data dir) + is_inside = False + try: + is_inside = pype_version.path.resolve().relative_to( + bootstrap.data_dir) + except ValueError: + # if relative path cannot be calculated, Pype version is not + # inside user data dir + pass + + if not is_inside: + # install latest version to user data dir + version_path = bootstrap.install_version( + pype_version, force=True) + + if pype_version.path.is_file(): + print(">>> Extracting zip file ...") + version_path = bootstrap.extract_pype(pype_version) + pype_version.path = version_path + + _initialize_environment(pype_version) + return version_path + + +def _bootstrap_from_code(use_version): + """Bootstrap live code (or the one coming with frozen Pype. + + Args: + use_version: (str): specific version to use. + + Returns: + Path: path to sourced version. + + """ + # run through repos and add them to `sys.path` and `PYTHONPATH` + # set root + if getattr(sys, 'frozen', False): + pype_root = os.path.normpath( + os.path.dirname(sys.executable)) + local_version = bootstrap.get_version(Path(pype_root)) + else: + pype_root = os.path.normpath( + os.path.dirname(os.path.realpath(__file__))) + # get current version of Pype + local_version = bootstrap.get_local_live_version() + + os.environ["PYPE_VERSION"] = local_version + if use_version and use_version != local_version: + pype_versions = bootstrap.find_pype(include_zips=True) + version_path = BootstrapRepos.get_version_path_from_list( + use_version, pype_versions) + if version_path: + # use specified + bootstrap.add_paths_from_directory(version_path) + os.environ["PYPE_VERSION"] = use_version + else: + version_path = pype_root + os.environ["PYPE_ROOT"] = pype_root + repos = os.listdir(os.path.join(pype_root, "repos")) + repos = [os.path.join(pype_root, "repos", repo) for repo in repos] + # add self to python paths + repos.insert(0, pype_root) + for repo in repos: + sys.path.append(repo) + + # add venv 'site-packages' to PYTHONPATH + python_path = os.getenv("PYTHONPATH", "") + split_paths = python_path.split(os.pathsep) + split_paths += repos + # add pype tools + split_paths.append(os.path.join(os.environ["PYPE_ROOT"], "pype", "tools")) + # last one should be venv site-packages + # this is slightly convoluted as we can get here from frozen code too + # in case when we are running without any version installed. + if not getattr(sys, 'frozen', False): + split_paths.append(site.getsitepackages()[-1]) + # add common pype vendor + # (common for multiple Python interpreter versions) + split_paths.append(os.path.join( + os.environ["PYPE_ROOT"], "pype", "vendor", "python", "common")) + os.environ["PYTHONPATH"] = os.pathsep.join(split_paths) + + return Path(version_path) + + +def boot(): + """Bootstrap Pype.""" + version_path = None + + # ------------------------------------------------------------------------ + # Play animation + # ------------------------------------------------------------------------ + + from igniter.terminal_splash import play_animation + + # don't play for silenced commands + if all(item not in sys.argv for item in silent_commands): + play_animation() + + # ------------------------------------------------------------------------ + # Process arguments + # ------------------------------------------------------------------------ + + use_version, use_staging = _process_arguments() + + # ------------------------------------------------------------------------ + # Determine mongodb connection + # ------------------------------------------------------------------------ + + try: + pype_mongo = _determine_mongodb() + except RuntimeError as e: + # without mongodb url we are done for. + print(f"!!! {e}") + sys.exit(1) + + os.environ["PYPE_MONGO"] = pype_mongo + + # ------------------------------------------------------------------------ + # Load environments from database + # ------------------------------------------------------------------------ + # set PYPE_ROOT to running location until proper version can be + # determined. + if getattr(sys, 'frozen', False): + os.environ["PYPE_ROOT"] = os.path.dirname(sys.executable) + else: + os.environ["PYPE_ROOT"] = os.path.dirname(__file__) + set_environments() + + # ------------------------------------------------------------------------ + # Find Pype versions + # ------------------------------------------------------------------------ + + if getattr(sys, 'frozen', False): + # find versions of Pype to be used with frozen code + try: + version_path = _find_frozen_pype(use_version, use_staging) + except RuntimeError as e: + # no version to run + print(f"!!! {e}") + sys.exit(1) + else: + version_path = _bootstrap_from_code(use_version) + + # set this to point either to `python` from venv in case of live code + # or to `pype` or `pype_console` in case of frozen code + os.environ["PYPE_EXECUTABLE"] = sys.executable + + if getattr(sys, 'frozen', False): + os.environ["PYPE_REPOS_ROOT"] = os.environ["PYPE_ROOT"] + else: + os.environ["PYPE_REPOS_ROOT"] = os.path.join( + os.environ["PYPE_ROOT"], "repos") + + # delete Pype module from cache so it is used from specific version + try: + del sys.modules["pype"] + del sys.modules["pype.version"] + except AttributeError: + pass + except KeyError: + pass + + from pype import cli + from pype.lib import terminal as t + from pype.version import __version__ + print(">>> loading environments ...") + set_modules_environments() + + assert version_path, "Version path not defined." + info = get_info() + info.insert(0, f">>> Using Pype from [ {version_path} ]") + + t_width = 20 + try: + t_width = os.get_terminal_size().columns - 2 + except ValueError: + # running without terminal + pass + except OSError: + # running without terminal + pass + + _header = f"*** Pype [{__version__}] " + + info.insert(0, _header + "-" * (t_width - len(_header))) + for i in info: + # don't show for running scripts + if all(item not in sys.argv for item in silent_commands): + t.echo(i) + + try: + cli.main(obj={}, prog_name="pype") + except Exception: + exc_info = sys.exc_info() + print("!!! Pype crashed:") + traceback.print_exception(*exc_info) + sys.exit(1) + + +def get_info() -> list: + """Print additional information to console.""" + from pype.lib.mongo import get_default_components + from pype.lib.log import PypeLogger + + components = get_default_components() + + inf = [] + if not getattr(sys, 'frozen', False): + inf.append(("Pype variant", "staging")) + else: + inf.append(("Pype variant", "production")) + inf.append(("Running pype from", os.environ.get('PYPE_ROOT'))) + inf.append(("Using mongodb", components["host"])) + + if os.environ.get("FTRACK_SERVER"): + inf.append(("Using FTrack at", + os.environ.get("FTRACK_SERVER"))) + + if os.environ.get('DEADLINE_REST_URL'): + inf.append(("Using Deadline webservice at", + os.environ.get("DEADLINE_REST_URL"))) + + if os.environ.get('MUSTER_REST_URL'): + inf.append(("Using Muster at", + os.environ.get("MUSTER_REST_URL"))) + + # Reinitialize + PypeLogger.initialize() + + log_components = PypeLogger.log_mongo_url_components + if log_components["host"]: + inf.append(("Logging to MongoDB", log_components["host"])) + inf.append((" - port", log_components["port"] or "")) + inf.append((" - database", PypeLogger.log_database_name)) + inf.append((" - collection", PypeLogger.log_collection_name)) + inf.append((" - user", log_components["username"] or "")) + if log_components["auth_db"]: + inf.append((" - auth source", log_components["auth_db"])) + + maximum = max(len(i[0]) for i in inf) + formatted = [] + for info in inf: + padding = (maximum - len(info[0])) + 1 + formatted.append( + "... {}:{}[ {} ]".format(info[0], " " * padding, info[1])) + return formatted + + +if __name__ == "__main__": + boot() diff --git a/tests/igniter/test_bootstrap_repos.py b/tests/igniter/test_bootstrap_repos.py index 868b356771..59469b0687 100644 --- a/tests/igniter/test_bootstrap_repos.py +++ b/tests/igniter/test_bootstrap_repos.py @@ -2,9 +2,13 @@ """Test suite for repos bootstrapping (install).""" import os import sys +from collections import namedtuple from pathlib import Path -import pytest +from zipfile import ZipFile + import appdirs +import pytest + from igniter.bootstrap_repos import BootstrapRepos from igniter.bootstrap_repos import PypeVersion from pype.lib import PypeSettingsRegistry @@ -29,7 +33,8 @@ def test_pype_version(): assert str(v3) == "1.2.3-staging" v4 = PypeVersion(1, 2, 3, variant="staging", client="client") - assert str(v4) == "1.2.3-staging-client" + assert str(v4) == "1.2.3-client-staging" + assert v3 < v4 v5 = PypeVersion(1, 2, 3, variant="foo", client="x") assert str(v5) == "1.2.3-x" @@ -96,7 +101,7 @@ def test_pype_version(): with pytest.raises(ValueError): _ = PypeVersion(version="booobaa") - v11 = PypeVersion(version="4.6.7-staging-client") + v11 = PypeVersion(version="4.6.7-client-staging") assert v11.major == 4 assert v11.minor == 6 assert v11.subversion == 7 @@ -104,6 +109,11 @@ def test_pype_version(): assert v11.client == "client" +def test_get_main_version(): + ver = PypeVersion(1, 2, 3, variant="staging", client="foo") + assert ver.get_main_version() == "1.2.3" + + def test_get_version_path_from_list(): versions = [ PypeVersion(1, 2, 3, path=Path('/foo/bar')), @@ -116,17 +126,28 @@ def test_get_version_path_from_list(): assert path == Path("/bar/baz") +def test_search_string_for_pype_version(printer): + strings = [ + ("3.0.1", True), + ("foo-3.0", False), + ("foo-3.0.1", True), + ("3", False), + ("foo-3.0.1-client-staging", True), + ("foo-3.0.1-bar-baz", True) + ] + for ver_string in strings: + printer(f"testing {ver_string[0]} should be {ver_string[1]}") + assert PypeVersion.version_in_str(ver_string[0])[0] == ver_string[1] + + def test_install_live_repos(fix_bootstrap, printer): rf = fix_bootstrap.install_live_repos() sep = os.path.sep expected_paths = [ - f"{rf}{sep}acre", f"{rf}{sep}avalon-core", f"{rf}{sep}avalon-unreal-integration", f"{rf}{sep}maya-look-assigner", - f"{rf}{sep}pyblish-base", - f"{rf}{sep}pype", - f"{rf}{sep}pype-config" + f"{rf}{sep}pype" ] printer("testing zip creation") assert os.path.exists(rf), "zip archive was not created" @@ -147,92 +168,217 @@ def test_install_live_repos(fix_bootstrap, printer): def test_find_pype(fix_bootstrap, tmp_path_factory, monkeypatch, printer): + test_pype = namedtuple("Pype", "prefix version suffix type valid") + test_versions_1 = [ - "pype-repositories-v5.5.1.zip", - "pype-repositories-v5.5.2-client.zip", - "pype-repositories-v5.5.3-client-strange.zip", - "pype-repositories-v5.5.4-staging.zip", - "pype-repositories-v5.5.5-staging-client.zip", - "pype-repositories-v5.6.3.zip", - "pype-repositories-v5.6.3-staging.zip" + test_pype(prefix="foo-v", version="5.5.1", + suffix=".zip", type="zip", valid=False), + test_pype(prefix="bar-v", version="5.5.2-client", + suffix=".zip", type="zip", valid=True), + test_pype(prefix="baz-v", version="5.5.3-client-strange", + suffix=".zip", type="zip", valid=True), + test_pype(prefix="bum-v", version="5.5.4-staging", + suffix=".zip", type="zip", valid=True), + test_pype(prefix="zum-v", version="5.5.5-client-staging", + suffix=".zip", type="zip", valid=True), + test_pype(prefix="fam-v", version="5.6.3", + suffix=".zip", type="zip", valid=True), + test_pype(prefix="foo-v", version="5.6.3-staging", + suffix=".zip", type="zip", valid=True), + test_pype(prefix="fim-v", version="5.6.3", + suffix=".zip", type="zip", valid=False), + test_pype(prefix="foo-v", version="5.6.4", + suffix=".txt", type="txt", valid=False), + test_pype(prefix="foo-v", version="5.7.1", + suffix="", type="dir", valid=False), ] test_versions_2 = [ - "pype-repositories-v7.2.6.zip", - "pype-repositories-v7.2.7-client.zip", - "pype-repositories-v7.2.8-client-strange.zip", - "pype-repositories-v7.2.9-staging.zip", - "pype-repositories-v7.2.10-staging-client.zip", - "pype-repositories-v7.0.1.zip", + test_pype(prefix="foo-v", version="10.0.0", + suffix=".txt", type="txt", valid=False), + test_pype(prefix="lom-v", version="7.2.6", + suffix=".zip", type="zip", valid=True), + test_pype(prefix="bom-v", version="7.2.7-client", + suffix=".zip", type="zip", valid=True), + test_pype(prefix="woo-v", version="7.2.8-client-strange", + suffix=".zip", type="zip", valid=True), + test_pype(prefix="loo-v", version="7.2.10-client-staging", + suffix=".zip", type="zip", valid=True), + test_pype(prefix="kok-v", version="7.0.1", + suffix=".zip", type="zip", valid=True) ] test_versions_3 = [ - "pype-repositories-v3.0.0.zip", - "pype-repositories-v3.0.1.zip", - "pype-repositories-v4.1.0.zip", - "pype-repositories-v4.1.2.zip", - "pype-repositories-v3.0.1-client.zip", - "pype-repositories-v3.0.1-client-strange.zip", - "pype-repositories-v3.0.1-staging.zip", - "pype-repositories-v3.0.1-staging-client.zip", - "pype-repositories-v3.2.0.zip", + test_pype(prefix="foo-v", version="3.0.0", + suffix=".zip", type="zip", valid=True), + test_pype(prefix="goo-v", version="3.0.1", + suffix=".zip", type="zip", valid=True), + test_pype(prefix="hoo-v", version="4.1.0", + suffix=".zip", type="zip", valid=True), + test_pype(prefix="foo-v", version="4.1.2", + suffix=".zip", type="zip", valid=True), + test_pype(prefix="foo-v", version="3.0.1-client", + suffix=".zip", type="zip", valid=True), + test_pype(prefix="foo-v", version="3.0.1-client-strange", + suffix=".zip", type="zip", valid=True), + test_pype(prefix="foo-v", version="3.0.1-staging", + suffix=".zip", type="zip", valid=True), + test_pype(prefix="foo-v", version="3.0.1-client-staging", + suffix=".zip", type="zip", valid=True), + test_pype(prefix="foo-v", version="3.2.0", + suffix=".zip", type="zip", valid=True) ] + test_versions_4 = [ + test_pype(prefix="foo-v", version="10.0.0", + suffix="", type="dir", valid=True), + test_pype(prefix="lom-v", version="11.2.6", + suffix=".zip", type="dir", valid=False), + test_pype(prefix="bom-v", version="7.2.7-client", + suffix=".zip", type="zip", valid=True), + test_pype(prefix="woo-v", version="7.2.8-client-strange", + suffix=".zip", type="txt", valid=False) + ] + + def _create_invalid_zip(path: Path): + with ZipFile(path, "w") as zf: + zf.writestr("test.foo", "test") + + def _create_valid_zip(path: Path, version: str): + with ZipFile(path, "w") as zf: + zf.writestr( + "pype/pype/version.py", f"__version__ = '{version}'\n\n") + + def _create_invalid_dir(path: Path): + path.mkdir(parents=True, exist_ok=True) + with open(path / "invalid", "w") as fp: + fp.write("invalid") + + def _create_valid_dir(path: Path, version: str): + pype_path = path / "pype" / "pype" + version_path = pype_path / "version.py" + pype_path.mkdir(parents=True, exist_ok=True) + with open(version_path, "w") as fp: + fp.write(f"__version__ = '{version}'\n\n") + + def _build_test_item(path, item): + test_path = path / "{}{}{}".format(item.prefix, + item.version, + item.suffix) + if item.type == "zip": + if item.valid: + _create_valid_zip(test_path, item.version) + else: + _create_invalid_zip(test_path) + elif item.type == "dir": + if item.valid: + _create_valid_dir(test_path, item.version) + else: + _create_invalid_dir(test_path) + else: + with open(test_path, "w") as fp: + fp.write("foo") + # in PYPE_PATH e_path = tmp_path_factory.mktemp("environ") + + # create files and directories for test for test_file in test_versions_1: - with open(e_path / test_file, "w") as fp: - fp.write(test_file) + _build_test_item(e_path, test_file) # in pypePath registry - r_path = tmp_path_factory.mktemp("pypePath") + p_path = tmp_path_factory.mktemp("pypePath") for test_file in test_versions_2: - with open(r_path / test_file, "w") as fp: - fp.write(test_file) + _build_test_item(p_path, test_file) # in data dir - for test_file in test_versions_3: - with open(os.path.join(fix_bootstrap.data_dir, test_file), "w") as fp: - fp.write(test_file) + d_path = tmp_path_factory.mktemp("dataPath") + for test_file in test_versions_2: + _build_test_item(d_path, test_file) - result = fix_bootstrap.find_pype() + # in provided path + g_path = tmp_path_factory.mktemp("providedPath") + for test_file in test_versions_3: + _build_test_item(g_path, test_file) + + # dir vs zip preference + dir_path = tmp_path_factory.mktemp("dirZipPath") + for test_file in test_versions_4: + _build_test_item(dir_path, test_file) + + printer("testing finding Pype in given path ...") + result = fix_bootstrap.find_pype(g_path, include_zips=True) # we should have results as file were created assert result is not None, "no Pype version found" # latest item in `result` should be latest version found. - assert result[-1].path == Path( - fix_bootstrap.data_dir / test_versions_3[3] - ), "not a latest version of Pype 3" + expected_path = Path( + g_path / "{}{}{}".format( + test_versions_3[3].prefix, + test_versions_3[3].version, + test_versions_3[3].suffix + ) + ) + assert result, "nothing found" + assert result[-1].path == expected_path, "not a latest version of Pype 3" monkeypatch.setenv("PYPE_PATH", e_path.as_posix()) - result = fix_bootstrap.find_pype() + result = fix_bootstrap.find_pype(include_zips=True) # we should have results as file were created assert result is not None, "no Pype version found" # latest item in `result` should be latest version found. - assert result[-1].path == Path( - e_path / test_versions_1[5] - ), "not a latest version of Pype 1" + expected_path = Path( + e_path / "{}{}{}".format( + test_versions_1[5].prefix, + test_versions_1[5].version, + test_versions_1[5].suffix + ) + ) + assert result, "nothing found" + assert result[-1].path == expected_path, "not a latest version of Pype 1" monkeypatch.delenv("PYPE_PATH", raising=False) # mock appdirs user_data_dir def mock_user_data_dir(*args, **kwargs): - return r_path.as_posix() + return d_path.as_posix() monkeypatch.setattr(appdirs, "user_data_dir", mock_user_data_dir) fix_bootstrap.registry = PypeSettingsRegistry() - fix_bootstrap.registry.set_item("pypePath", r_path.as_posix()) + fix_bootstrap.registry.set_item("pypePath", d_path.as_posix()) - result = fix_bootstrap.find_pype() + result = fix_bootstrap.find_pype(include_zips=True) # we should have results as file were created assert result is not None, "no Pype version found" # latest item in `result` should be latest version found. - assert result[-1].path == Path( - r_path / test_versions_2[4] - ), "not a latest version of Pype 2" + expected_path = Path( + d_path / "{}{}{}".format( + test_versions_2[3].prefix, + test_versions_2[3].version, + test_versions_2[3].suffix + ) + ) + assert result, "nothing found" + assert result[-1].path == expected_path, "not a latest version of Pype 2" - result = fix_bootstrap.find_pype(e_path) + result = fix_bootstrap.find_pype(e_path, include_zips=True) assert result is not None, "no Pype version found" - assert result[-1].path == Path( - e_path / test_versions_1[5] - ), "not a latest version of Pype 1" + expected_path = Path( + e_path / "{}{}{}".format( + test_versions_1[5].prefix, + test_versions_1[5].version, + test_versions_1[5].suffix + ) + ) + assert result[-1].path == expected_path, "not a latest version of Pype 1" + + result = fix_bootstrap.find_pype(dir_path, include_zips=True) + assert result is not None, "no Pype versions found" + expected_path = Path( + dir_path / "{}{}{}".format( + test_versions_4[0].prefix, + test_versions_4[0].version, + test_versions_4[0].suffix + ) + ) + assert result[-1].path == expected_path, "not a latest version of Pype 4" diff --git a/tools/build.ps1 b/tools/build.ps1 new file mode 100644 index 0000000000..9e067acb34 --- /dev/null +++ b/tools/build.ps1 @@ -0,0 +1,201 @@ +<# +.SYNOPSIS + Helper script to build Pype. + +.DESCRIPTION + This script will detect Python installation, create venv and install + all necessary packages from `requirements.txt` needed by Pype to be + included during application freeze on Windows. + +.EXAMPLE + +PS> .\build.ps1 + +#> + +function Start-Progress { + param([ScriptBlock]$code) + $scroll = "/-\|/-\|" + $idx = 0 + $job = Invoke-Command -ComputerName $env:ComputerName -ScriptBlock { $code } -AsJob + + $origpos = $host.UI.RawUI.CursorPosition + + # $origpos.Y -= 1 + + while (($job.State -eq "Running") -and ($job.State -ne "NotStarted")) + { + $host.UI.RawUI.CursorPosition = $origpos + Write-Host $scroll[$idx] -NoNewline + $idx++ + if ($idx -ge $scroll.Length) + { + $idx = 0 + } + Start-Sleep -Milliseconds 100 + } + # It's over - clear the activity indicator. + $host.UI.RawUI.CursorPosition = $origpos + Write-Host ' ' + <# + .SYNOPSIS + Display spinner for running job + .PARAMETER code + Job to display spinner for + #> +} + + +function Exit-WithCode($exitcode) { + # Only exit this host process if it's a child of another PowerShell parent process... + $parentPID = (Get-CimInstance -ClassName Win32_Process -Filter "ProcessId=$PID" | Select-Object -Property ParentProcessId).ParentProcessId + $parentProcName = (Get-CimInstance -ClassName Win32_Process -Filter "ProcessId=$parentPID" | Select-Object -Property Name).Name + if ('powershell.exe' -eq $parentProcName) { $host.SetShouldExit($exitcode) } + + exit $exitcode +} + +function Show-PSWarning() { + if ($PSVersionTable.PSVersion.Major -lt 7) { + Write-Host "!!! " -NoNewline -ForegroundColor Red + Write-Host "You are using old version of PowerShell. $($PSVersionTable.PSVersion.Major).$($PSVersionTable.PSVersion.Minor)" + Write-Host "Please update to at least 7.0 - " -NoNewline -ForegroundColor Gray + Write-Host "https://github.com/PowerShell/PowerShell/releases" -ForegroundColor White + Exit-WithCode 1 + } +} + +$art = @" + + + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ______ ___ ___ ___ + \ \ \___/ /\ \ \ \\ \\ \ + \ \____\ \ \_____\ \__\\__\\__\ + \/____/ \/_____/ . PYPE Club . + +"@ + +Write-Host $art -ForegroundColor DarkGreen + +# Enable if PS 7.x is needed. +# Show-PSWarning + +$current_dir = Get-Location +$script_dir = Split-Path -Path $MyInvocation.MyCommand.Definition -Parent +$pype_root = (Get-Item $script_dir).parent.FullName + +Set-Location -Path $pype_root + +$version_file = Get-Content -Path "$($pype_root)\pype\version.py" +$result = [regex]::Matches($version_file, '__version__ = "(?\d+\.\d+.\d+)"') +$pype_version = $result[0].Groups['version'].Value +if (-not $pype_version) { + Write-Host "!!! " -ForegroundColor yellow -NoNewline + Write-Host "Cannot determine Pype version." + Exit-WithCode 1 +} + +# Create build directory if not exist +if (-not (Test-Path -PathType Container -Path "$($pype_root)\build")) { + New-Item -ItemType Directory -Force -Path "$($pype_root)\build" +} + +Write-Host "--- " -NoNewline -ForegroundColor yellow +Write-Host "Cleaning build directory ..." +try { + Remove-Item -Recurse -Force "$($pype_root)\build\*" +} +catch { + Write-Host "!!! " -NoNewline -ForegroundColor Red + Write-Host "Cannot clean build directory, possibly because process is using it." + Write-Host $_.Exception.Message + Exit-WithCode 1 +} + +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "Building Pype [ " -NoNewline -ForegroundColor white +Write-host $pype_version -NoNewline -ForegroundColor green +Write-Host " ] ..." -ForegroundColor white + +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "Detecting host Python ... " -NoNewline +if (-not (Get-Command "python" -ErrorAction SilentlyContinue)) { + Write-Host "!!! Python not detected" -ForegroundColor red + Exit-WithCode 1 +} +$version_command = @" +import sys +print('{0}.{1}'.format(sys.version_info[0], sys.version_info[1])) +"@ + +$p = & python -c $version_command +$env:PYTHON_VERSION = $p +$m = $p -match '(\d+)\.(\d+)' +if(-not $m) { + Write-Host "!!! Cannot determine version" -ForegroundColor red + Exit-WithCode 1 +} +# We are supporting python 3.6 and up +if(($matches[1] -lt 3) -or ($matches[2] -lt 7)) { + Write-Host "FAILED Version [ $p ] is old and unsupported" -ForegroundColor red + Exit-WithCode 1 +} +Write-Host "OK [ $p ]" -ForegroundColor green + +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "Entering venv ..." +try { + . ("$($pype_root)\venv\Scripts\Activate.ps1") +} +catch { + Write-Host "!!! Failed to activate" -ForegroundColor red + Write-Host ">>> " -NoNewline -ForegroundColor green + Write-Host "Trying to create env ..." + & "$($script_dir)\create_env.ps1" + try { + . ("$($pype_root)\venv\Scripts\Activate.ps1") + } + catch { + Write-Host "!!! Failed to activate" -ForegroundColor red + Write-Host $_.Exception.Message + Exit-WithCode 1 + } +} + +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "Cleaning cache files ... " -NoNewline +Get-ChildItem $pype_root -Filter "*.pyc" -Force -Recurse | Remove-Item -Force +Get-ChildItem $pype_root -Filter "*.pyo" -Force -Recurse | Remove-Item -Force +Get-ChildItem $pype_root -Filter "__pycache__" -Force -Recurse | Remove-Item -Force -Recurse +Write-Host "OK" -ForegroundColor green + +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "Building Pype ..." +$out = & python setup.py build 2>&1 +if ($LASTEXITCODE -ne 0) +{ + Write-Host "!!! " -NoNewLine -ForegroundColor Red + Write-Host "Build failed. Check the log: " -NoNewline + Write-Host ".\build\build.log" -ForegroundColor Yellow + deactivate + Exit-WithCode $LASTEXITCODE +} + +Set-Content -Path "$($pype_root)\build\build.log" -Value $out +& python -B "$($pype_root)\tools\build_dependencies.py" + +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "deactivating venv ..." +deactivate + +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "restoring current directory" +Set-Location -Path $current_dir + +Write-Host "*** " -NoNewline -ForegroundColor Cyan +Write-Host "All done. You will find Pype and build log in " -NoNewLine +Write-Host "'.\build'" -NoNewline -ForegroundColor Green +Write-Host " directory." diff --git a/build.sh b/tools/build.sh similarity index 100% rename from build.sh rename to tools/build.sh diff --git a/tools/build_dependencies.py b/tools/build_dependencies.py new file mode 100644 index 0000000000..d28b631eca --- /dev/null +++ b/tools/build_dependencies.py @@ -0,0 +1,122 @@ +# -*- coding: utf-8 -*- +"""Script to fix frozen dependencies. + +Because Pype code needs to run under different versions of Python interpreter +(yes, even Python 2) we need to include all dependencies as source code +without Python's system stuff. Cx-freeze puts everything into lib and compile +it as .pyc/.pyo files and that doesn't work for hosts like Maya 2020 with +their own Python interpreter and libraries. + +This script will take ``site-packages`` and copy them to built Pype under +``dependencies`` directory. It will then compare stuff inside with ``lib`` +folder in frozen Pype, removing duplicities from there. + +This must be executed after build finished and it is done by build PowerShell +script. + +Note: Speedcopy can be used for copying if server-side copy is important for +speed. + +""" +import os +import sys +import site +from distutils.util import get_platform +from pathlib import Path +import shutil +import blessed +import time + + +term = blessed.Terminal() + + +def _print(msg: str, type: int = 0) -> None: + """Print message to console. + + Args: + msg (str): message to print + type (int): type of message (0 info, 1 error, 2 note) + + """ + if type == 0: + header = term.aquamarine3(">>> ") + elif type == 1: + header = term.orangered2("!!! ") + elif type == 2: + header = term.tan1("... ") + else: + header = term.darkolivegreen3("--- ") + + print("{}{}".format(header, msg)) + + +_print("Starting dependency cleanup ...") +start_time = time.time_ns() + +# path to venv site packages +sites = site.getsitepackages() + +# WARNING: this assumes that all we've got is path to venv itself and +# another path ending with 'site-packages' as is default. But because +# this must run under different platform, we cannot easily check if this path +# is the one, because under Linux and macOS site-packages are in different +# location. +site_pkg = None +for s in sites: + site_pkg = Path(s) + if site_pkg.name == "site-packages": + break + +_print("Getting venv site-packages ...") +assert site_pkg, "No venv site-packages are found." +_print(f"Working with: {site_pkg}", 2) + +# now, copy it to build directory +build_dir = None +if sys.platform.startswith("linux"): + # TODO: what is it under linux? + raise NotImplementedError("not implemented for linux yet") +elif sys.platform == "darwin": + # TODO: what is it under macOS? + raise NotImplementedError("not implemented for macOS yet") +elif sys.platform == "win32": + # string is formatted as cx_freeze is doing it + build_dir = "exe.{}-{}".format(get_platform(), sys.version[0:3]) + +# create full path +build_dir = Path(os.path.dirname(__file__)).parent / "build" / build_dir + +_print(f"Using build at {build_dir}", 2) +if not build_dir.exists(): + _print("Build directory doesn't exist", 1) + _print("Probably freezing of code failed. Check ./build/build.log", 3) + sys.exit(1) + +deps_dir = build_dir / "dependencies" + +# copy all files +_print("Copying dependencies ...") +shutil.copytree(site_pkg.as_posix(), deps_dir.as_posix()) + +# iterate over frozen libs and create list to delete +libs_dir = build_dir / "lib" + +to_delete = [] +_print("Finding duplicates ...") +for d in libs_dir.iterdir(): + if (deps_dir / d.name) in deps_dir.iterdir(): + to_delete.append(d) + _print(f"found {d}", 3) + +# delete duplicates +_print(f"Deleting {len(to_delete)} duplicates ...") +for d in to_delete: + if d.is_dir(): + shutil.rmtree(d) + else: + d.unlink() + +end_time = time.time_ns() +total_time = (end_time - start_time) / 1000000000 +_print(f"Dependency cleanup done in {total_time} secs.") diff --git a/tools/create_env.ps1 b/tools/create_env.ps1 new file mode 100644 index 0000000000..38737a23a9 --- /dev/null +++ b/tools/create_env.ps1 @@ -0,0 +1,137 @@ +<# +.SYNOPSIS + Helper script create virtual env. + +.DESCRIPTION + This script will detect Python installation, create venv and install + all necessary packages from `requirements.txt` needed by Pype to be + included during application freeze on Windows. + +.EXAMPLE + +PS> .\create_env.ps1 + +#> + +function Exit-WithCode($exitcode) { + # Only exit this host process if it's a child of another PowerShell parent process... + $parentPID = (Get-CimInstance -ClassName Win32_Process -Filter "ProcessId=$PID" | Select-Object -Property ParentProcessId).ParentProcessId + $parentProcName = (Get-CimInstance -ClassName Win32_Process -Filter "ProcessId=$parentPID" | Select-Object -Property Name).Name + if ('powershell.exe' -eq $parentProcName) { $host.SetShouldExit($exitcode) } + + exit $exitcode +} + + +function Show-PSWarning() { + if ($PSVersionTable.PSVersion.Major -lt 7) { + Write-Host "!!! " -NoNewline -ForegroundColor Red + Write-Host "You are using old version of PowerShell. $($PSVersionTable.PSVersion.Major).$($PSVersionTable.PSVersion.Minor)" + Write-Host "Please update to at least 7.0 - " -NoNewline -ForegroundColor Gray + Write-Host "https://github.com/PowerShell/PowerShell/releases" -ForegroundColor White + Exit-WithCode 1 + } +} +$current_dir = Split-Path -Path $MyInvocation.MyCommand.Definition -Parent +$pype_root = (Get-Item $current_dir).parent.FullName + +$art = @" + + + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ______ ___ ___ ___ + \ \ \___/ /\ \ \ \\ \\ \ + \ \____\ \ \_____\ \__\\__\\__\ + \/____/ \/_____/ . PYPE Club . + +"@ + +Write-Host $art -ForegroundColor DarkGreen + +# Enable if PS 7.x is needed. +# Show-PSWarning + +$version_file = Get-Content -Path "$($pype_root)\pype\version.py" +$result = [regex]::Matches($version_file, '__version__ = "(?\d+\.\d+.\d+)"') +$pype_version = $result[0].Groups['version'].Value +if (-not $pype_version) { + Write-Host "!!! " -ForegroundColor yellow -NoNewline + Write-Host "Cannot determine Pype version." + Exit-WithCode 1 +} + +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "Detecting host Python ... " -NoNewline +if (-not (Get-Command "python" -ErrorAction SilentlyContinue)) { + Write-Host "!!! Python not detected" -ForegroundColor red + Exit-WithCode 1 +} +$version_command = @' +import sys +print('{0}.{1}'.format(sys.version_info[0], sys.version_info[1])) +'@ + +$p = & python -c $version_command +$env:PYTHON_VERSION = $p +$m = $p -match '(\d+)\.(\d+)' +if(-not $m) { + Write-Host "!!! Cannot determine version" -ForegroundColor red + Exit-WithCode 1 +} +# We are supporting python 3.6 and up +if(($matches[1] -lt 3) -or ($matches[2] -lt 7)) { + Write-Host "FAILED Version [ $p ] is old and unsupported" -ForegroundColor red + Exit-WithCode 1 +} +Write-Host "OK [ $p ]" -ForegroundColor green + +# Create venv directory if not exist +if (-not (Test-Path -PathType Container -Path "$($pype_root)\venv")) { + New-Item -ItemType Directory -Force -Path "$($pype_root)\venv" +} + +Write-Host "--- " -NoNewline -ForegroundColor yellow +Write-Host "Cleaning venv directory ..." + +try { + Remove-Item -Recurse -Force "$($pype_root)\venv\*" +} +catch { + Write-Host "!!! " -NoNewline -ForegroundColor red + Write-Host "Cannot clean venv directory, possibly another process is using it." + Write-Host $_.Exception.Message + Exit-WithCode 1 +} + +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "Creating virtual env ..." +& python -m venv venv +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "Entering venv ..." +try { + . ("$($pype_root)\venv\Scripts\Activate.ps1") +} +catch { + Write-Host "!!! Failed to activate" -ForegroundColor red + Write-Host $_.Exception.Message + Exit-WithCode 1 +} +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "Updating pip ..." +& python -m pip install --upgrade pip + +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "Installing packages to new venv ..." +& pip install -r "$($pype_root)\requirements.txt" + +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "Cleaning cache files ... " -NoNewline +Get-ChildItem "$($pype_root)" -Filter "*.pyc" -Force -Recurse | Remove-Item -Force +Get-ChildItem "$($pype_root)" -Filter "__pycache__" -Force -Recurse | Remove-Item -Force -Recurse +Write-Host "OK" -ForegroundColor green + +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "Deactivating venv ..." +deactivate diff --git a/build.ps1 b/tools/create_zip.ps1 similarity index 64% rename from build.ps1 rename to tools/create_zip.ps1 index 39ce90e36a..35bddfeb4e 100644 --- a/build.ps1 +++ b/tools/create_zip.ps1 @@ -1,6 +1,6 @@ <# .SYNOPSIS - Helper script to build Pype. + Helper script create distributable Pype zip. .DESCRIPTION This script will detect Python installation, create venv and install @@ -9,11 +9,10 @@ .EXAMPLE -PS> .\build.ps1 +PS> .\create_zip.ps1 #> - function Exit-WithCode($exitcode) { # Only exit this host process if it's a child of another PowerShell parent process... $parentPID = (Get-CimInstance -ClassName Win32_Process -Filter "ProcessId=$PID" | Select-Object -Property ParentProcessId).ParentProcessId @@ -23,7 +22,20 @@ function Exit-WithCode($exitcode) { exit $exitcode } -$art = @' + +function Show-PSWarning() { + if ($PSVersionTable.PSVersion.Major -lt 7) { + Write-Host "!!! " -NoNewline -ForegroundColor Red + Write-Host "You are using old version of PowerShell. $($PSVersionTable.PSVersion.Major).$($PSVersionTable.PSVersion.Minor)" + Write-Host "Please update to at least 7.0 - " -NoNewline -ForegroundColor Gray + Write-Host "https://github.com/PowerShell/PowerShell/releases" -ForegroundColor White + Exit-WithCode 1 + } +} +$current_dir = Split-Path -Path $MyInvocation.MyCommand.Definition -Parent +$pype_root = (Get-Item $current_dir).parent.FullName + +$art = @" ____________ @@ -34,11 +46,14 @@ $art = @' \ \____\ \ \_____\ \__\\__\\__\ \/____/ \/_____/ . PYPE Club . -'@ +"@ Write-Host $art -ForegroundColor DarkGreen -$version_file = Get-Content -Path ".\pype\version.py" +# Enable if PS 7.x is needed. +# Show-PSWarning + +$version_file = Get-Content -Path "$($pype_root)\pype\version.py" $result = [regex]::Matches($version_file, '__version__ = "(?\d+\.\d+.\d+)"') $pype_version = $result[0].Groups['version'].Value if (-not $pype_version) { @@ -47,11 +62,6 @@ if (-not $pype_version) { Exit-WithCode 1 } -Write-Host ">>> " -NoNewline -ForegroundColor green -Write-Host "Building Pype [ " -NoNewline -ForegroundColor white -Write-host $pype_version -NoNewline -ForegroundColor green -Write-Host " ]..." -ForegroundColor white - Write-Host ">>> " -NoNewline -ForegroundColor green Write-Host "Detecting host Python ... " -NoNewline if (-not (Get-Command "python" -ErrorAction SilentlyContinue)) { @@ -76,30 +86,24 @@ if(($matches[1] -lt 3) -or ($matches[2] -lt 7)) { Exit-WithCode 1 } Write-Host "OK [ $p ]" -ForegroundColor green -Write-Host ">>> " -NoNewline -ForegroundColor green -Write-Host "Creating virtual env ..." -& python -m venv venv + Write-Host ">>> " -NoNewline -ForegroundColor green Write-Host "Entering venv ..." try { - . (".\venv\Scripts\Activate.ps1") + . ("$($pype_root)\venv\Scripts\Activate.ps1") } catch { - Write-Host "!!! Failed to activate" -ForegroundColor red - Write-Host $_.Exception.Message - Exit-WithCode 1 + Write-Host "!!! Failed to activate" -ForegroundColor red + Write-Host $_.Exception.Message + Exit-WithCode 1 } Write-Host ">>> " -NoNewline -ForegroundColor green -Write-Host "Installing packages to new venv ..." -& pip install -r .\requirements.txt +Write-Host "Generating zip from current sources ..." +Write-Host "... " -NoNewline -ForegroundColor Magenta +Write-Host "arguments: " -NoNewline -ForegroundColor Gray +Write-Host $ARGS -ForegroundColor White +& python "$($pype_root)\start.py" generate-zip $ARGS Write-Host ">>> " -NoNewline -ForegroundColor green -Write-Host "Cleaning cache files ... " -NoNewline -Get-ChildItem . -Filter "*.pyc" -Force -Recurse | Remove-Item -Force -Get-ChildItem . -Filter "__pycache__" -Force -Recurse | Remove-Item -Force -Recurse -Write-Host "OK" -ForegroundColor green - -Write-Host ">>> " -NoNewline -ForegroundColor green -Write-Host "Building Pype ..." -& python setup.py build +Write-Host "Deactivating venv ..." deactivate diff --git a/tools/make_docs.ps1 b/tools/make_docs.ps1 index 475448d05e..951cab4c8e 100644 --- a/tools/make_docs.ps1 +++ b/tools/make_docs.ps1 @@ -1,5 +1,48 @@ -& .\venv\Scripts\Activate.ps1 -sphinx-apidoc.exe -M -e -d 10 -o .\docs\source igniter -sphinx-apidoc.exe -M -e -d 10 -o .\docs\source pype vendor, pype\vendor +<# +.SYNOPSIS + Helper script to update Pype Sphinx sources. + +.DESCRIPTION + This script will run apidoc over Pype sources and generate new source rst + files for documentation. Then it will run build_sphinx to create test html + documentation build. + +.EXAMPLE + +PS> .\make_docs.ps1 + +#> + +$current_dir = Get-Location +$script_dir = Split-Path -Path $MyInvocation.MyCommand.Definition -Parent +$pype_root = (Get-Item $script_dir).parent.FullName + +$art = @" + + + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ______ ___ ___ ___ + \ \ \___/ /\ \ \ \\ \\ \ + \ \____\ \ \_____\ \__\\__\\__\ + \/____/ \/_____/ . PYPE Club . + +"@ + +Write-Host $art -ForegroundColor DarkGreen + + +& "$($pype_root)\venv\Scripts\Activate.ps1" +Write-Host "This will not overwrite existing source rst files, only scan and add new." +Set-Location -Path $pype_root +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "Running apidoc ..." +sphinx-apidoc.exe -M -e -d 10 --ext-intersphinx --ext-todo --ext-coverage --ext-viewcode -o "$($pype_root)\docs\source" igniter +sphinx-apidoc.exe -M -e -d 10 --ext-intersphinx --ext-todo --ext-coverage --ext-viewcode -o "$($pype_root)\docs\source" pype vendor, pype\vendor + +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "Building html ..." python setup.py build_sphinx -deactivate \ No newline at end of file +deactivate +Set-Location -Path $current_dir diff --git a/tools/run_mongo.ps1 b/tools/run_mongo.ps1 new file mode 100644 index 0000000000..1b6d95ca57 --- /dev/null +++ b/tools/run_mongo.ps1 @@ -0,0 +1,91 @@ +<# +.SYNOPSIS + Helper script to run mongodb. + +.DESCRIPTION + This script will detect mongodb, add it to the PATH and launch it on specified port and db location. + +.EXAMPLE + +PS> .\run_mongo.ps1 + +#> + +$art = @" + + + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ______ ___ ___ ___ + \ \ \___/ /\ \ \ \\ \\ \ + \ \____\ \ \_____\ \__\\__\\__\ + \/____/ \/_____/ . PYPE Club . + +"@ + +Write-Host $art -ForegroundColor DarkGreen + +function Exit-WithCode($exitcode) { + # Only exit this host process if it's a child of another PowerShell parent process... + $parentPID = (Get-CimInstance -ClassName Win32_Process -Filter "ProcessId=$PID" | Select-Object -Property ParentProcessId).ParentProcessId + $parentProcName = (Get-CimInstance -ClassName Win32_Process -Filter "ProcessId=$parentPID" | Select-Object -Property Name).Name + if ('powershell.exe' -eq $parentProcName) { $host.SetShouldExit($exitcode) } + + exit $exitcode +} + + +function Find-Mongo { + Write-Host ">>> " -NoNewLine -ForegroundColor Green + Write-Host "Detecting MongoDB ... " -NoNewline + if (-not (Get-Command "mongod" -ErrorAction SilentlyContinue)) { + if(Test-Path 'C:\Program Files\MongoDB\Server\*\bin\mongod.exe' -PathType Leaf) { + # we have mongo server installed on standard Windows location + # so we can inject it to the PATH. We'll use latest version available. + $mongoVersions = Get-ChildItem -Directory 'C:\Program Files\MongoDB\Server' | Sort-Object -Property {$_.Name -as [int]} + if(Test-Path "C:\Program Files\MongoDB\Server\$($mongoVersions[-1])\bin\mongod.exe" -PathType Leaf) { + $env:PATH="$($env:PATH);C:\Program Files\MongoDB\Server\$($mongoVersions[-1])\bin\" + Write-Host "OK" -ForegroundColor Green + Write-Host " - auto-added from [ " -NoNewline + Write-Host "C:\Program Files\MongoDB\Server\$($mongoVersions[-1])\bin\" -NoNewLine -ForegroundColor Cyan + Write-Host " ]" + } else { + Write-Host "FAILED " -NoNewLine -ForegroundColor Red + Write-Host "MongoDB not detected" -ForegroundColor Yellow + Write-Host "Tried to find it on standard location [ " -NoNewline -ForegroundColor Gray + Write-Host "C:\Program Files\MongoDB\Server\$($mongoVersions[-1])\bin\" -NoNewline -ForegroundColor White + Write-Host " ] but failed." -ForegroundColor Gray + Exit-WithCode 1 + } + } else { + Write-Host "FAILED " -NoNewLine -ForegroundColor Red + Write-Host "MongoDB not detected in PATH" -ForegroundColor Yellow + Exit-WithCode 1 + } + + } else { + Write-Host "OK" -ForegroundColor Green + } + <# + .SYNOPSIS + Function to detect mongod in path. + .DESCRIPTION + This will test presence of mongod in PATH. If it's not there, it will try + to find it in default install location. It support different mongo versions + (using latest if found). When mongod is found, path to it is added to PATH + #> +} + +$script_dir = Split-Path -Path $MyInvocation.MyCommand.Definition -Parent +$pype_root = (Get-Item $script_dir).parent.FullName + +# mongodb port +$port = 2707 + +# path to database +$dbpath = (Get-Item $pype_root).parent.FullName + "\mongo_db_data" + +Find-Mongo +$mongo = Get-Command "mongod" | Select-Object -ExpandProperty Definition +Start-Process -FilePath $mongo "--dbpath $($dbpath) --port $($port)" diff --git a/tools/run_settings.ps1 b/tools/run_settings.ps1 new file mode 100644 index 0000000000..725e0cd8a0 --- /dev/null +++ b/tools/run_settings.ps1 @@ -0,0 +1,20 @@ +<# +.SYNOPSIS + Helper script to Pype Settings UI + +.DESCRIPTION + This script will run Pype and open Settings UI. + +.EXAMPLE + +PS> .\run_settings.ps1 + +#> + +$script_dir = Split-Path -Path $MyInvocation.MyCommand.Definition -Parent +$pype_root = (Get-Item $script_dir).parent.FullName + +& "$($pype_root)\venv\Scripts\Activate.ps1" + +python "$($pype_root)\start.py" settings --dev +deactivate \ No newline at end of file diff --git a/tools/run_tests.bat b/tools/run_tests.bat deleted file mode 100644 index 48ccf9e7cc..0000000000 --- a/tools/run_tests.bat +++ /dev/null @@ -1,2 +0,0 @@ -set PYTHONPATH=".;%PYTHONPATH%" -pytest -x --capture=sys --print -W ignore::DeprecationWarning ./tests diff --git a/tools/run_tests.ps1 b/tools/run_tests.ps1 new file mode 100644 index 0000000000..4159114ad3 --- /dev/null +++ b/tools/run_tests.ps1 @@ -0,0 +1,141 @@ +<# +.SYNOPSIS + Helper script to build Pype. + +.DESCRIPTION + This script will detect Python installation, create venv and install + all necessary packages from `requirements.txt` needed by Pype to be + included during application freeze on Windows. + +.EXAMPLE + +PS> .\run_test.ps1 + +#> + +function Exit-WithCode($exitcode) { + # Only exit this host process if it's a child of another PowerShell parent process... + $parentPID = (Get-CimInstance -ClassName Win32_Process -Filter "ProcessId=$PID" | Select-Object -Property ParentProcessId).ParentProcessId + $parentProcName = (Get-CimInstance -ClassName Win32_Process -Filter "ProcessId=$parentPID" | Select-Object -Property Name).Name + if ('powershell.exe' -eq $parentProcName) { $host.SetShouldExit($exitcode) } + + exit $exitcode +} + +function Show-PSWarning() { + if ($PSVersionTable.PSVersion.Major -lt 7) { + Write-Host "!!! " -NoNewline -ForegroundColor Red + Write-Host "You are using old version of PowerShell. $($PSVersionTable.PSVersion.Major).$($PSVersionTable.PSVersion.Minor)" + Write-Host "Please update to at least 7.0 - " -NoNewline -ForegroundColor Gray + Write-Host "https://github.com/PowerShell/PowerShell/releases" -ForegroundColor White + Exit-WithCode 1 + } +} +$art = @" + + + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ______ ___ ___ ___ + \ \ \___/ /\ \ \ \\ \\ \ + \ \____\ \ \_____\ \__\\__\\__\ + \/____/ \/_____/ . PYPE Club . + +"@ + +Write-Host $art -ForegroundColor DarkGreen + +# Enable if PS 7.x is needed. +# Show-PSWarning + +$current_dir = Get-Location +$script_dir = Split-Path -Path $MyInvocation.MyCommand.Definition -Parent +$pype_root = (Get-Item $script_dir).parent.FullName + +Set-Location -Path $pype_root + +$version_file = Get-Content -Path "$($pype_root)\pype\version.py" +$result = [regex]::Matches($version_file, '__version__ = "(?\d+\.\d+.\d+)"') +$pype_version = $result[0].Groups['version'].Value +if (-not $pype_version) { + Write-Host "!!! " -ForegroundColor yellow -NoNewline + Write-Host "Cannot determine Pype version." + Exit-WithCode 1 +} + +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "Building Pype [ " -NoNewline -ForegroundColor white +Write-host $pype_version -NoNewline -ForegroundColor green +Write-Host " ] ..." -ForegroundColor white + +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "Detecting host Python ... " -NoNewline +if (-not (Get-Command "python" -ErrorAction SilentlyContinue)) { + Write-Host "!!! Python not detected" -ForegroundColor red + Exit-WithCode 1 +} +$version_command = @" +import sys +print('{0}.{1}'.format(sys.version_info[0], sys.version_info[1])) +"@ + +$p = & python -c $version_command +$env:PYTHON_VERSION = $p +$m = $p -match '(\d+)\.(\d+)' +if(-not $m) { + Write-Host "!!! Cannot determine version" -ForegroundColor red + Exit-WithCode 1 +} +# We are supporting python 3.6 and up +if(($matches[1] -lt 3) -or ($matches[2] -lt 7)) { + Write-Host "FAILED Version [ $p ] is old and unsupported" -ForegroundColor red + Exit-WithCode 1 +} +Write-Host "OK [ $p ]" -ForegroundColor green + +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "Entering venv ..." +try { + . ("$($pype_root)\venv\Scripts\Activate.ps1") +} +catch { + Write-Host "!!! Failed to activate" -ForegroundColor red + Write-Host ">>> " -NoNewline -ForegroundColor green + Write-Host "Trying to create env ..." + & "$($script_dir)\create_env.ps1" + try { + . ("$($pype_root)\venv\Scripts\Activate.ps1") + } + catch { + Write-Host "!!! Failed to activate" -ForegroundColor red + Write-Host $_.Exception.Message + Exit-WithCode 1 + } +} + +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "Cleaning cache files ... " -NoNewline +Get-ChildItem $pype_root -Filter "*.pyc" -Force -Recurse | Remove-Item -Force +Get-ChildItem $pype_root -Filter "__pycache__" -Force -Recurse | Remove-Item -Force -Recurse +Write-Host "OK" -ForegroundColor green + +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "Testing Pype ..." +$original_pythonpath = $env:PYTHONPATH +$env:PYTHONPATH="$($pype_root);$($env:PYTHONPATH)" +pytest -x --capture=sys --print -W ignore::DeprecationWarning "$($pype_root)/tests" +$env:PYTHONPATH = $original_pythonpath +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "deactivating venv ..." +deactivate + +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "restoring current directory" +Set-Location -Path $current_dir + + + + + + diff --git a/tools/run_tray.ps1 b/tools/run_tray.ps1 new file mode 100644 index 0000000000..0b75354756 --- /dev/null +++ b/tools/run_tray.ps1 @@ -0,0 +1,19 @@ +<# +.SYNOPSIS + Helper script Pype Tray. + +.DESCRIPTION + + +.EXAMPLE + +PS> .\run_tray.ps1 + +#> +$script_dir = Split-Path -Path $MyInvocation.MyCommand.Definition -Parent +$pype_root = (Get-Item $script_dir).parent.FullName + +& "$($pype_root)\venv\Scripts\Activate.ps1" + +python "$($pype_root)\start.py" tray --debug +deactivate