mirror of
https://github.com/ynput/ayon-core.git
synced 2025-12-24 21:04:40 +01:00
add script to download and extract dependencies
This commit is contained in:
Ondrej Samohel
parent
b940d3ba9d
commit
52a5183b7b
6 changed files with 378 additions and 6 deletions
|
|
@ -50,7 +50,8 @@ git clone --recurse-submodules git@github.com:pypeclub/pype.git
|
|||
#### To build Pype:
|
||||
|
||||
1) Run `.\tools\create_env.ps1` to create virtual environment in `.\venv`
|
||||
2) Run `.\tools\build.ps1` to build pype executables in `.\build\`
|
||||
2) Run `.\tools\fetch_thirdparty_libs.ps1` to download third-party dependencies like ffmpeg and oiio. Those will be included in build.
|
||||
3) Run `.\tools\build.ps1` to build pype executables in `.\build\`
|
||||
|
||||
To create distributable Pype versions, run `./tools/create_zip.ps1` - that will
|
||||
create zip file with name `pype-vx.x.x.zip` parsed from current pype repository and
|
||||
|
|
@ -105,7 +106,8 @@ pyenv local 3.7.9
|
|||
#### To build Pype:
|
||||
|
||||
1) Run `.\tools\create_env.sh` to create virtual environment in `.\venv`
|
||||
2) Run `.\tools\build.sh` to build Pype executables in `.\build\`
|
||||
2) Run `.\tools\fetch_thirdparty_libs.sh` to download third-party dependencies like ffmpeg and oiio. Those will be included in build.
|
||||
3) Run `.\tools\build.sh` to build Pype executables in `.\build\`
|
||||
|
||||
|
||||
### Linux
|
||||
|
|
|
|||
32
poetry.lock
generated
32
poetry.lock
generated
|
|
@ -296,6 +296,18 @@ category = "dev"
|
|||
optional = false
|
||||
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
|
||||
|
||||
[[package]]
|
||||
name = "enlighten"
|
||||
version = "1.9.0"
|
||||
description = "Enlighten Progress Bar"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = "*"
|
||||
|
||||
[package.dependencies]
|
||||
blessed = ">=1.17.7"
|
||||
prefixed = ">=0.3.2"
|
||||
|
||||
[[package]]
|
||||
name = "evdev"
|
||||
version = "1.4.0"
|
||||
|
|
@ -637,7 +649,7 @@ view = ["PySide2 (>=5.11,<6.0)"]
|
|||
|
||||
[package.source]
|
||||
type = "legacy"
|
||||
url = "https://d.r1.wbsprt.com/pype.club/distribute"
|
||||
url = "https://distribute.openpype.io/wheels"
|
||||
reference = "pype"
|
||||
|
||||
[[package]]
|
||||
|
|
@ -696,6 +708,14 @@ importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""}
|
|||
[package.extras]
|
||||
dev = ["pre-commit", "tox"]
|
||||
|
||||
[[package]]
|
||||
name = "prefixed"
|
||||
version = "0.3.2"
|
||||
description = "Prefixed alternative numeric library"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = "*"
|
||||
|
||||
[[package]]
|
||||
name = "protobuf"
|
||||
version = "3.15.6"
|
||||
|
|
@ -1391,7 +1411,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pyt
|
|||
[metadata]
|
||||
lock-version = "1.1"
|
||||
python-versions = "3.7.*"
|
||||
content-hash = "4905515073ad2bf2a8517d513d68e48669b6a829f24e540b2dd60bc70cbea26b"
|
||||
content-hash = "b356e327dbaa1aa38dbf1463901f64539f2c8d07be8d8a017e83b8a1554dbff9"
|
||||
|
||||
[metadata.files]
|
||||
acre = []
|
||||
|
|
@ -1636,6 +1656,10 @@ docutils = [
|
|||
{file = "docutils-0.16-py2.py3-none-any.whl", hash = "sha256:0c5b78adfbf7762415433f5515cd5c9e762339e23369dbe8000d84a4bf4ab3af"},
|
||||
{file = "docutils-0.16.tar.gz", hash = "sha256:c2de3a60e9e7d07be26b7f2b00ca0309c207e06c100f9cc2a94931fc75a478fc"},
|
||||
]
|
||||
enlighten = [
|
||||
{file = "enlighten-1.9.0-py2.py3-none-any.whl", hash = "sha256:5c59e41505702243c6b26437403e371d2a146ac72de5f706376f738ea8f32659"},
|
||||
{file = "enlighten-1.9.0.tar.gz", hash = "sha256:539cc308ccc0c3bfb50feb1b2da94c1a1ac21e80fe95e984221de8966d48f428"},
|
||||
]
|
||||
evdev = [
|
||||
{file = "evdev-1.4.0.tar.gz", hash = "sha256:8782740eb1a86b187334c07feb5127d3faa0b236e113206dfe3ae8f77fb1aaf1"},
|
||||
]
|
||||
|
|
@ -1896,6 +1920,10 @@ pluggy = [
|
|||
{file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"},
|
||||
{file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"},
|
||||
]
|
||||
prefixed = [
|
||||
{file = "prefixed-0.3.2-py2.py3-none-any.whl", hash = "sha256:5e107306462d63f2f03c529dbf11b0026fdfec621a9a008ca639d71de22995c3"},
|
||||
{file = "prefixed-0.3.2.tar.gz", hash = "sha256:ca48277ba5fa8346dd4b760847da930c7b84416387c39e93affef086add2c029"},
|
||||
]
|
||||
protobuf = [
|
||||
{file = "protobuf-3.15.6-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:1771ef20e88759c4d81db213e89b7a1fc53937968e12af6603c658ee4bcbfa38"},
|
||||
{file = "protobuf-3.15.6-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:1a66261a402d05c8ad8c1fde8631837307bf8d7e7740a4f3941fc3277c2e1528"},
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ jinxed = [
|
|||
{ version = "^1.0.1", markers = "sys_platform == 'linux'" }
|
||||
]
|
||||
python3-xlib = { version="*", markers = "sys_platform == 'linux'"}
|
||||
enlighten = "^1.9.0"
|
||||
|
||||
[tool.poetry.dev-dependencies]
|
||||
flake8 = "^3.7"
|
||||
|
|
@ -61,8 +62,8 @@ sphinx-rtd-theme = "*"
|
|||
sphinxcontrib-websupport = "*"
|
||||
sphinx-qt-documentation = "*"
|
||||
recommonmark = "*"
|
||||
tqdm = "*"
|
||||
wheel = "*"
|
||||
enlighten = "*" # cool terminal progress bars
|
||||
|
||||
[tool.poetry.urls]
|
||||
"Bug Tracker" = "https://github.com/pypeclub/pype/issues"
|
||||
|
|
@ -70,8 +71,34 @@ wheel = "*"
|
|||
|
||||
[[tool.poetry.source]]
|
||||
name = "pype"
|
||||
url = "https://d.r1.wbsprt.com/pype.club/distribute/"
|
||||
url = "https://distribute.openpype.io/wheels/"
|
||||
|
||||
[build-system]
|
||||
requires = ["poetry-core>=1.0.0"]
|
||||
build-backend = "poetry.core.masonry.api"
|
||||
|
||||
[pype]
|
||||
|
||||
[pype.thirdparty.ffmpeg.windows]
|
||||
url = "https://distribute.openpype.io/thirdparty/ffmpeg-4.13-windows.zip"
|
||||
hash = "43988ebcba98313635f06f2ca7e2dd52670710ebceefaa77107321b1def30472"
|
||||
|
||||
[pype.thirdparty.ffmpeg.linux]
|
||||
url = "https://distribute.openpype.io/thirdparty/ffmpeg-20200504-linux.tgz"
|
||||
hash = "sha256:..."
|
||||
|
||||
[pype.thirdparty.ffmpeg.darwin]
|
||||
url = "https://distribute.openpype.io/thirdparty/ffmpeg-20200504-darwin.tgz"
|
||||
hash = "sha256:..."
|
||||
|
||||
[pype.thirdparty.oiio.windows]
|
||||
url = "https://distribute.openpype.io/thirdparty/oiio_tools-2.2.0-windows.zip"
|
||||
hash = "fd2e00278e01e85dcee7b4a6969d1a16f13016ec16700fb0366dbb1b1f3c37ad"
|
||||
|
||||
[pype.thirdparty.oiio.linux]
|
||||
url = "https://distribute.openpype.io/thirdparty/oiio-2.2.0-linux.tgz"
|
||||
hash = "sha256:..."
|
||||
|
||||
[pype.thirdparty.oiio.darwin]
|
||||
url = "https://distribute.openpype.io/thirdparty/oiio-2.2.0-darwin.tgz"
|
||||
hash = "sha256:..."
|
||||
21
tools/fetch_thirdparty_libs.ps1
Normal file
21
tools/fetch_thirdparty_libs.ps1
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
<#
|
||||
.SYNOPSIS
|
||||
Download and extract third-party dependencies for Pype.
|
||||
|
||||
.DESCRIPTION
|
||||
This will download third-party dependencies specified in pyproject.toml
|
||||
and extract them to vendor/bin folder.
|
||||
#>
|
||||
|
||||
.EXAMPLE
|
||||
|
||||
PS> .\fetch_thirdparty_libs.ps1
|
||||
|
||||
#>
|
||||
$current_dir = Get-Location
|
||||
$script_dir = Split-Path -Path $MyInvocation.MyCommand.Definition -Parent
|
||||
$pype_root = (Get-Item $script_dir).parent.FullName
|
||||
Set-Location -Path $pype_root
|
||||
|
||||
& poetry run python "$($pype_root)\tools\fetch_thirdparty_libs.py"
|
||||
Set-Location -Path $current_dir
|
||||
165
tools/fetch_thirdparty_libs.py
Normal file
165
tools/fetch_thirdparty_libs.py
Normal file
|
|
@ -0,0 +1,165 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
"""Fetch, verify and process third-party dependencies of Pype.
|
||||
|
||||
Those should be defined in `pyproject.toml` in Pype sources root.
|
||||
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import toml
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
import requests
|
||||
import enlighten
|
||||
import platform
|
||||
import blessed
|
||||
import tempfile
|
||||
import math
|
||||
import hashlib
|
||||
import tarfile
|
||||
import zipfile
|
||||
import time
|
||||
|
||||
|
||||
term = blessed.Terminal()
|
||||
manager = enlighten.get_manager()
|
||||
hash_buffer_size = 65536
|
||||
|
||||
|
||||
def sha256_sum(filename: Path):
|
||||
"""Calculate sha256 hash for given file.
|
||||
|
||||
Args:
|
||||
filename (Path): path to file.
|
||||
|
||||
Returns:
|
||||
str: hex hash.
|
||||
|
||||
"""
|
||||
_hash = hashlib.sha256()
|
||||
with open(filename, 'rb', buffering=0) as f:
|
||||
buffer = bytearray(128*1024)
|
||||
mv = memoryview(buffer)
|
||||
for n in iter(lambda: f.readinto(mv), 0):
|
||||
_hash.update(mv[:n])
|
||||
return _hash.hexdigest()
|
||||
|
||||
|
||||
def _print(msg: str, message_type: int = 0) -> None:
|
||||
"""Print message to console.
|
||||
|
||||
Args:
|
||||
msg (str): message to print
|
||||
message_type (int): type of message (0 info, 1 error, 2 note)
|
||||
|
||||
"""
|
||||
if message_type == 0:
|
||||
header = term.aquamarine3(">>> ")
|
||||
elif message_type == 1:
|
||||
header = term.orangered2("!!! ")
|
||||
elif message_type == 2:
|
||||
header = term.tan1("... ")
|
||||
else:
|
||||
header = term.darkolivegreen3("--- ")
|
||||
|
||||
print("{}{}".format(header, msg))
|
||||
|
||||
|
||||
_print("Processing third-party dependencies ...")
|
||||
start_time = time.time_ns()
|
||||
pype_root = Path(os.path.dirname(__file__)).parent
|
||||
pyproject = toml.load(pype_root / "pyproject.toml")
|
||||
platform_name = platform.system().lower()
|
||||
|
||||
try:
|
||||
thirdparty = pyproject["pype"]["thirdparty"]
|
||||
except AttributeError:
|
||||
_print("No third-party libraries specified in pyproject.toml", 1)
|
||||
sys.exit(1)
|
||||
|
||||
for k, v in thirdparty.items():
|
||||
_print(f"processing {k}")
|
||||
destination_path = pype_root / "vendor" / "bin" / k / platform_name
|
||||
url = v.get(platform_name).get("url")
|
||||
|
||||
|
||||
if not v.get(platform_name):
|
||||
_print(("missing definition for current "
|
||||
f"platform [ {platform_name} ]"), 1)
|
||||
sys.exit(1)
|
||||
|
||||
parsed_url = urlparse(url)
|
||||
|
||||
# check if file is already extracted in /vendor/bin
|
||||
if destination_path.exists():
|
||||
_print("destination path already exists, deleting ...", 2)
|
||||
if destination_path.is_dir():
|
||||
try:
|
||||
shutil.rmtree(destination_path)
|
||||
except OSError as e:
|
||||
_print("cannot delete folder.", 1)
|
||||
raise SystemExit(e)
|
||||
|
||||
# download file
|
||||
_print(f"Downloading {url} ...")
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
temp_file = Path(temp_dir) / Path(parsed_url.path).name
|
||||
|
||||
r = requests.get(url, stream=True)
|
||||
content_len = int(r.headers.get('Content-Length', '0')) or None
|
||||
with manager.counter(color='green',
|
||||
total=content_len and math.ceil(content_len / 2 ** 20), # noqa: E501
|
||||
unit='MiB', leave=False) as counter:
|
||||
with open(temp_file, 'wb', buffering=2 ** 24) as file_handle:
|
||||
for chunk in r.iter_content(chunk_size=2 ** 20):
|
||||
file_handle.write(chunk)
|
||||
counter.update()
|
||||
|
||||
# get file with checksum
|
||||
_print("Calculating sha256 ...", 2)
|
||||
calc_checksum = sha256_sum(temp_file)
|
||||
if v.get(platform_name).get("hash") != calc_checksum:
|
||||
_print("Downloaded files checksum invalid.")
|
||||
sys.exit(1)
|
||||
|
||||
_print("File OK", 3)
|
||||
if not destination_path.exists():
|
||||
destination_path.mkdir(parents=True)
|
||||
|
||||
# extract to destination
|
||||
archive_type = temp_file.suffix.lstrip(".")
|
||||
_print(f"Extracting {archive_type} file to {destination_path}")
|
||||
if archive_type in ['zip']:
|
||||
zip_file = zipfile.ZipFile(temp_file)
|
||||
zip_file.extractall(destination_path)
|
||||
zip_file.close()
|
||||
|
||||
elif archive_type in [
|
||||
'tar', 'tgz', 'tar.gz', 'tar.xz', 'tar.bz2'
|
||||
]:
|
||||
if archive_type == 'tar':
|
||||
tar_type = 'r:'
|
||||
elif archive_type.endswith('xz'):
|
||||
tar_type = 'r:xz'
|
||||
elif archive_type.endswith('gz'):
|
||||
tar_type = 'r:gz'
|
||||
elif archive_type.endswith('bz2'):
|
||||
tar_type = 'r:bz2'
|
||||
else:
|
||||
tar_type = 'r:*'
|
||||
try:
|
||||
tar_file = tarfile.open(temp_file, tar_type)
|
||||
except tarfile.ReadError:
|
||||
raise SystemExit(
|
||||
"corrupted archive: also consider to download the "
|
||||
"archive manually, add its path to the url, run "
|
||||
"`./pype deploy`"
|
||||
)
|
||||
tar_file.extractall(destination_path)
|
||||
tar_file.close()
|
||||
_print("Extraction OK", 3)
|
||||
|
||||
end_time = time.time_ns()
|
||||
total_time = (end_time - start_time) / 1000000000
|
||||
_print(f"Downloading and extracting took {total_time} secs.")
|
||||
129
tools/fetch_thirdparty_libs.sh
Executable file
129
tools/fetch_thirdparty_libs.sh
Executable file
|
|
@ -0,0 +1,129 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# Run Pype Tray
|
||||
|
||||
|
||||
art () {
|
||||
cat <<-EOF
|
||||
____________
|
||||
/\\ ___ \\
|
||||
\\ \\ \\/_\\ \\
|
||||
\\ \\ _____/ ______ ___ ___ ___
|
||||
\\ \\ \\___/ /\\ \\ \\ \\\\ \\\\ \\
|
||||
\\ \\____\\ \\ \\_____\\ \\__\\\\__\\\\__\\
|
||||
\\/____/ \\/_____/ . PYPE Club .
|
||||
|
||||
EOF
|
||||
}
|
||||
|
||||
# Colors for terminal
|
||||
|
||||
RST='\033[0m' # Text Reset
|
||||
|
||||
# Regular Colors
|
||||
Black='\033[0;30m' # Black
|
||||
Red='\033[0;31m' # Red
|
||||
Green='\033[0;32m' # Green
|
||||
Yellow='\033[0;33m' # Yellow
|
||||
Blue='\033[0;34m' # Blue
|
||||
Purple='\033[0;35m' # Purple
|
||||
Cyan='\033[0;36m' # Cyan
|
||||
White='\033[0;37m' # White
|
||||
|
||||
# Bold
|
||||
BBlack='\033[1;30m' # Black
|
||||
BRed='\033[1;31m' # Red
|
||||
BGreen='\033[1;32m' # Green
|
||||
BYellow='\033[1;33m' # Yellow
|
||||
BBlue='\033[1;34m' # Blue
|
||||
BPurple='\033[1;35m' # Purple
|
||||
BCyan='\033[1;36m' # Cyan
|
||||
BWhite='\033[1;37m' # White
|
||||
|
||||
# Bold High Intensity
|
||||
BIBlack='\033[1;90m' # Black
|
||||
BIRed='\033[1;91m' # Red
|
||||
BIGreen='\033[1;92m' # Green
|
||||
BIYellow='\033[1;93m' # Yellow
|
||||
BIBlue='\033[1;94m' # Blue
|
||||
BIPurple='\033[1;95m' # Purple
|
||||
BICyan='\033[1;96m' # Cyan
|
||||
BIWhite='\033[1;97m' # White
|
||||
|
||||
|
||||
##############################################################################
|
||||
# Detect required version of python
|
||||
# Globals:
|
||||
# colors
|
||||
# PYTHON
|
||||
# Arguments:
|
||||
# None
|
||||
# Returns:
|
||||
# None
|
||||
###############################################################################
|
||||
detect_python () {
|
||||
echo -e "${BIGreen}>>>${RST} Using python \c"
|
||||
local version_command="import sys;print('{0}.{1}'.format(sys.version_info[0], sys.version_info[1]))"
|
||||
local python_version="$(python3 <<< ${version_command})"
|
||||
oIFS="$IFS"
|
||||
IFS=.
|
||||
set -- $python_version
|
||||
IFS="$oIFS"
|
||||
if [ "$1" -ge "3" ] && [ "$2" -ge "6" ] ; then
|
||||
if [ "$2" -gt "7" ] ; then
|
||||
echo -e "${BIWhite}[${RST} ${BIRed}$1.$2 ${BIWhite}]${RST} - ${BIRed}FAILED${RST} ${BIYellow}Version is new and unsupported, use${RST} ${BIPurple}3.7.x${RST}"; return 1;
|
||||
else
|
||||
echo -e "${BIWhite}[${RST} ${BIGreen}$1.$2${RST} ${BIWhite}]${RST}"
|
||||
fi
|
||||
PYTHON="python3"
|
||||
else
|
||||
command -v python3 >/dev/null 2>&1 || { echo -e "${BIRed}$1.$2$ - ${BIRed}FAILED${RST} ${BIYellow}Version is old and unsupported${RST}"; return 1; }
|
||||
fi
|
||||
}
|
||||
|
||||
##############################################################################
|
||||
# Clean pyc files in specified directory
|
||||
# Globals:
|
||||
# None
|
||||
# Arguments:
|
||||
# Optional path to clean
|
||||
# Returns:
|
||||
# None
|
||||
###############################################################################
|
||||
clean_pyc () {
|
||||
local path
|
||||
path=$pype_root
|
||||
echo -e "${BIGreen}>>>${RST} Cleaning pyc at [ ${BIWhite}$path${RST} ] ... \c"
|
||||
find "$path" -regex '^.*\(__pycache__\|\.py[co]\)$' -delete
|
||||
echo -e "${BIGreen}DONE${RST}"
|
||||
}
|
||||
|
||||
##############################################################################
|
||||
# Return absolute path
|
||||
# Globals:
|
||||
# None
|
||||
# Arguments:
|
||||
# Path to resolve
|
||||
# Returns:
|
||||
# None
|
||||
###############################################################################
|
||||
realpath () {
|
||||
echo $(cd $(dirname "$1"); pwd)/$(basename "$1")
|
||||
}
|
||||
|
||||
# Main
|
||||
main () {
|
||||
echo -e "${BGreen}"
|
||||
art
|
||||
echo -e "${RST}"
|
||||
detect_python || return 1
|
||||
|
||||
# Directories
|
||||
pype_root=$(realpath $(dirname $(dirname "${BASH_SOURCE[0]}")))
|
||||
pushd "$pype_root" > /dev/null || return > /dev/null
|
||||
|
||||
echo -e "${BIGreen}>>>${RST} Running Pype tool ..."
|
||||
poetry run python3 "$pype_root/tools/fetch_thirdparty_libs.py"
|
||||
}
|
||||
|
||||
main
|
||||
Loading…
Add table
Add a link
Reference in a new issue