add script to download and extract dependencies

tools/fetch_thirdparty_libs.ps1

@@ -0,0 +1,21 @@
+<#
+.SYNOPSIS
+  Download and extract third-party dependencies for Pype.
+
+.DESCRIPTION
+  This will download third-party dependencies specified in pyproject.toml
+  and extract them to vendor/bin folder.
+ #>
+
+.EXAMPLE
+
+PS> .\fetch_thirdparty_libs.ps1
+
+#>
+$current_dir = Get-Location
+$script_dir = Split-Path -Path $MyInvocation.MyCommand.Definition -Parent
+$pype_root = (Get-Item $script_dir).parent.FullName
+Set-Location -Path $pype_root
+
+& poetry run python "$($pype_root)\tools\fetch_thirdparty_libs.py"
+Set-Location -Path $current_dir

tools/fetch_thirdparty_libs.py

@@ -0,0 +1,165 @@
+# -*- coding: utf-8 -*-
+"""Fetch, verify and process third-party dependencies of Pype.
+
+Those should be defined in `pyproject.toml` in Pype sources root.
+
+"""
+import os
+import sys
+import toml
+import shutil
+from pathlib import Path
+from urllib.parse import urlparse
+import requests
+import enlighten
+import platform
+import blessed
+import tempfile
+import math
+import hashlib
+import tarfile
+import zipfile
+import time
+
+
+term = blessed.Terminal()
+manager = enlighten.get_manager()
+hash_buffer_size = 65536
+
+
+def sha256_sum(filename: Path):
+    """Calculate sha256 hash for given file.
+
+    Args:
+        filename (Path): path to file.
+
+    Returns:
+        str: hex hash.
+
+    """
+    _hash = hashlib.sha256()
+    with open(filename, 'rb', buffering=0) as f:
+        buffer = bytearray(128*1024)
+        mv = memoryview(buffer)
+        for n in iter(lambda: f.readinto(mv), 0):
+            _hash.update(mv[:n])
+    return _hash.hexdigest()
+
+
+def _print(msg: str, message_type: int = 0) -> None:
+    """Print message to console.
+
+    Args:
+        msg (str): message to print
+        message_type (int): type of message (0 info, 1 error, 2 note)
+
+    """
+    if message_type == 0:
+        header = term.aquamarine3(">>> ")
+    elif message_type == 1:
+        header = term.orangered2("!!! ")
+    elif message_type == 2:
+        header = term.tan1("... ")
+    else:
+        header = term.darkolivegreen3("--- ")
+
+    print("{}{}".format(header, msg))
+
+
+_print("Processing third-party dependencies ...")
+start_time = time.time_ns()
+pype_root = Path(os.path.dirname(__file__)).parent
+pyproject = toml.load(pype_root / "pyproject.toml")
+platform_name = platform.system().lower()
+
+try:
+    thirdparty = pyproject["pype"]["thirdparty"]
+except AttributeError:
+    _print("No third-party libraries specified in pyproject.toml", 1)
+    sys.exit(1)
+
+for k, v in thirdparty.items():
+    _print(f"processing {k}")
+    destination_path = pype_root / "vendor" / "bin" / k / platform_name
+    url = v.get(platform_name).get("url")
+
+
+    if not v.get(platform_name):
+        _print(("missing definition for current "
+                f"platform [ {platform_name} ]"), 1)
+        sys.exit(1)
+
+    parsed_url = urlparse(url)
+
+    # check if file is already extracted in /vendor/bin
+    if destination_path.exists():
+        _print("destination path already exists, deleting ...", 2)
+        if destination_path.is_dir():
+            try:
+                shutil.rmtree(destination_path)
+            except OSError as e:
+                _print("cannot delete folder.", 1)
+                raise SystemExit(e)
+
+    # download file
+    _print(f"Downloading {url} ...")
+    with tempfile.TemporaryDirectory() as temp_dir:
+        temp_file = Path(temp_dir) / Path(parsed_url.path).name
+
+        r = requests.get(url, stream=True)
+        content_len = int(r.headers.get('Content-Length', '0')) or None
+        with manager.counter(color='green',
+                             total=content_len and math.ceil(content_len / 2 ** 20),  # noqa: E501
+                             unit='MiB', leave=False) as counter:
+            with open(temp_file, 'wb', buffering=2 ** 24) as file_handle:
+                for chunk in r.iter_content(chunk_size=2 ** 20):
+                    file_handle.write(chunk)
+                    counter.update()
+
+        # get file with checksum
+        _print("Calculating sha256 ...", 2)
+        calc_checksum = sha256_sum(temp_file)
+        if v.get(platform_name).get("hash") != calc_checksum:
+            _print("Downloaded files checksum invalid.")
+            sys.exit(1)
+
+        _print("File OK", 3)
+        if not destination_path.exists():
+            destination_path.mkdir(parents=True)
+
+        # extract to destination
+        archive_type = temp_file.suffix.lstrip(".")
+        _print(f"Extracting {archive_type} file to {destination_path}")
+        if archive_type in ['zip']:
+            zip_file = zipfile.ZipFile(temp_file)
+            zip_file.extractall(destination_path)
+            zip_file.close()
+
+        elif archive_type in [
+            'tar', 'tgz', 'tar.gz', 'tar.xz', 'tar.bz2'
+        ]:
+            if archive_type == 'tar':
+                tar_type = 'r:'
+            elif archive_type.endswith('xz'):
+                tar_type = 'r:xz'
+            elif archive_type.endswith('gz'):
+                tar_type = 'r:gz'
+            elif archive_type.endswith('bz2'):
+                tar_type = 'r:bz2'
+            else:
+                tar_type = 'r:*'
+            try:
+                tar_file = tarfile.open(temp_file, tar_type)
+            except tarfile.ReadError:
+                raise SystemExit(
+                    "corrupted archive: also consider to download the "
+                    "archive manually, add its path to the url, run "
+                    "`./pype deploy`"
+                )
+            tar_file.extractall(destination_path)
+            tar_file.close()
+        _print("Extraction OK", 3)
+
+end_time = time.time_ns()
+total_time = (end_time - start_time) / 1000000000
+_print(f"Downloading and extracting took {total_time} secs.")

tools/fetch_thirdparty_libs.sh

@@ -0,0 +1,129 @@
+#!/usr/bin/env bash
+
+# Run Pype Tray
+
+
+art () {
+  cat <<-EOF
+  ____________
+ /\\      ___  \\
+ \\ \\     \\/_\\  \\
+  \\ \\     _____/ ______   ___ ___ ___
+   \\ \\    \\___/ /\\     \\  \\  \\\\  \\\\  \\
+    \\ \\____\\    \\ \\_____\\  \\__\\\\__\\\\__\\
+     \\/____/     \\/_____/  . PYPE Club .
+
+EOF
+}
+
+# Colors for terminal
+
+RST='\033[0m'             # Text Reset
+
+# Regular Colors
+Black='\033[0;30m'        # Black
+Red='\033[0;31m'          # Red
+Green='\033[0;32m'        # Green
+Yellow='\033[0;33m'       # Yellow
+Blue='\033[0;34m'         # Blue
+Purple='\033[0;35m'       # Purple
+Cyan='\033[0;36m'         # Cyan
+White='\033[0;37m'        # White
+
+# Bold
+BBlack='\033[1;30m'       # Black
+BRed='\033[1;31m'         # Red
+BGreen='\033[1;32m'       # Green
+BYellow='\033[1;33m'      # Yellow
+BBlue='\033[1;34m'        # Blue
+BPurple='\033[1;35m'      # Purple
+BCyan='\033[1;36m'        # Cyan
+BWhite='\033[1;37m'       # White
+
+# Bold High Intensity
+BIBlack='\033[1;90m'      # Black
+BIRed='\033[1;91m'        # Red
+BIGreen='\033[1;92m'      # Green
+BIYellow='\033[1;93m'     # Yellow
+BIBlue='\033[1;94m'       # Blue
+BIPurple='\033[1;95m'     # Purple
+BICyan='\033[1;96m'       # Cyan
+BIWhite='\033[1;97m'      # White
+
+
+##############################################################################
+# Detect required version of python
+# Globals:
+#   colors
+#   PYTHON
+# Arguments:
+#   None
+# Returns:
+#   None
+###############################################################################
+detect_python () {
+  echo -e "${BIGreen}>>>${RST} Using python \c"
+  local version_command="import sys;print('{0}.{1}'.format(sys.version_info[0], sys.version_info[1]))"
+  local python_version="$(python3 <<< ${version_command})"
+  oIFS="$IFS"
+  IFS=.
+  set -- $python_version
+  IFS="$oIFS"
+  if [ "$1" -ge "3" ] && [ "$2" -ge "6" ] ; then
+    if [ "$2" -gt "7" ] ; then
+      echo -e "${BIWhite}[${RST} ${BIRed}$1.$2 ${BIWhite}]${RST} - ${BIRed}FAILED${RST} ${BIYellow}Version is new and unsupported, use${RST} ${BIPurple}3.7.x${RST}"; return 1;
+    else
+      echo -e "${BIWhite}[${RST} ${BIGreen}$1.$2${RST} ${BIWhite}]${RST}"
+    fi
+    PYTHON="python3"
+  else
+    command -v python3 >/dev/null 2>&1 || { echo -e "${BIRed}$1.$2$ - ${BIRed}FAILED${RST} ${BIYellow}Version is old and unsupported${RST}"; return 1; }
+  fi
+}
+
+##############################################################################
+# Clean pyc files in specified directory
+# Globals:
+#   None
+# Arguments:
+#   Optional path to clean
+# Returns:
+#   None
+###############################################################################
+clean_pyc () {
+  local path
+  path=$pype_root
+  echo -e "${BIGreen}>>>${RST} Cleaning pyc at [ ${BIWhite}$path${RST} ] ... \c"
+  find "$path" -regex '^.*\(__pycache__\|\.py[co]\)$' -delete
+  echo -e "${BIGreen}DONE${RST}"
+}
+
+##############################################################################
+# Return absolute path
+# Globals:
+#   None
+# Arguments:
+#   Path to resolve
+# Returns:
+#   None
+###############################################################################
+realpath () {
+  echo $(cd $(dirname "$1"); pwd)/$(basename "$1")
+}
+
+# Main
+main () {
+  echo -e "${BGreen}"
+  art
+  echo -e "${RST}"
+  detect_python || return 1
+
+  # Directories
+  pype_root=$(realpath $(dirname $(dirname "${BASH_SOURCE[0]}")))
+  pushd "$pype_root" > /dev/null || return > /dev/null
+
+  echo -e "${BIGreen}>>>${RST} Running Pype tool ..."
+  poetry run python3 "$pype_root/tools/fetch_thirdparty_libs.py"
+}
+
+main