OP-2019 - merge develop

.github/pull_request_template.md

@@ -0,0 +1,16 @@
+## Brief description
+First sentence is brief description.
+
+## Description
+Next paragraf is more elaborate text with more info. This will be displayed for example in collapsed form under the first sentence in a changelog.
+
+## Additional info
+The rest will be ignored in changelog and should contain any additional
+technical information.
+
+## Documentation (add _"type: documentation"_ label)
+[feature_documentation](future_url_after_it_will_be_merged)
+
+## Testing notes:
+1. start with this step
+2. follow this step

.github/workflows/prerelease.yml

@@ -33,7 +33,7 @@ jobs:
       id: version
       if: steps.version_type.outputs.type != 'skip'
       run: |
-        RESULT=$(python ./tools/ci_tools.py --nightly)
+        RESULT=$(python ./tools/ci_tools.py --nightly --github_token ${{ secrets.GITHUB_TOKEN }})
 
         echo ::set-output name=next_tag::$RESULT
 

Dockerfile.centos7

@@ -41,6 +41,8 @@ RUN yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.n
 	    ncurses \
 	    ncurses-devel \
         qt5-qtbase-devel \
+        xcb-util-wm \
+        xcb-util-renderutil \
     && yum clean all
 
 # we need to build our own patchelf
@@ -92,7 +94,8 @@ RUN source $HOME/.bashrc \
 RUN cp /usr/lib64/libffi* ./build/exe.linux-x86_64-3.7/lib \
     && cp /usr/lib64/libssl* ./build/exe.linux-x86_64-3.7/lib \
     && cp /usr/lib64/libcrypto* ./build/exe.linux-x86_64-3.7/lib \
-    && cp /root/.pyenv/versions/${OPENPYPE_PYTHON_VERSION}/lib/libpython* ./build/exe.linux-x86_64-3.7/lib
+    && cp /root/.pyenv/versions/${OPENPYPE_PYTHON_VERSION}/lib/libpython* ./build/exe.linux-x86_64-3.7/lib \
+    && cp /usr/lib64/libxcb* ./build/exe.linux-x86_64-3.7/vendor/python/PySide2/Qt/lib
 
 RUN cd /opt/openpype \
     rm -rf ./vendor/bin

igniter/bootstrap_repos.py

@@ -10,6 +10,7 @@
 from pathlib import Path
 from typing import Union, Callable, List, Tuple
 import hashlib
+import platform
 
 from zipfile import ZipFile, BadZipFile
 
@@ -196,21 +197,23 @@ def get_main_version(self) -> str:
         return str(self.finalize_version())
 
     @staticmethod
-    def version_in_str(string: str) -> Tuple:
+    def version_in_str(string: str) -> Union[None, OpenPypeVersion]:
         """Find OpenPype version in given string.
 
         Args:
             string (str):  string to search.
 
         Returns:
-            tuple: True/False and OpenPypeVersion if found.
+            OpenPypeVersion: of detected or None.
 
         """
         m = re.search(OpenPypeVersion._VERSION_REGEX, string)
         if not m:
-            return False, None
+            return None
         version = OpenPypeVersion.parse(string[m.start():m.end()])
-        return True, version
+        if "staging" in string[m.start():m.end()]:
+            version.staging = True
+        return version
 
     @classmethod
     def parse(cls, version):
@@ -531,6 +534,7 @@ def _create_openpype_zip(self, zip_path: Path, openpype_path: Path) -> None:
                 processed_path = file
                 self._print(f"- processing {processed_path}")
 
+
                 checksums.append(
                     (
                         sha256sum(file.as_posix()),
@@ -542,7 +546,10 @@ def _create_openpype_zip(self, zip_path: Path, openpype_path: Path) -> None:
 
             checksums_str = ""
             for c in checksums:
-                checksums_str += "{}:{}\n".format(c[0], c[1])
+                file_str = c[1]
+                if platform.system().lower() == "windows":
+                    file_str = c[1].as_posix().replace("\\", "/")
+                checksums_str += "{}:{}\n".format(c[0], file_str)
             zip_file.writestr("checksums", checksums_str)
             # test if zip is ok
             zip_file.testzip()
@@ -563,6 +570,8 @@ def validate_openpype_version(self, path: Path) -> tuple:
                 and string with reason as second.
 
         """
+        if os.getenv("OPENPYPE_DONT_VALIDATE_VERSION"):
+            return True, "Disabled validation"
         if not path.exists():
             return False, "Path doesn't exist"
 
@@ -589,13 +598,16 @@ def _validate_zip(path: Path) -> tuple:
 
             # calculate and compare checksums in the zip file
             for file in checksums:
+                file_name = file[1]
+                if platform.system().lower() == "windows":
+                    file_name = file_name.replace("/", "\\")
                 h = hashlib.sha256()
                 try:
-                    h.update(zip_file.read(file[1]))
+                    h.update(zip_file.read(file_name))
                 except FileNotFoundError:
-                    return False, f"Missing file [ {file[1]} ]"
+                    return False, f"Missing file [ {file_name} ]"
                 if h.hexdigest() != file[0]:
-                    return False, f"Invalid checksum on {file[1]}"
+                    return False, f"Invalid checksum on {file_name}"
 
             # get list of files in zip minus `checksums` file itself
             # and turn in to set to compare against list of files
@@ -604,7 +616,7 @@ def _validate_zip(path: Path) -> tuple:
             files_in_zip = zip_file.namelist()
             files_in_zip.remove("checksums")
             files_in_zip = set(files_in_zip)
-        files_in_checksum = set([file[1] for file in checksums])
+        files_in_checksum = {file[1] for file in checksums}
         diff = files_in_zip.difference(files_in_checksum)
         if diff:
             return False, f"Missing files {diff}"
@@ -628,16 +640,19 @@ def _validate_dir(path: Path) -> tuple:
         ]
         files_in_dir.remove("checksums")
         files_in_dir = set(files_in_dir)
-        files_in_checksum = set([file[1] for file in checksums])
+        files_in_checksum = {file[1] for file in checksums}
 
         for file in checksums:
+            file_name = file[1]
+            if platform.system().lower() == "windows":
+                file_name = file_name.replace("/", "\\")
             try:
-                current = sha256sum((path / file[1]).as_posix())
+                current = sha256sum((path / file_name).as_posix())
             except FileNotFoundError:
-                return False, f"Missing file [ {file[1]} ]"
+                return False, f"Missing file [ {file_name} ]"
 
             if file[0] != current:
-                return False, f"Invalid checksum on {file[1]}"
+                return False, f"Invalid checksum on {file_name}"
         diff = files_in_dir.difference(files_in_checksum)
         if diff:
             return False, f"Missing files {diff}"
@@ -1161,9 +1176,9 @@ def get_openpype_versions(self,
             name = item.name if item.is_dir() else item.stem
             result = OpenPypeVersion.version_in_str(name)
 
-            if result[0]:
+            if result:
                 detected_version: OpenPypeVersion
-                detected_version = result[1]
+                detected_version = result
 
                 if item.is_dir() and not self._is_openpype_in_dir(
                     item, detected_version

igniter/tools.py

@@ -59,7 +59,7 @@ def validate_mongo_connection(cnx: str) -> (bool, str):
         return False, "Not mongodb schema"
 
     kwargs = {
-        "serverSelectionTimeoutMS": 2000
+        "serverSelectionTimeoutMS": os.environ.get("AVALON_TIMEOUT", 2000)
     }
     # Add certificate path if should be required
     if should_add_certificate_path_to_mongo_url(cnx):

igniter/version.py

@@ -1,4 +1,4 @@
 # -*- coding: utf-8 -*-
 """Definition of Igniter version."""
 
-__version__ = "1.0.1"
+__version__ = "1.0.2"

openpype/api.py

@@ -17,6 +17,7 @@
     version_up,
     get_asset,
     get_hierarchy,
+    get_workdir_data,
     get_version_from_path,
     get_last_version_from_path,
     get_app_environments_for_context,

openpype/cli.py

@@ -158,15 +158,17 @@ def extractenvironments(output_json_path, project, asset, task, app):
 @click.option("-d", "--debug", is_flag=True, help="Print debug messages")
 @click.option("-t", "--targets", help="Targets module", default=None,
               multiple=True)
-def publish(debug, paths, targets):
+@click.option("-g", "--gui", is_flag=True,
+              help="Show Publish UI", default=False)
+def publish(debug, paths, targets, gui):
     """Start CLI publishing.
 
     Publish collects json from paths provided as an argument.
     More than one path is allowed.
     """
     if debug:
         os.environ['OPENPYPE_DEBUG'] = '3'
-    PypeCommands.publish(list(paths), targets)
+    PypeCommands.publish(list(paths), targets, gui)
 
 
 @main.command()
@@ -354,6 +356,56 @@ def run(script):
               "--pyargs",
               help="Run tests from package",
               default=None)
-def runtests(folder, mark, pyargs):
+@click.option("-t",
+              "--test_data_folder",
+              help="Unzipped directory path of test file",
+              default=None)
+@click.option("-s",
+              "--persist",
+              help="Persist test DB and published files after test end",
+              default=None)
+@click.option("-a",
+              "--app_variant",
+              help="Provide specific app variant for test, empty for latest",
+              default=None)
+def runtests(folder, mark, pyargs, test_data_folder, persist, app_variant):
     """Run all automatic tests after proper initialization via start.py"""
-    PypeCommands().run_tests(folder, mark, pyargs)
+    PypeCommands().run_tests(folder, mark, pyargs, test_data_folder,
+                             persist, app_variant)
+
+
+@main.command()
+@click.option("-d", "--debug",
+              is_flag=True, help=("Run process in debug mode"))
+@click.option("-a", "--active_site", required=True,
+              help="Name of active stie")
+def syncserver(debug, active_site):
+    """Run sync site server in background.
+
+        Some Site Sync use cases need to expose site to another one.
+        For example if majority of artists work in studio, they are not using
+        SS at all, but if you want to expose published assets to 'studio' site
+        to SFTP for only a couple of artists, some background process must
+        mark published assets to live on multiple sites (they might be
+        physically in same location - mounted shared disk).
+
+        Process mimics OP Tray with specific 'active_site' name, all
+        configuration for this "dummy" user comes from Setting or Local
+        Settings (configured by starting OP Tray with env
+        var OPENPYPE_LOCAL_ID set to 'active_site'.
+    """
+    if debug:
+        os.environ['OPENPYPE_DEBUG'] = '3'
+    PypeCommands().syncserver(active_site)
+
+
+@main.command()
+@click.argument("directory")
+def repack_version(directory):
+    """Repack OpenPype version from directory.
+
+    This command will re-create zip file from specified directory,
+    recalculating file checksums. It will try to use version detected in
+    directory name.
+    """
+    PypeCommands().repack_version(directory)

openpype/hooks/pre_foundry_apps.py

@@ -13,7 +13,7 @@ class LaunchFoundryAppsWindows(PreLaunchHook):
 
     # Should be as last hook because must change launch arguments to string
     order = 1000
-    app_groups = ["nuke", "nukex", "hiero", "nukestudio"]
+    app_groups = ["nuke", "nukex", "hiero", "nukestudio", "aftereffects"]
     platforms = ["windows"]
 
     def execute(self):

openpype/hooks/pre_non_python_host_launch.py

@@ -49,7 +49,3 @@ def execute(self):
         if remainders:
             self.launch_context.launch_args.extend(remainders)
 
-        # This must be set otherwise it wouldn't be possible to catch output
-        #   when build OpenPype is used.
-        self.launch_context.kwargs["stdout"] = subprocess.DEVNULL
-        self.launch_context.kwargs["stderr"] = subprocess.DEVNULL

openpype/hosts/aftereffects/api/__init__.py

@@ -4,7 +4,7 @@
 
 from avalon import io
 from avalon import api as avalon
-from avalon.vendor import Qt
+from Qt import QtWidgets
 from openpype import lib, api
 import pyblish.api as pyblish
 import openpype.hosts.aftereffects
@@ -41,10 +41,10 @@ def check_inventory():
 
     # Warn about outdated containers.
     print("Starting new QApplication..")
-    app = Qt.QtWidgets.QApplication(sys.argv)
+    app = QtWidgets.QApplication(sys.argv)
 
-    message_box = Qt.QtWidgets.QMessageBox()
-    message_box.setIcon(Qt.QtWidgets.QMessageBox.Warning)
+    message_box = QtWidgets.QMessageBox()
+    message_box.setIcon(QtWidgets.QMessageBox.Warning)
     msg = "There are outdated containers in the scene."
     message_box.setText(msg)
     message_box.exec_()

openpype/hosts/aftereffects/plugins/create/create_render.py

@@ -1,5 +1,5 @@
 import openpype.api
-from avalon.vendor import Qt
+from Qt import QtWidgets
 from avalon import aftereffects
 
 import logging
@@ -56,7 +56,7 @@ def process(self):
         stub.rename_item(item.id, stub.PUBLISH_ICON + self.data["subset"])
 
     def _show_msg(self, txt):
-        msg = Qt.QtWidgets.QMessageBox()
-        msg.setIcon(Qt.QtWidgets.QMessageBox.Warning)
+        msg = QtWidgets.QMessageBox()
+        msg.setIcon(QtWidgets.QMessageBox.Warning)
         msg.setText(txt)
         msg.exec_()

openpype/hosts/aftereffects/plugins/publish/extract_local_render.py

@@ -19,9 +19,10 @@ def process(self, instance):
         staging_dir = instance.data["stagingDir"]
         self.log.info("staging_dir::{}".format(staging_dir))
 
+        stub.render(staging_dir)
+
         # pull file name from Render Queue Output module
         render_q = stub.get_render_info()
-        stub.render(staging_dir)
         if not render_q:
             raise ValueError("No file extension set in Render Queue")
         _, ext = os.path.splitext(os.path.basename(render_q.file_name))

openpype/hosts/flame/api/lib.py

@@ -47,10 +47,8 @@ def get(self, k, default=None):
         def setdefault(self, k, default=None):
             return self.master[self.name].setdefault(k, default)
 
-        def pop(self, k, v=object()):
-            if v is object():
-                return self.master[self.name].pop(k)
-            return self.master[self.name].pop(k, v)
+        def pop(self, *args, **kwargs):
+            return self.master[self.name].pop(*args, **kwargs)
 
         def update(self, mapping=(), **kwargs):
             self.master[self.name].update(mapping, **kwargs)