Testing: dump_databases flag (#5955)

* dump_databases flag

* Remove wrongly placed code.

* Turn flag into format and support json export.

* Added new argument to readme

---------

Co-authored-by: kalisp <petr.kalis@gmail.com>

openpype/cli.py

@@ -296,12 +296,15 @@ def run(script):
 @click.option("--mongo_url",
               help="MongoDB for testing.",
               default=None)
+@click.option("--dump_databases",
+              help="Dump all databases to data folder.",
+              default=None)
 def runtests(folder, mark, pyargs, test_data_folder, persist, app_variant,
-             timeout, setup_only, mongo_url, app_group):
+             timeout, setup_only, mongo_url, app_group, dump_databases):
     """Run all automatic tests after proper initialization via start.py"""
     PypeCommands().run_tests(folder, mark, pyargs, test_data_folder,
                              persist, app_variant, timeout, setup_only,
-                             mongo_url, app_group)
+                             mongo_url, app_group, dump_databases)
 
 
 @main.command(help="DEPRECATED - run sync server")

openpype/pype_commands.py

@@ -214,7 +214,7 @@ class PypeCommands:
 
     def run_tests(self, folder, mark, pyargs,
                   test_data_folder, persist, app_variant, timeout, setup_only,
-                  mongo_url, app_group):
+                  mongo_url, app_group, dump_databases):
         """
             Runs tests from 'folder'
 
@@ -275,6 +275,13 @@ class PypeCommands:
         if mongo_url:
             args.extend(["--mongo_url", mongo_url])
 
+        if dump_databases:
+            msg = "dump_databases format is not recognized: {}".format(
+                dump_databases
+            )
+            assert dump_databases in ["bson", "json"], msg
+            args.extend(["--dump_databases", dump_databases])
+
         print("run_tests args: {}".format(args))
         import pytest
         pytest.main(args)

tests/conftest.py

@@ -39,6 +39,11 @@ def pytest_addoption(parser):
         help="Provide url of the Mongo database."
     )
 
+    parser.addoption(
+        "--dump_databases", action="store", default=None,
+        help="Dump databases to data folder."
+    )
+
 
 @pytest.fixture(scope="module")
 def test_data_folder(request):
@@ -75,6 +80,11 @@ def mongo_url(request):
     return request.config.getoption("--mongo_url")
 
 
+@pytest.fixture(scope="module")
+def dump_databases(request):
+    return request.config.getoption("--dump_databases")
+
+
 @pytest.hookimpl(tryfirst=True, hookwrapper=True)
 def pytest_runtest_makereport(item, call):
     # execute all other hooks to obtain the report object

tests/integration/README.md

@@ -29,7 +29,7 @@ Command line arguments
  - "--timeout" - "Provide specific timeout value for test case",
  - "--setup_only" - "Only create dbs, do not run tests",
  - "--mongo_url" - "MongoDB for testing.",
-
+ - "--dump_databases" - ("json"|"bson") export database in expected format after successful test (to output folder in temp location - which is made persistent by this, must be cleared manually)
 Run Tray for test
 -----------------
 In case of failed test you might want to run it manually and visually debug what happened.

tests/lib/db_handler.py

@@ -2,7 +2,7 @@
     Helper class for automatic testing, provides dump and restore via command
     line utilities.
 
-    Expect mongodump, mongoimport and mongorestore present at PATH
+    Expect mongodump, mongoexport, mongoimport and mongorestore present at PATH
 """
 import os
 import pymongo
@@ -148,7 +148,7 @@ class DBHandler:
         self.client.drop_database(db_name)
 
     def backup_to_dump(self, db_name, dump_dir, overwrite=False,
-                       collection=None):
+                       collection=None, format="bson"):
         """
             Helper method for running mongodump for specific 'db_name'
         """
@@ -160,15 +160,24 @@ class DBHandler:
             raise RuntimeError("Backup already exists, "
                                "run with overwrite=True")
 
-        query = self._dump_query(self.uri, dump_dir,
+        collections = [collection]
-                                 db_name=db_name, collection=collection)
+        if format == "json" and collection is None:
-        print("Mongodump query:: {}".format(query))
+            collections = self.client[db_name].list_collection_names()
-        subprocess.run(query)
+
+        for collection in collections:
+            query = self._dump_query(self.uri, dump_dir,
+                                     db_name=db_name, collection=collection,
+                                     format=format)
+            print("Mongodump query:: {}".format(query))
+            process = subprocess.run(query)
+            assert process.returncode == 0, "Mongo dump failed."
 
     def _db_exists(self, db_name):
         return db_name in self.client.list_database_names()
 
-    def _dump_query(self, uri, output_path, db_name=None, collection=None):
+    def _dump_query(
+        self, uri, output_path, db_name=None, collection=None, format="bson"
+    ):
         """Prepares dump query based on 'db_name' or 'collection'."""
         db_part = coll_part = ""
         if db_name:
@@ -177,11 +186,22 @@ class DBHandler:
             if not db_name:
                 raise ValueError("db_name must be present")
             coll_part = "--collection={}".format(collection)
-        query = "\"{}\" --uri=\"{}\" --out={} {} {}".format(
-            "mongodump", uri, output_path, db_part, coll_part
-        )
 
-        return query
+        tool = "mongodump"
+        query = "{} --uri=\"{}\""
+
+        if format == "json":
+            assert collection, "Collection is needed for json export."
+
+            query += " --jsonArray --pretty"
+            tool = "mongoexport"
+            output_path = os.path.join(
+                output_path, "{}.{}.json".format(db_name, collection)
+            )
+
+        query += " --out={} {} {}"
+
+        return query.format(tool, uri, output_path, db_part, coll_part)
 
     def _restore_query(self, uri, dump_dir,
                        db_name=None, db_name_out=None,

tests/lib/testing_classes.py

@@ -70,7 +70,9 @@ class ModuleUnitTest(BaseTest):
         )
 
     @pytest.fixture(scope="module")
-    def download_test_data(self, test_data_folder, persist, request):
+    def download_test_data(
+        self, test_data_folder, persist, request, dump_databases
+    ):
         test_data_folder = test_data_folder or self.TEST_DATA_FOLDER
         if test_data_folder:
             print("Using existing folder {}".format(test_data_folder))
@@ -100,13 +102,13 @@ class ModuleUnitTest(BaseTest):
                 if ext and ext.lstrip('.') in handler_class.IMPLEMENTED_ZIP_FORMATS:  # noqa: E501
                     handler_class.unzip(os.path.join(tmpdir, file_name))
 
-                yield tmpdir
+            yield tmpdir
 
-                persist = (persist or self.PERSIST or
+            persist = (persist or self.PERSIST or
-                           self.is_test_failed(request))
+                       self.is_test_failed(request) or dump_databases)
-                if not persist:
+            if not persist:
-                    print("Removing {}".format(tmpdir))
+                print("Removing {}".format(tmpdir))
-                    shutil.rmtree(tmpdir)
+                shutil.rmtree(tmpdir)
 
     @pytest.fixture(scope="module")
     def output_folder_url(self, download_test_data):
@@ -163,7 +165,7 @@ class ModuleUnitTest(BaseTest):
 
     @pytest.fixture(scope="module")
     def db_setup(self, download_test_data, env_var, monkeypatch_session,
-                 request, mongo_url):
+                 request, mongo_url, dump_databases, persist):
         """Restore prepared MongoDB dumps into selected DB."""
         backup_dir = os.path.join(download_test_data, "input", "dumps")
         uri = os.environ.get("OPENPYPE_MONGO")
@@ -178,7 +180,17 @@ class ModuleUnitTest(BaseTest):
 
         yield db_handler
 
-        persist = self.PERSIST or self.is_test_failed(request)
+        if dump_databases:
+            print("Dumping databases to {}".format(download_test_data))
+            output_dir = os.path.join(download_test_data, "output", "dumps")
+            db_handler.backup_to_dump(
+                self.TEST_DB_NAME, output_dir, format=dump_databases
+            )
+            db_handler.backup_to_dump(
+                self.TEST_OPENPYPE_NAME, output_dir, format=dump_databases
+            )
+
+        persist = persist or self.PERSIST or self.is_test_failed(request)
         if not persist:
             db_handler.teardown(self.TEST_DB_NAME)
             db_handler.teardown(self.TEST_OPENPYPE_NAME)