From d8b7fca9657fda4e95337cb8ce0eaaa46c9f119e Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 7 Jul 2021 18:51:55 +0200 Subject: [PATCH] #1784 - added base implementation for helper DB class Added example of usage of helper class to test SyncServerModule (WIP) --- tests/README.md | 12 +++ tests/__init__.py | 0 tests/integration/README.md | 6 ++ tests/lib/DBHandler.py | 144 ++++++++++++++++++++++++++++++++++++ tests/lib/README.md | 1 + tests/lib/__init__.py | 0 6 files changed, 163 insertions(+) create mode 100644 tests/__init__.py create mode 100644 tests/integration/README.md create mode 100644 tests/lib/DBHandler.py create mode 100644 tests/lib/README.md create mode 100644 tests/lib/__init__.py diff --git a/tests/README.md b/tests/README.md index e69de29bb2..727b89a86e 100644 --- a/tests/README.md +++ b/tests/README.md @@ -0,0 +1,12 @@ +Automatic tests for OpenPype +============================ +Structure: +- integration - end to end tests, slow + - openpype/modules/MODULE_NAME - structure follow directory structure in code base + - fixture - sample data `(MongoDB dumps, test files etc.)` + - `tests.py` - single or more pytest files for MODULE_NAME +- unit - quick unit test + - MODULE_NAME + - fixture + - `tests.py` + diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/README.md b/tests/integration/README.md new file mode 100644 index 0000000000..00d8a4c10d --- /dev/null +++ b/tests/integration/README.md @@ -0,0 +1,6 @@ +Integration test for OpenPype +============================= +Contains end-to-end tests for automatic testing of OP. + +Should run headless publish on all hosts to check basic publish use cases automatically +to limit regression issues. diff --git a/tests/lib/DBHandler.py b/tests/lib/DBHandler.py new file mode 100644 index 0000000000..258ff67df7 --- /dev/null +++ b/tests/lib/DBHandler.py @@ -0,0 +1,144 @@ +""" + Helper class for automatic testing, provides dump and restore via command + line utilities. + + Expect mongodump and mongorestore present at MONGODB_UTILS_DIR +""" +import os +import pymongo +import subprocess + + +class DBHandler(): + + # vendorize ?? + MONGODB_UTILS_DIR = "c:\\Program Files\\MongoDB\\Server\\4.4\\bin" + + def __init__(self, uri=None, host=None, port=None, + user=None, password=None): + """'uri' or rest of separate credentials""" + if uri: + self.uri = uri + if host: + if all([user, password]): + host = "{}:{}@{}".format(user, password, host) + uri = 'mongodb://{}:{}'.format(host, port or 27017) + + assert uri, "Must have uri to MongoDB" + self.client = pymongo.MongoClient(uri) + self.db = None + + def setup_empty(self, name): + # not much sense + self.db = self.client[name] + + def setup_from_dump(self, db_name, dump_dir, overwrite=False, + collection=None, db_name_out=None): + """ + Restores 'db_name' from 'dump_dir'. + + Works with BSON folders exported by mongodump + + Args: + db_name (str): source DB name + dump_dir (str): folder with dumped subfolders + overwrite (bool): True if overwrite target + collection (str): name of source project + db_name_out (str): name of target DB, if empty restores to + source 'db_name' + """ + db_name_out = db_name_out or db_name + if self._db_exists(db_name) and not overwrite: + raise RuntimeError("DB {} already exists".format(db_name_out) + + "Run with overwrite=True") + + dir_path = os.path.join(dump_dir, db_name) + if not os.path.exists(dir_path): + raise RuntimeError( + "Backup folder {} doesn't exist".format(dir_path)) + + query = self._restore_query(self.uri, dump_dir, + db_name=db_name, db_name_out=db_name_out, + collection=collection) + print("mongorestore query:: {}".format(query)) + subprocess.run(query) + + def teardown(self, db_name): + """Drops 'db_name' if exists.""" + if not self._db_exists(db_name): + print("{} doesn't exist".format(db_name)) + return + + self.client.drop_database(db_name) + + def backup_to_dump(self, db_name, dump_dir, overwrite=False): + """ + Helper class for running mongodump for specific 'db_name' + """ + if not self._db_exists(db_name) and not overwrite: + raise RuntimeError("DB {} doesn't exists".format(db_name)) + + dir_path = os.path.join(dump_dir, db_name) + if os.path.exists(dir_path) and not overwrite: + raise RuntimeError("Backup already exists, " + "run with overwrite=True") + + query = self._dump_query(self.uri, dump_dir, db_name=db_name) + print("Mongodump query:: {}".format(query)) + subprocess.run(query) + + def _db_exists(self, db_name): + return db_name in self.client.list_database_names() + + def _dump_query(self, uri, + output_path, + db_name=None, collection=None): + + utility_path = os.path.join(self.MONGODB_UTILS_DIR, "mongodump") + + db_part = coll_part = "" + if db_name: + db_part = "--db={}".format(db_name) + if collection: + if not db_name: + raise ValueError("db_name must be present") + coll_part = "--nsInclude={}.{}".format(db_name, collection) + query = "\"{}\" --uri=\"{}\" --out={} {} {}".format( + utility_path, uri, output_path, db_part, coll_part + ) + + return query + + def _restore_query(self, uri, dump_dir, + db_name=None, db_name_out=None, + collection=None, drop=True): + + utility_path = os.path.join(self.MONGODB_UTILS_DIR, "mongorestore") + + db_part = coll_part = drop_part = "" + if db_name: + db_part = "--nsInclude={}.* --nsFrom={}.*".format(db_name, db_name) + if collection: + assert db_name, "Must provide db name too" + db_part = "--nsInclude={}.{} --nsFrom={}.{}".format(db_name, + collection, + db_name, + collection) + if drop: + drop_part = "--drop" + + if db_name_out: + db_part += " --nsTo={}.*".format(db_name_out) + + query = "\"{}\" --uri=\"{}\" --dir=\"{}\" {} {} {}".format( + utility_path, uri, dump_dir, db_part, coll_part, drop_part + ) + + return query + +# handler = DBHandler(uri="mongodb://localhost:27017") +# +# backup_dir = "c:\\projects\\dumps" +# +# handler.backup_to_dump("openpype", backup_dir, True) +# handler.setup_from_dump("test_db", backup_dir, True) diff --git a/tests/lib/README.md b/tests/lib/README.md new file mode 100644 index 0000000000..043dd3b8e9 --- /dev/null +++ b/tests/lib/README.md @@ -0,0 +1 @@ +Folder for libs and tooling for automatic testing. \ No newline at end of file diff --git a/tests/lib/__init__.py b/tests/lib/__init__.py new file mode 100644 index 0000000000..e69de29bb2