microsoft · JonathanZhu11 · Aug 7, 2020 · Jul 13, 2020 · Jul 13, 2020 · Jul 13, 2020
@@ -11,3 +11,17 @@ matrix:
         - ls
       before_script:
         - sudo bash "../Travis-CI/installODBC.sh"
+    - language: python
+      python: 
+        - 3.7
+      before_install:
+        - cd Python
+        - sudo bash "../Travis-CI/installODBC.sh"
+      before_script:
+        - bash ./buildandinstall.sh
+        - pip install --find-links=dist sqlmlutils
+      script: 
+        - pytest tests
+
+notifications:
+  email: false
@@ -10,7 +10,7 @@ pip install sqlmlutils
 ```
 To install from file, run:
 ```
-pip install Python/dist/sqlmlutils-1.0.1.zip
+pip install --find-links=Python/dist sqlmlutils
 ```
 
 If you are developing on your own branch and want to rebuild and install the package, you can use the buildandinstall.cmd script that is included.
@@ -231,7 +231,7 @@ pkgmanager.uninstall("astor")
 1. Make sure a SQL Server with an updated ML Services Python is running on localhost. 
 2. Restore the AirlineTestDB from the .bak file in this repo 
 3. Make sure Trusted (Windows) authentication works for connecting to the database
-4. Setup a user with db_owner role (and not server admin) with uid: "Tester" and password "FakeT3sterPwd!"
+4. Setup a user with db_owner role (and not server admin) with uid: "AirlineUser" and password "FakeT3sterPwd!"
 
 ### Notable TODOs and open issues
 

@@ -1,2 +1,3 @@
+del /q dist\*
 python.exe setup.py sdist --formats=zip
-python.exe -m pip install --upgrade --upgrade-strategy only-if-needed dist\sqlmlutils-1.0.1.zip
+python.exe -m pip install --upgrade --upgrade-strategy only-if-needed --find-links=dist sqlmlutils
@@ -0,0 +1,3 @@
+rm -f dist/*
+python setup.py sdist --formats=zip
+python -m pip install --upgrade --upgrade-strategy only-if-needed --find-links=dist sqlmlutils
@@ -3,4 +3,5 @@ pyodbc>=4.0.25
 dill>=0.2.6
 pkginfo>=1.4.2
 requirements-parser>=0.2.0
-pandas>=0.19.2
+pandas>=0.19.2
+wheel>=0.32.3
@@ -6,10 +6,10 @@
 setup(
     name='sqlmlutils',
     packages=['sqlmlutils', 'sqlmlutils/packagemanagement'],
-    version='1.0.1',
+    version='1.0.2',
     url='https://github.com/Microsoft/sqlmlutils/Python',
     license='MIT License',
-    desciption='A client side package for working with SQL Server',
+    description='A client side package for working with SQL Server',
     long_description='A client side package for working with SQL Server Machine Learning Python Services. '
                 'sqlmlutils enables easy package installation and remote code execution on your SQL Server machine.',
     author='Microsoft',
@@ -19,8 +19,9 @@
         'pyodbc',
         'dill',
         'pkginfo',
-        'requirements-parser',
-        'pandas'
+        'requirements-parser',	
+        'pandas',
+        'wheel'
     ],
     python_requires='>=3.5'
 )
@@ -3,6 +3,7 @@
 
 import operator
 
+from pkg_resources import Requirement
 from distutils.version import LooseVersion
 
 class DependencyResolver:
@@ -28,9 +29,8 @@ def get_required_installs(self, target_requirements):
         for requirement in target_requirements:
             reqmet = self._package_exists_on_server(requirement.name)
 
-            for spec in requirement.specs:
-                reqmet = reqmet & self._check_if_installed_package_meets_spec(
-                    self._server_packages, requirement.name, spec)
+            reqmet = reqmet and self._check_if_installed_package_meets_spec(
+                     self._server_packages, requirement)
 
             if not reqmet or requirement.name == self._target_package:
                 required_packages.append(self.clean_requirement_name(requirement.name))
@@ -39,26 +39,20 @@ def get_required_installs(self, target_requirements):
     def _package_exists_on_server(self, pkgname):
         return any([self.clean_requirement_name(pkgname.lower()) ==
                     self.clean_requirement_name(serverpkg[0].lower())
-                    for serverpkg in self._server_packages])
+                    for serverpkg in self._server_packages])        
 
     @staticmethod
     def clean_requirement_name(reqname: str):
         return reqname.replace("-", "_")
 
     @staticmethod
-    def _check_if_installed_package_meets_spec(package_tuples, name, spec):
-        op_str = spec[0]
-        req_version = spec[1]
-
+    def _check_if_installed_package_meets_spec(package_tuples, requirement):
         installed_package_name_and_version = [package for package in package_tuples \
-            if DependencyResolver.clean_requirement_name(name.lower()) == \
+            if DependencyResolver.clean_requirement_name(requirement.name.lower()) == \
                 DependencyResolver.clean_requirement_name(package[0].lower())]
 
         if not installed_package_name_and_version:
             return False
 
-        installed_package_name_and_version = installed_package_name_and_version[0]
-        installed_version = installed_package_name_and_version[1]
-
-        operator_map = {'>': 'gt', '>=': 'ge', '<': 'lt', '==': 'eq', '<=': 'le', '!=': 'ne'}
-        return getattr(operator, operator_map[op_str])(LooseVersion(installed_version), LooseVersion(req_version))
+        installed_version = installed_package_name_and_version[0][1]
+        return Requirement.parse(requirement.line).specifier.contains(installed_version)
@@ -10,7 +10,7 @@
 pipversion = LooseVersion(pip.__version__ )
 
 if pipversion >= LooseVersion("19.3"):
-    from pip._internal import pep425tags
+    from wheel import pep425tags
     from pip._internal.main import main as pipmain
 elif pipversion > LooseVersion("10"):
     from pip._internal import pep425tags

@@ -11,11 +11,19 @@
 uid = os.environ['USER'] if 'USER' in os.environ else ""
 pwd = os.environ['PASSWORD'] if 'PASSWORD' in os.environ else ""
 
-scope = Scope.public_scope() if uid == "" else Scope.private_scope()
+uidAirlineUser = "AirlineUserdbowner"
+pwdAirlineUser = os.environ['PASSWORD_AIRLINE_USER'] if 'PASSWORD_AIRLINE_USER' in os.environ else "FakeT3sterPwd!"
 
+scope = Scope.public_scope() if uid == "" else Scope.private_scope()
 
 connection = ConnectionInfo(driver=driver,
                             server=server,
                             database=database,
                             uid=uid,
-                            pwd=pwd)
+                            pwd=pwd)
+
+airline_user_connection = ConnectionInfo(driver=driver,
+                            server=server,
+                            database=database,
+                            uid=uidAirlineUser,
+                            pwd=pwdAirlineUser)
@@ -3,6 +3,7 @@
 
 import io
 import os
+import sys
 import subprocess
 import tempfile
 from contextlib import redirect_stdout
@@ -13,7 +14,7 @@
 from package_helper_functions import _get_sql_package_table, _get_package_names_list
 from sqlmlutils.packagemanagement.pipdownloader import PipDownloader
 
-from conftest import connection
+from conftest import connection, airline_user_connection
 
 path_to_packages = os.path.join((os.path.dirname(os.path.realpath(__file__))), "scripts", "test_packages")
 _SUCCESS_TOKEN = "SUCCESS"
@@ -24,6 +25,7 @@
 originals = _get_sql_package_table(connection)
 
 def check_package(package_name: str, exists: bool, class_to_check: str = ""):
+    """Check and assert whether a package exists, and if a class is in the module"""
     if exists:
         themodule = __import__(package_name)
         assert themodule is not None
@@ -33,25 +35,13 @@ def check_package(package_name: str, exists: bool, class_to_check: str = ""):
         with pytest.raises(Exception):
             __import__(package_name)
 
-
-def _execute_sql(script: str) -> bool:
-    tmpfile = tempfile.NamedTemporaryFile(delete=False)
-    tmpfile.write(script.encode())
-    tmpfile.close()
-    command = ["sqlcmd", "-d", "AirlineTestDB", "-i", tmpfile.name]
-    try:
-        output = subprocess.check_output(command, stderr=subprocess.STDOUT, shell=True).decode()
-        return _SUCCESS_TOKEN in output
-    finally:
-        os.remove(tmpfile.name)
-
-
 def _drop(package_name: str, ddl_name: str):
+    """Uninstall a package and check that it is gone"""
     pkgmanager.uninstall(package_name)
     pyexecutor.execute_function_in_sql(check_package, package_name=package_name, exists=False)
 
-
 def _create(module_name: str, package_file: str, class_to_check: str, drop: bool = True):
+    """Install a package and check that it is installed"""
     try:
         pyexecutor.execute_function_in_sql(check_package, package_name=module_name, exists=False)
         pkgmanager.install(package_file)
@@ -60,8 +50,8 @@ def _create(module_name: str, package_file: str, class_to_check: str, drop: bool
         if drop:
             _drop(package_name=module_name, ddl_name=module_name)
 
-
 def _remove_all_new_packages(manager):
+    """Drop all packages that were not there in the original list"""
     df = _get_sql_package_table(connection)
 
     libs = {df['external_library_id'][i]: (df['name'][i], df['scope'][i]) for i in range(len(df.index))}
@@ -83,36 +73,29 @@ def _remove_all_new_packages(manager):
                     manager.uninstall(pkg, scope=Scope.public_scope())
 
 
-packages = ["absl-py==0.1.13", "astor==0.8.1", "bleach==1.5.0",
-            "html5lib==1.0.1", "Markdown==2.6.11", "termcolor==1.1.0", "webencodings==0.5.1"]
+# Download the package zips we will use for these tests
+# 
+packages = ["astor==0.8.1", "html5lib==1.0.1", "termcolor==1.1.0"]
 
 for package in packages:
     pipdownloader = PipDownloader(connection, path_to_packages, package)
     pipdownloader.download_single()
 
 def test_install_basic_zip_package():
+    """Test a basic zip package"""
     package = os.path.join(path_to_packages, "testpackageA-0.0.1.zip")
     module_name = "testpackageA"
 
     _remove_all_new_packages(pkgmanager)
 
     _create(module_name=module_name, package_file=package, class_to_check="ClassA")
 
-
-def test_install_basic_zip_package_different_name():
-    package = os.path.join(path_to_packages, "testpackageA-0.0.1.zip")
-    module_name = "testpackageA"
-
-    _remove_all_new_packages(pkgmanager)
-
-    _create(module_name=module_name, package_file=package, class_to_check="ClassA")
-
-
 def test_install_whl_files():
-    packages = ["webencodings-0.5.1-py2.py3-none-any.whl", "html5lib-1.0.1-py2.py3-none-any.whl",
+    """Test some basic wheel files"""
+    packages = ["html5lib-1.0.1-py2.py3-none-any.whl",
                 "astor-0.8.1-py2.py3-none-any.whl"]
-    module_names = ["webencodings",  "html5lib", "astor"]
-    classes_to_check = ["LABELS",  "parse", "code_gen"]
+    module_names = ["html5lib", "astor"]
+    classes_to_check = ["parse", "code_gen"]
 
     _remove_all_new_packages(pkgmanager)
 
@@ -122,6 +105,7 @@ def test_install_whl_files():
 
 
 def test_install_targz_files():
+    """Test a basic tar.gz file"""
     packages = ["termcolor-1.1.0.tar.gz"]
     module_names = ["termcolor"]
     ddl_names = ["termcolor"]
@@ -133,9 +117,8 @@ def test_install_targz_files():
         full_package = os.path.join(path_to_packages, package)
         _create(module_name=module, package_file=full_package, class_to_check=class_to_check)
 
-
 def test_install_bad_package_badzipfile():
-
+    """Test a zip that is not a package, then make sure it is not in the external_libraries table"""
     _remove_all_new_packages(pkgmanager)
 
     with tempfile.TemporaryDirectory() as temporary_directory:
@@ -147,21 +130,12 @@ def test_install_bad_package_badzipfile():
 
         assert "badpackageA" not in _get_package_names_list(connection)
 
-        query = """
-declare @val int;
-set @val = (select count(*) from sys.external_libraries where name='badpackageA')
-if @val = 0
-    print('{}')
-""".format(_SUCCESS_TOKEN)
-
-        assert _execute_sql(query)
-
-
 def test_package_already_exists_on_sql_table():
-
+    """Test the 'upgrade' parameter in installation"""
     _remove_all_new_packages(pkgmanager)
 
     # Install a downgraded version of the package first
+    #
     package = os.path.join(path_to_packages, "testpackageA-0.0.1.zip")
     pkgmanager.install(package)
 
@@ -175,6 +149,7 @@ def check_version():
     package = os.path.join(path_to_packages, "testpackageA-0.0.2.zip")
 
     # Without upgrade
+    #
     output = io.StringIO()
     with redirect_stdout(output):
         pkgmanager.install(package, upgrade=False)
@@ -184,6 +159,7 @@ def check_version():
     assert version == "0.0.1"
 
     # With upgrade
+    #
     pkgmanager.install(package, upgrade=True)
 
     version = pyexecutor.execute_function_in_sql(check_version)
@@ -192,9 +168,8 @@ def check_version():
     pkgmanager.uninstall("testpackageA")
 
 
-# TODO: more tests for drop external library
 def test_scope():
-
+    """Test installing in a private scope with a db_owner (not dbo) user"""
     _remove_all_new_packages(pkgmanager)
 
     package = os.path.join(path_to_packages, "testpackageA-0.0.1.zip")
@@ -203,13 +178,13 @@ def get_location():
         import testpackageA
         return testpackageA.__file__
 
-    _revotesterconnection = ConnectionInfo(server="localhost",
-                                                      database="AirlineTestDB",
-                                                      uid="Tester",
-                                                      pwd="FakeT3sterPwd!")
-    revopkgmanager = SQLPackageManager(_revotesterconnection)
-    revoexecutor = SQLPythonExecutor(_revotesterconnection)
+    # The airline_user_connection is NOT dbo, so it has access to both Private and Public scopes
+    # 
+    revopkgmanager = SQLPackageManager(airline_user_connection)
+    revoexecutor = SQLPythonExecutor(airline_user_connection)
 
+    # Install a package into the private scope
+    #
     revopkgmanager.install(package, scope=Scope.private_scope())
     private_location = revoexecutor.execute_function_in_sql(get_location)
 
@@ -219,6 +194,8 @@ def get_location():
 
     revopkgmanager.uninstall(pkg_name, scope=Scope.private_scope())
 
+    # Try the same installation in public scope
+    #
     revopkgmanager.install(package, scope=Scope.public_scope())
     public_location = revoexecutor.execute_function_in_sql(get_location)
 
@@ -227,5 +204,7 @@ def get_location():
 
     revopkgmanager.uninstall(pkg_name, scope=Scope.public_scope())
 
+    # Make sure the package was removed properly
+    #
     revoexecutor.execute_function_in_sql(check_package, package_name=pkg_name, exists=False)
     pyexecutor.execute_function_in_sql(check_package, package_name=pkg_name, exists=False)