Merge branch 'master' into master

filmor · web-flow · commit 9bea24df237a · 2019-05-08T17:28:21.000+02:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -9,6 +9,14 @@ This document follows the conventions laid out in [Keep a CHANGELOG][].
 
 ### Added
 
+### Changed
+
+### Fixed
+
+## [2.4.0][]
+
+### Added
+
 -   Added support for embedding python into dotnet core 2.0 (NetStandard 2.0)
 -   Added new build system (pythonnet.15.sln) based on dotnetcore-sdk/xplat(crossplatform msbuild).
     Currently there two side-by-side build systems that produces the same output (net40) from the same sources.
diff --git a/README.rst b/README.rst
@@ -48,7 +48,7 @@ Embedding Python in .NET
 -  All python objects should be declared as ``dynamic`` type.
 -  Mathematical operations involving python and literal/managed types
    must have the python object first, eg. ``np.pi * 2`` works,
-   ``2 * np.pi`` doesn’t.
+   ``2 * np.pi`` doesn't.
 
 Example
 ~~~~~~~
diff --git a/setup.py b/setup.py
@@ -637,7 +637,7 @@ def run(self):
 
 setup(
     name="pythonnet",
-    version="2.4.0-rc2",
+    version="2.4.1-dev",
     description=".Net and Mono integration for Python",
     url="https://pythonnet.github.io/",
     license="MIT",
@@ -662,7 +662,6 @@ def run(self):
         "Programming Language :: Python :: 2",
         "Programming Language :: Python :: 2.7",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.4",
         "Programming Language :: Python :: 3.5",
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs
@@ -25,4 +25,4 @@
 // Version Information. Keeping it simple. May need to revisit for Nuget
 // See: https://codingforsmarties.wordpress.com/2016/01/21/how-to-version-assemblies-destined-for-nuget/
 // AssemblyVersion can only be numeric
-[assembly: AssemblyVersion("2.4.0")]
+[assembly: AssemblyVersion("2.4.1")]
diff --git a/src/clrmodule/ClrModule.cs b/src/clrmodule/ClrModule.cs
@@ -53,7 +53,7 @@ public static void initclr()
         {
 #if USE_PYTHON_RUNTIME_VERSION
             // Has no effect until SNK works. Keep updated anyways.
-            Version = new Version("2.4.0"),
+            Version = new Version("2.4.1"),
 #endif
             CultureInfo = CultureInfo.InvariantCulture
         };
diff --git a/src/clrmodule/clrmodule.15.csproj b/src/clrmodule/clrmodule.15.csproj
@@ -9,7 +9,7 @@
     <RootNamespace>clrmodule</RootNamespace>
     <AssemblyName>clrmodule</AssemblyName>
     <PackageId>clrmodule</PackageId>
-    <VersionPrefix>2.4.0</VersionPrefix>
+    <VersionPrefix>2.4.1</VersionPrefix>
     <GenerateAssemblyVersionAttribute>false</GenerateAssemblyVersionAttribute>
     <GenerateAssemblyCompanyAttribute>false</GenerateAssemblyCompanyAttribute>
     <GenerateAssemblyConfigurationAttribute>false</GenerateAssemblyConfigurationAttribute>
diff --git a/src/console/Console.15.csproj b/src/console/Console.15.csproj
@@ -8,7 +8,7 @@
     <AssemblyName>nPython</AssemblyName>
     <RootNamespace>Python.Runtime</RootNamespace>
     <PackageId>nPython</PackageId>
-    <VersionPrefix>2.4.0</VersionPrefix>
+    <VersionPrefix>2.4.1</VersionPrefix>
     <GenerateAssemblyVersionAttribute>false</GenerateAssemblyVersionAttribute>
     <GenerateAssemblyCompanyAttribute>false</GenerateAssemblyCompanyAttribute>
     <GenerateAssemblyConfigurationAttribute>false</GenerateAssemblyConfigurationAttribute>
diff --git a/src/embed_tests/Python.EmbeddingTest.15.csproj b/src/embed_tests/Python.EmbeddingTest.15.csproj
@@ -10,7 +10,7 @@
     <AssemblyName>Python.EmbeddingTest</AssemblyName>
     <RootNamespace>Python.EmbeddingTest</RootNamespace>
     <PackageId>Python.EmbeddingTest</PackageId>
-    <VersionPrefix>2.4.0</VersionPrefix>
+    <VersionPrefix>2.4.1</VersionPrefix>
     <GenerateAssemblyVersionAttribute>false</GenerateAssemblyVersionAttribute>
     <GenerateAssemblyCompanyAttribute>false</GenerateAssemblyCompanyAttribute>
     <GenerateAssemblyConfigurationAttribute>false</GenerateAssemblyConfigurationAttribute>
diff --git a/src/embed_tests/TestFinalizer.cs b/src/embed_tests/TestFinalizer.cs
@@ -36,16 +36,18 @@ public void CollectBasicObject()
         {
             Assert.IsTrue(Finalizer.Instance.Enable);
 
-            int thId = Thread.CurrentThread.ManagedThreadId;
             Finalizer.Instance.Threshold = 1;
             bool called = false;
+            var objectCount = 0;
             EventHandler<Finalizer.CollectArgs> handler = (s, e) =>
             {
-                Assert.AreEqual(thId, Thread.CurrentThread.ManagedThreadId);
-                Assert.GreaterOrEqual(e.ObjectCount, 1);
+                objectCount = e.ObjectCount;
                 called = true;
             };
 
+            Assert.IsFalse(called);
+            Finalizer.Instance.CollectOnce += handler;
+
             WeakReference shortWeak;
             WeakReference longWeak;
             {
@@ -61,18 +63,16 @@ public void CollectBasicObject()
                 Assert.NotZero(garbage.Count);
                 Assert.IsTrue(garbage.Any(T => ReferenceEquals(T.Target, longWeak.Target)));
             }
-
-            Assert.IsFalse(called);
-            Finalizer.Instance.CollectOnce += handler;
             try
             {
-                Finalizer.Instance.CallPendingFinalizers();
+                Finalizer.Instance.Collect(forceDispose: false);
             }
             finally
             {
                 Finalizer.Instance.CollectOnce -= handler;
             }
             Assert.IsTrue(called);
+            Assert.GreaterOrEqual(objectCount, 1);
         }
 
         private static void MakeAGarbage(out WeakReference shortWeak, out WeakReference longWeak)
@@ -85,7 +85,7 @@ private static void MakeAGarbage(out WeakReference shortWeak, out WeakReference
 
         private static long CompareWithFinalizerOn(PyObject pyCollect, bool enbale)
         {
-            // Must larger than 512 bytes make sure Python use 
+            // Must larger than 512 bytes make sure Python use
             string str = new string('1', 1024);
             Finalizer.Instance.Enable = true;
             FullGCCollect();
@@ -164,10 +164,11 @@ internal static void CreateMyPyObject(IntPtr op)
         public void ErrorHandling()
         {
             bool called = false;
+            var errorMessage = "";
             EventHandler<Finalizer.ErrorArgs> handleFunc = (sender, args) =>
             {
                 called = true;
-                Assert.AreEqual(args.Error.Message, "MyPyObject");
+                errorMessage = args.Error.Message;
             };
             Finalizer.Instance.Threshold = 1;
             Finalizer.Instance.ErrorHandler += handleFunc;
@@ -193,6 +194,7 @@ public void ErrorHandling()
             {
                 Finalizer.Instance.ErrorHandler -= handleFunc;
             }
+            Assert.AreEqual(errorMessage, "MyPyObject");
         }
 
         [Test]
diff --git a/src/runtime/Python.Runtime.15.csproj b/src/runtime/Python.Runtime.15.csproj
@@ -8,7 +8,7 @@
     <RootNamespace>Python.Runtime</RootNamespace>
     <AssemblyName>Python.Runtime</AssemblyName>
     <PackageId>Python.Runtime</PackageId>
-    <VersionPrefix>2.4.0</VersionPrefix>
+    <VersionPrefix>2.4.1</VersionPrefix>
     <GenerateAssemblyVersionAttribute>false</GenerateAssemblyVersionAttribute>
     <GenerateAssemblyCompanyAttribute>false</GenerateAssemblyCompanyAttribute>
     <GenerateAssemblyConfigurationAttribute>false</GenerateAssemblyConfigurationAttribute>
diff --git a/src/runtime/finalizer.cs b/src/runtime/finalizer.cs
@@ -1,10 +1,9 @@
 using System;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
-using System.Diagnostics;
 using System.Linq;
-using System.Runtime.InteropServices;
 using System.Threading;
+using System.Threading.Tasks;
 
 namespace Python.Runtime
 {
@@ -28,20 +27,10 @@ public class ErrorArgs : EventArgs
         public int Threshold { get; set; }
         public bool Enable { get; set; }
 
-        [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)]
-        struct PendingArgs
-        {
-            public bool cancelled;
-        }
-
-        [UnmanagedFunctionPointer(CallingConvention.Cdecl)]
-        private delegate int PendingCall(IntPtr arg);
-        private readonly PendingCall _collectAction;
-
         private ConcurrentQueue<IPyDisposable> _objQueue = new ConcurrentQueue<IPyDisposable>();
         private bool _pending = false;
         private readonly object _collectingLock = new object();
-        private IntPtr _pendingArgs = IntPtr.Zero;
+        private Task _finalizerTask;
 
         #region FINALIZER_CHECK
 
@@ -84,23 +73,20 @@ private Finalizer()
         {
             Enable = true;
             Threshold = 200;
-            _collectAction = OnPendingCollect;
         }
 
-        public void CallPendingFinalizers()
+        public void Collect(bool forceDispose = true)
         {
-            if (Thread.CurrentThread.ManagedThreadId != Runtime.MainManagedThreadId)
+            if (Instance._finalizerTask != null
+                && !Instance._finalizerTask.IsCompleted)
             {
-                throw new Exception("PendingCall should execute in main Python thread");
+                var ts = PythonEngine.BeginAllowThreads();
+                Instance._finalizerTask.Wait();
+                PythonEngine.EndAllowThreads(ts);
             }
-            Runtime.Py_MakePendingCalls();
-        }
-
-        public void Collect()
-        {
-            using (var gilState = new Py.GILState())
+            else if (forceDispose)
             {
-                DisposeAll();
+                Instance.DisposeAll();
             }
         }
 
@@ -141,25 +127,7 @@ internal static void Shutdown()
                 Instance._objQueue = new ConcurrentQueue<IPyDisposable>();
                 return;
             }
-            Instance.DisposeAll();
-            if (Thread.CurrentThread.ManagedThreadId != Runtime.MainManagedThreadId)
-            {
-                if (Instance._pendingArgs == IntPtr.Zero)
-                {
-                    Instance.ResetPending();
-                    return;
-                }
-                // Not in main thread just cancel the pending operation to avoid error in different domain
-                // It will make a memory leak
-                unsafe
-                {
-                    PendingArgs* args = (PendingArgs*)Instance._pendingArgs;
-                    args->cancelled = true;
-                }
-                Instance.ResetPending();
-                return;
-            }
-            Instance.CallPendingFinalizers();
+            Instance.Collect(forceDispose: true);
         }
 
         private void AddPendingCollect()
@@ -171,16 +139,17 @@ private void AddPendingCollect()
                     if (!_pending)
                     {
                         _pending = true;
-                        var args = new PendingArgs { cancelled = false };
-                        _pendingArgs = Marshal.AllocHGlobal(Marshal.SizeOf(typeof(PendingArgs)));
-                        Marshal.StructureToPtr(args, _pendingArgs, false);
-                        IntPtr func = Marshal.GetFunctionPointerForDelegate(_collectAction);
-                        if (Runtime.Py_AddPendingCall(func, _pendingArgs) != 0)
+                        // should already be complete but just in case
+                        _finalizerTask?.Wait();
+
+                        _finalizerTask = Task.Factory.StartNew(() =>
                         {
-                            // Full queue, append next time
-                            FreePendingArgs();
-                            _pending = false;
-                        }
+                            using (Py.GIL())
+                            {
+                                Instance.DisposeAll();
+                                _pending = false;
+                            }
+                        });
                     }
                 }
                 finally
@@ -190,29 +159,6 @@ private void AddPendingCollect()
             }
         }
 
-        private static int OnPendingCollect(IntPtr arg)
-        {
-            Debug.Assert(arg == Instance._pendingArgs);
-            try
-            {
-                unsafe
-                {
-                    PendingArgs* pendingArgs = (PendingArgs*)arg;
-                    if (pendingArgs->cancelled)
-                    {
-                        return 0;
-                    }
-                }
-                Instance.DisposeAll();
-            }
-            finally
-            {
-                Instance.FreePendingArgs();
-                Instance.ResetPending();
-            }
-            return 0;
-        }
-
         private void DisposeAll()
         {
             CollectOnce?.Invoke(this, new CollectArgs()
@@ -246,23 +192,6 @@ private void DisposeAll()
             }
         }
 
-        private void FreePendingArgs()
-        {
-            if (_pendingArgs != IntPtr.Zero)
-            {
-                Marshal.FreeHGlobal(_pendingArgs);
-                _pendingArgs = IntPtr.Zero;
-            }
-        }
-
-        private void ResetPending()
-        {
-            lock (_collectingLock)
-            {
-                _pending = false;
-            }
-        }
-
 #if FINALIZER_CHECK
         private void ValidateRefCount()
         {
diff --git a/src/runtime/resources/clr.py b/src/runtime/resources/clr.py
@@ -2,7 +2,7 @@
 Code in this module gets loaded into the main clr module.
 """
 
-__version__ = "2.4.0-rc2"
+__version__ = "2.4.1"
 
 
 class clrproperty(object):
diff --git a/src/testing/Python.Test.15.csproj b/src/testing/Python.Test.15.csproj
@@ -7,7 +7,7 @@
     <AssemblyName>Python.Test</AssemblyName>
     <RootNamespace>Python.Test</RootNamespace>
     <PackageId>Python.Test</PackageId>
-    <VersionPrefix>2.4.0</VersionPrefix>
+    <VersionPrefix>2.4.1</VersionPrefix>
     <OutputPath>bin\</OutputPath>
     <AppendTargetFrameworkToOutputPath Condition="'$(TargetFramework)'=='net40'">false</AppendTargetFrameworkToOutputPath>
     <DocumentationFile Condition="'$(TargetFramework)'=='net40'">$(OutputPath)\$(AssemblyName).xml</DocumentationFile>
diff --git a/src/tests/test_exceptions.py b/src/tests/test_exceptions.py
@@ -270,12 +270,6 @@ def test_str_of_exception():
     with pytest.raises(FormatException) as cm:
         Convert.ToDateTime('this will fail')
 
-    e = cm.value
-    # fix for international installation
-    msg = text_type(e).encode("utf8")
-    fnd = text_type('System.Convert.ToDateTime').encode("utf8")
-    assert msg.find(fnd) > -1, msg
-
 
 def test_python_compat_of_managed_exceptions():
     """Test managed exceptions compatible with Python's implementation"""

Original file line number	Diff line number	Diff line change
`@@ -53,7 +53,7 @@ public static void initclr()`
`53`	`53`	`{`
`54`	`54`	`#if USE_PYTHON_RUNTIME_VERSION`
`55`	`55`	`// Has no effect until SNK works. Keep updated anyways.`
`56`		`- Version = new Version("2.4.0"),`
	`56`	`+ Version = new Version("2.4.1"),`
`57`	`57`	`#endif`
`58`	`58`	`CultureInfo = CultureInfo.InvariantCulture`
`59`	`59`	`};`
Original file line number	Diff line number	Diff line change
`@@ -36,16 +36,18 @@ public void CollectBasicObject()`
`36`	`36`	`{`
`37`	`37`	`Assert.IsTrue(Finalizer.Instance.Enable);`
`38`	`38`
`39`		`- int thId = Thread.CurrentThread.ManagedThreadId;`
`40`	`39`	`Finalizer.Instance.Threshold = 1;`
`41`	`40`	`bool called = false;`
	`41`	`+ var objectCount = 0;`
`42`	`42`	`EventHandler<Finalizer.CollectArgs> handler = (s, e) =>`
`43`	`43`	`{`
`44`		`- Assert.AreEqual(thId, Thread.CurrentThread.ManagedThreadId);`
`45`		`- Assert.GreaterOrEqual(e.ObjectCount, 1);`
	`44`	`+ objectCount = e.ObjectCount;`
`46`	`45`	`called = true;`
`47`	`46`	`};`
`48`	`47`
	`48`	`+ Assert.IsFalse(called);`
	`49`	`+ Finalizer.Instance.CollectOnce += handler;`
	`50`	`+`
`49`	`51`	`WeakReference shortWeak;`
`50`	`52`	`WeakReference longWeak;`
`51`	`53`	`{`
`@@ -61,18 +63,16 @@ public void CollectBasicObject()`
`61`	`63`	`Assert.NotZero(garbage.Count);`
`62`	`64`	`Assert.IsTrue(garbage.Any(T => ReferenceEquals(T.Target, longWeak.Target)));`
`63`	`65`	`}`
`64`		`-`
`65`		`- Assert.IsFalse(called);`
`66`		`- Finalizer.Instance.CollectOnce += handler;`
`67`	`66`	`try`
`68`	`67`	`{`
`69`		`- Finalizer.Instance.CallPendingFinalizers();`
	`68`	`+ Finalizer.Instance.Collect(forceDispose: false);`
`70`	`69`	`}`
`71`	`70`	`finally`
`72`	`71`	`{`
`73`	`72`	`Finalizer.Instance.CollectOnce -= handler;`
`74`	`73`	`}`
`75`	`74`	`Assert.IsTrue(called);`
	`75`	`+ Assert.GreaterOrEqual(objectCount, 1);`
`76`	`76`	`}`
`77`	`77`
`78`	`78`	`private static void MakeAGarbage(out WeakReference shortWeak, out WeakReference longWeak)`
`@@ -85,7 +85,7 @@ private static void MakeAGarbage(out WeakReference shortWeak, out WeakReference`
`85`	`85`
`86`	`86`	`private static long CompareWithFinalizerOn(PyObject pyCollect, bool enbale)`
`87`	`87`	`{`
`88`		`- // Must larger than 512 bytes make sure Python use`
	`88`	`+ // Must larger than 512 bytes make sure Python use`
`89`	`89`	`string str = new string('1', 1024);`
`90`	`90`	`Finalizer.Instance.Enable = true;`
`91`	`91`	`FullGCCollect();`
`@@ -164,10 +164,11 @@ internal static void CreateMyPyObject(IntPtr op)`
`164`	`164`	`public void ErrorHandling()`
`165`	`165`	`{`
`166`	`166`	`bool called = false;`
	`167`	`+ var errorMessage = "";`
`167`	`168`	`EventHandler<Finalizer.ErrorArgs> handleFunc = (sender, args) =>`
`168`	`169`	`{`
`169`	`170`	`called = true;`
`170`		`- Assert.AreEqual(args.Error.Message, "MyPyObject");`
	`171`	`+ errorMessage = args.Error.Message;`
`171`	`172`	`};`
`172`	`173`	`Finalizer.Instance.Threshold = 1;`
`173`	`174`	`Finalizer.Instance.ErrorHandler += handleFunc;`
`@@ -193,6 +194,7 @@ public void ErrorHandling()`
`193`	`194`	`{`
`194`	`195`	`Finalizer.Instance.ErrorHandler -= handleFunc;`
`195`	`196`	`}`
	`197`	`+ Assert.AreEqual(errorMessage, "MyPyObject");`
`196`	`198`	`}`
`197`	`199`
`198`	`200`	`[Test]`