disable Manual/Tutorial tests on Windows Python2.7

Author: liqunfu

Tests/EndToEndTests/CNTKv2Python/Manual/Manual_How_to_create_user_minibatch_sources_test.py

@@ -5,6 +5,8 @@
 # ==============================================================================
 
 import os
+import sys
+import pytest
 
 abs_path = os.path.dirname(os.path.abspath(__file__))
 
@@ -14,6 +16,8 @@
 
 
 def test_manual_learners_cntk_noErrors(nb):
+    if os.getenv("OS")=="Windows_NT" and sys.version_info[0] == 2:
+        pytest.skip('tests with Python 2.7 on Windows are not stable in the CI environment. ')
     errors = [output for cell in nb.cells if 'outputs' in cell
               for output in cell['outputs'] if output.output_type == "error"]
     assert errors == []

Tests/EndToEndTests/CNTKv2Python/Manual/Manual_How_to_debug_test.py

@@ -5,6 +5,8 @@
 # ==============================================================================
 
 import os
+import sys
+import pytest
 import re
 import numpy as np
 
@@ -16,6 +18,8 @@
 #Note: Given this is a manual for debugging, we check only for functional correctness of API.
 
 def test_debugging_cntk_noErrors(nb):
+    if os.getenv("OS")=="Windows_NT" and sys.version_info[0] == 2:
+        pytest.skip('tests with Python 2.7 on Windows are not stable in the CI environment. ')
     errors = [output for cell in nb.cells if 'outputs' in cell
               for output in cell['outputs'] if output.output_type == "error"]
     assert errors == []

Tests/EndToEndTests/CNTKv2Python/Manual/Manual_How_to_feed_data_test.py

@@ -5,6 +5,8 @@
 # ==============================================================================
 
 import os
+import sys
+import pytest
 import re
 import numpy as np
 
@@ -15,6 +17,8 @@
 #Note: Given this is a manual for data reading, we check only for functional correctness of API.
 
 def test_manual_how_to_feed_data_noErrors(nb):
+    if os.getenv("OS")=="Windows_NT" and sys.version_info[0] == 2:
+        pytest.skip('tests with Python 2.7 on Windows are not stable in the CI environment. ')
     errors = [output for cell in nb.cells if 'outputs' in cell
               for output in cell['outputs'] if output.output_type == "error"]
     assert errors == []

Tests/EndToEndTests/CNTKv2Python/Manual/Manual_How_to_train_using_declarative_and_imperative_API_test.py

@@ -7,6 +7,7 @@
 import os
 import re
 import sys
+import pytest
 
 abs_path = os.path.dirname(os.path.abspath(__file__))
 sys.path.append(os.path.join(abs_path, "..", "Tutorials"))
@@ -16,12 +17,16 @@
 notebook = os.path.join(abs_path, "..", "..", "..", "..", "Manual", "Manual_How_to_train_using_declarative_and_imperative_API.ipynb")
 
 def test_cntk_how_to_train_no_errors(nb):
+    if os.getenv("OS")=="Windows_NT" and sys.version_info[0] == 2:
+        pytest.skip('tests with Python 2.7 on Windows are not stable in the CI environment. ')
     errors = [output for cell in nb.cells if 'outputs' in cell
               for output in cell['outputs'] if output.output_type == "error"]
     assert errors == []
 
 expectedOutput = r'Minibatch\[ 191- 200\]: loss = '
 def test_cntk_how_to_train_eval_correct(nb):
+    if os.getenv("OS")=="Windows_NT" and sys.version_info[0] == 2:
+        pytest.skip('tests with Python 2.7 on Windows are not stable in the CI environment. ')
     testCells = [cell for cell in nb.cells
                 if cell.cell_type == 'code']
     assert len(testCells) == 5

Tests/EndToEndTests/CNTKv2Python/Manual/Manual_How_to_use_learners_test.py

@@ -5,6 +5,8 @@
 # ==============================================================================
 
 import os
+import sys
+import pytest
 
 abs_path = os.path.dirname(os.path.abspath(__file__))
 
@@ -14,6 +16,8 @@
 
 
 def test_manual_learners_cntk_noErrors(nb):
+    if os.getenv("OS")=="Windows_NT" and sys.version_info[0] == 2:
+        pytest.skip('tests with Python 2.7 on Windows are not stable in the CI environment. ')
     errors = [output for cell in nb.cells if 'outputs' in cell
               for output in cell['outputs'] if output.output_type == "error"]
     assert errors == []

Tests/EndToEndTests/CNTKv2Python/Manual/Manual_How_to_write_a_custom_deserializer_test.py

@@ -7,6 +7,7 @@
 import os
 import re
 import sys
+import pytest
 
 abs_path = os.path.dirname(os.path.abspath(__file__))
 sys.path.append(os.path.join(abs_path, "..", "Tutorials"))
@@ -16,12 +17,16 @@
 notebook = os.path.join(abs_path, "..", "..", "..", "..", "Manual", "Manual_How_to_write_a_custom_deserializer.ipynb")
 
 def test_cntk_how_to_train_no_errors(nb):
+    if os.getenv("OS")=="Windows_NT" and sys.version_info[0] == 2:
+        pytest.skip('tests with Python 2.7 on Windows are not stable in the CI environment. ')
     errors = [output for cell in nb.cells if 'outputs' in cell
               for output in cell['outputs'] if output.output_type == "error"]
     assert errors == []
 
 expectedOutput = 'Total number of samples 200000'
 def test_csv_correct(nb):
+    if os.getenv("OS")=="Windows_NT" and sys.version_info[0] == 2:
+        pytest.skip('tests with Python 2.7 on Windows are not stable in the CI environment. ')
     testCells = [cell for cell in nb.cells
                 if cell.cell_type == 'code']
     assert len(testCells) == 6

Tests/EndToEndTests/CNTKv2Python/Manual/conftest.py

@@ -47,6 +47,11 @@ def nb(tmpdir_factory, request, device_id):
     import subprocess
     from cntk.ops.tests.ops_test_utils import cntk_device
     from cntk.cntk_py import DeviceKind_GPU
+
+    # tests with Python 2.7 on Windows are not stable in the CI environment
+    if os.getenv("OS")=="Windows_NT" and sys.version_info[0] == 2:
+        return;
+
     inPath = getattr(request.module, "notebook")
 
     deviceIdsToRun = [-1, 0]

Tests/EndToEndTests/CNTKv2Python/Tutorials/CNTK_101_LogisticRegression_test.py

@@ -5,19 +5,25 @@
 # ==============================================================================
 
 import os
+import sys
+import pytest
 import re
 
 abs_path = os.path.dirname(os.path.abspath(__file__))
 notebook = os.path.join(abs_path, "..", "..", "..", "..", "Tutorials", "CNTK_101_LogisticRegression.ipynb")
 
 def test_cntk_101_logisticregression_noErrors(nb):
+    if os.getenv("OS")=="Windows_NT" and sys.version_info[0] == 2:
+        pytest.skip('tests with Python 2.7 on Windows are not stable in the CI environment. ')
     errors = [output for cell in nb.cells if 'outputs' in cell
               for output in cell['outputs'] if output.output_type == "error"]
     assert errors == []
 
 expectedEvalError = '0.12'
 
 def test_cntk_101_logisticregression_evalCorrect(nb):
+    if os.getenv("OS")=="Windows_NT" and sys.version_info[0] == 2:
+        pytest.skip('tests with Python 2.7 on Windows are not stable in the CI environment. ')
     testCell = [cell for cell in nb.cells
                 if cell.cell_type == 'code' and re.search(r'trainer\.test_minibatch', cell.source)]
     assert len(testCell) == 1

Tests/EndToEndTests/CNTKv2Python/Tutorials/CNTK_102_FeedForward_test.py

@@ -5,19 +5,25 @@
 # ==============================================================================
 
 import os
+import sys
+import pytest
 import re
 
 abs_path = os.path.dirname(os.path.abspath(__file__))
 notebook = os.path.join(abs_path, "..", "..", "..", "..", "Tutorials", "CNTK_102_FeedForward.ipynb")
 
 def test_cntk_102_feedforward_noErrors(nb):
+    if os.getenv("OS")=="Windows_NT" and sys.version_info[0] == 2:
+        pytest.skip('tests with Python 2.7 on Windows are not stable in the CI environment. ')
     errors = [output for cell in nb.cells if 'outputs' in cell
               for output in cell['outputs'] if output.output_type == "error"]
     assert errors == []
 
 expectedEvalError = '0.12'
 
 def test_cntk_102_feedforward_evalCorrect(nb):
+    if os.getenv("OS")=="Windows_NT" and sys.version_info[0] == 2:
+        pytest.skip('tests with Python 2.7 on Windows are not stable in the CI environment. ')
     testCell = [cell for cell in nb.cells
                 if cell.cell_type == 'code' and re.search(r'trainer\.test_minibatch', cell.source)]
     assert len(testCell) == 1

Tests/EndToEndTests/CNTKv2Python/Tutorials/CNTK_103A_MNIST_DataLoader_test.py

@@ -7,6 +7,7 @@
 import os
 import re
 import numpy as np
+import sys
 import pytest
 
 abs_path = os.path.dirname(os.path.abspath(__file__))
@@ -28,6 +29,8 @@ def clean_data(device_id):
 @pytest.mark.skipif(not reWeekly.search(os.environ.get('TEST_TAG')),
                     reason="only runs as part of the weekly tests")
 def test_cntk_103a_mnist_dataloader_noErrors(clean_data, nb):
+    if os.getenv("OS")=="Windows_NT" and sys.version_info[0] == 2:
+        pytest.skip('tests with Python 2.7 on Windows are not stable in the CI environment. ')
     errors = [output for cell in nb.cells if 'outputs' in cell
               for output in cell['outputs'] if output.output_type == "error"]
     assert errors == []

Tests/EndToEndTests/CNTKv2Python/Tutorials/CNTK_103B_MNIST_LogisticRegression_test.py

@@ -5,20 +5,26 @@
 # ==============================================================================
 
 import os
+import sys
+import pytest
 import re
 import numpy as np
 
 abs_path = os.path.dirname(os.path.abspath(__file__))
 notebook = os.path.join(abs_path, "..", "..", "..", "..", "Tutorials", "CNTK_103B_MNIST_LogisticRegression.ipynb")
 
 def test_cntk_103b_mnist_logisticregression_noErrors(nb):
+    if os.getenv("OS")=="Windows_NT" and sys.version_info[0] == 2:
+        pytest.skip('tests with Python 2.7 on Windows are not stable in the CI environment. ')
     errors = [output for cell in nb.cells if 'outputs' in cell
               for output in cell['outputs'] if output.output_type == "error"]
     assert errors == []
 
 expectedEvalErrorByDeviceId = { -1: 7.4, 0: 7.4 }
 
 def test_cntk_103b_mnist_logisticregression_evalCorrect(nb, device_id):
+    if os.getenv("OS")=="Windows_NT" and sys.version_info[0] == 2:
+        pytest.skip('tests with Python 2.7 on Windows are not stable in the CI environment. ')
     testCell = [cell for cell in nb.cells
                 if cell.cell_type == 'code' and re.search(r'trainer\.test_minibatch', cell.source)]
     assert len(testCell) == 1

Tests/EndToEndTests/CNTKv2Python/Tutorials/CNTK_103C_MNIST_MultiLayerPerceptron_test.py

@@ -5,20 +5,26 @@
 # ==============================================================================
 
 import os
+import sys
+import pytest
 import re
 import numpy as np
 
 abs_path = os.path.dirname(os.path.abspath(__file__))
 notebook = os.path.join(abs_path, "..", "..", "..", "..", "Tutorials", "CNTK_103C_MNIST_MultiLayerPerceptron.ipynb")
 
 def test_cntk_103c_mnist_multilayerperceptron_noErrors(nb):
+    if os.getenv("OS")=="Windows_NT" and sys.version_info[0] == 2:
+        pytest.skip('tests with Python 2.7 on Windows are not stable in the CI environment. ')
     errors = [output for cell in nb.cells if 'outputs' in cell
               for output in cell['outputs'] if output.output_type == "error"]
     assert errors == []
 
 expectedEvalErrorByDeviceId = { -1: 1.72, 0: 1.81 }
 
 def test_cntk_103c_mnist_multilayerperceptron_evalCorrect(nb, device_id):
+    if os.getenv("OS")=="Windows_NT" and sys.version_info[0] == 2:
+        pytest.skip('tests with Python 2.7 on Windows are not stable in the CI environment. ')
     testCell = [cell for cell in nb.cells
                 if cell.cell_type == 'code' and re.search(r'trainer\.test_minibatch', cell.source)]
     assert len(testCell) == 1

Tests/EndToEndTests/CNTKv2Python/Tutorials/CNTK_103D_MNIST_ConvolutionalNeuralNetwork_test.py

@@ -5,13 +5,17 @@
 # ==============================================================================
 
 import os
+import sys
+import pytest
 import re
 import numpy as np
 
 abs_path = os.path.dirname(os.path.abspath(__file__))
 notebook = os.path.join(abs_path, "..", "..", "..", "..", "Tutorials", "CNTK_103D_MNIST_ConvolutionalNeuralNetwork.ipynb")
 
 def test_cntk_103d_mnist_convolutionalneuralnetwork_noErrors(nb):
+    if os.getenv("OS")=="Windows_NT" and sys.version_info[0] == 2:
+        pytest.skip('tests with Python 2.7 on Windows are not stable in the CI environment. ')
     errors = [output for cell in nb.cells if 'outputs' in cell
               for output in cell['outputs'] if output.output_type == "error"]
     assert errors == []
@@ -20,6 +24,8 @@ def test_cntk_103d_mnist_convolutionalneuralnetwork_noErrors(nb):
 expectedEvalErrorByDeviceId = { -1: [1.35, 1.05] , 0: [1.35, 1.05] }
 
 def test_cntk_103d_mnist_convolutionalneuralnetwork_trainerror(nb, device_id):
+    if os.getenv("OS")=="Windows_NT" and sys.version_info[0] == 2:
+        pytest.skip('tests with Python 2.7 on Windows are not stable in the CI environment. ')
     metrics = []
     for cell in nb.cells:
         try:

Tests/EndToEndTests/CNTKv2Python/Tutorials/CNTK_104_Finance_Timeseries_Basic_with_Pandas_Numpy_test.py

@@ -5,12 +5,16 @@
 # ==============================================================================
 
 import os
+import sys
+import pytest
 import re
 
 abs_path = os.path.dirname(os.path.abspath(__file__))
 notebook = os.path.join(abs_path, "..", "..", "..", "..", "Tutorials", "CNTK_104_Finance_Timeseries_Basic_with_Pandas_Numpy.ipynb")
 
 def test_cntk_104_finance_timeseries_basic_with_pandas_numpy_noErrors(nb):
+    if os.getenv("OS")=="Windows_NT" and sys.version_info[0] == 2:
+        pytest.skip('tests with Python 2.7 on Windows are not stable in the CI environment. ')
     errors = [output for cell in nb.cells if 'outputs' in cell
               for output in cell['outputs'] if output.output_type == "error"]
     print(errors)
@@ -20,6 +24,8 @@ def test_cntk_104_finance_timeseries_basic_with_pandas_numpy_noErrors(nb):
 expectedEvalErrorLower = 0.40
 
 def test_cntk_104_finance_timeseries_basic_with_pandas_numpy_trainerror(nb):
+    if os.getenv("OS")=="Windows_NT" and sys.version_info[0] == 2:
+        pytest.skip('tests with Python 2.7 on Windows are not stable in the CI environment. ')
     testCell = [cell for cell in nb.cells
                 if cell.cell_type == 'code' and re.search('# Repeatable factor ', cell.source)]
     assert (float((testCell[0].outputs[0])['text']) < expectedEvalErrorUpper) or (float((testCell[0].outputs[0])['text']) > expectedEvalErrorLower)

Tests/EndToEndTests/CNTKv2Python/Tutorials/CNTK_105_Basic_Autoencoder_for_Dimensionality_Reduction_test.py

@@ -5,6 +5,8 @@
 # ==============================================================================
 
 import os
+import sys
+import pytest
 import re
 import numpy as np
 
@@ -15,6 +17,8 @@
 TOLERANCE_ABSOLUTE = 1E-1
 
 def test_cntk_105_basic_autoencoder_for_dimensionality_reduction_noErrors(nb):
+    if os.getenv("OS")=="Windows_NT" and sys.version_info[0] == 2:
+        pytest.skip('tests with Python 2.7 on Windows are not stable in the CI environment. ')
     errors = [output for cell in nb.cells if 'outputs' in cell
               for output in cell['outputs'] if output.output_type == "error"]
     print(errors)
@@ -23,6 +27,8 @@ def test_cntk_105_basic_autoencoder_for_dimensionality_reduction_noErrors(nb):
 expectedError = 3.05
 
 def test_cntk_105_basic_autoencoder_for_dimensionality_reduction_simple_trainerror(nb):
+    if os.getenv("OS")=="Windows_NT" and sys.version_info[0] == 2:
+        pytest.skip('tests with Python 2.7 on Windows are not stable in the CI environment. ')
     testCell = [cell for cell in nb.cells
                 if cell.cell_type == 'code' and re.search('# Simple autoencoder test error', cell.source)]
     assert np.isclose(float((testCell[0].outputs[0])['text']), expectedError, atol = TOLERANCE_ABSOLUTE)

Tests/EndToEndTests/CNTKv2Python/Tutorials/CNTK_106A_LSTM_Timeseries_with_Simulated_Data_test.py

@@ -5,6 +5,8 @@
 # ==============================================================================
 
 import os
+import sys
+import pytest
 import re
 import numpy as np
 from . import _all_close_or_less
@@ -13,13 +15,17 @@
 notebook = os.path.join(abs_path, "..", "..", "..", "..", "Tutorials", "CNTK_106A_LSTM_Timeseries_with_Simulated_Data.ipynb")
 
 def test_cntk_106A_lstm_timeseries_with_simulated_data_noErrors(nb):
+    if os.getenv("OS")=="Windows_NT" and sys.version_info[0] == 2:
+        pytest.skip('tests with Python 2.7 on Windows are not stable in the CI environment. ')
     errors = [output for cell in nb.cells if 'outputs' in cell
               for output in cell['outputs'] if output.output_type == "error"]
     assert errors == []
 
 expectedEvalErrorByDeviceId = { -1: 0.000049, 0: 0.000049 }
 
 def test_cntk_106A_lstm_timeseries_with_simulated_data_evalCorrect(nb, device_id):
+    if os.getenv("OS")=="Windows_NT" and sys.version_info[0] == 2:
+        pytest.skip('tests with Python 2.7 on Windows are not stable in the CI environment. ')
     testCell = [cell for cell in nb.cells
                 if cell.cell_type == 'code' and re.search('# Print validate and test error', cell.source)]
     assert len(testCell) == 1

Tests/EndToEndTests/CNTKv2Python/Tutorials/CNTK_106B_LSTM_Timeseries_with_IOT_Data_test.py

@@ -8,6 +8,7 @@
 import re
 import numpy as np
 import sys
+import pytest
 
 abs_path = os.path.dirname(os.path.abspath(__file__))
 
@@ -19,13 +20,17 @@
 notebook_timeoutSeconds = 1800
 
 def test_cntk_106B_lstm_timeseries_with_iot_data_noErrors(nb):
+    if os.getenv("OS")=="Windows_NT" and sys.version_info[0] == 2:
+        pytest.skip('tests with Python 2.7 on Windows are not stable in the CI environment. ')
     errors = [output for cell in nb.cells if 'outputs' in cell
               for output in cell['outputs'] if output.output_type == "error"]
     assert errors == []
 
 expectedEvalErrorByDeviceId = { -1: 0.000076, 0: 0.000076 }
 
 def test_cntk_106B_lstm_timeseries_with_iot_data_evalCorrect(nb, device_id):
+    if os.getenv("OS")=="Windows_NT" and sys.version_info[0] == 2:
+        pytest.skip('tests with Python 2.7 on Windows are not stable in the CI environment. ')
     testCell = [cell for cell in nb.cells
                 if cell.cell_type == 'code' and re.search('# Print the test error', cell.source)]
     assert len(testCell) == 1