Make ReaderBase thread safe and remove ThreadedReader

JiayiFeng · JiayiFeng · commit 0e9f1e2790e9 · 2018-07-09T15:01:10.000+08:00
diff --git a/paddle/fluid/framework/reader.cc b/paddle/fluid/framework/reader.cc
@@ -18,18 +18,21 @@ namespace paddle {
 namespace framework {
 
 void ReaderBase::ReadNext(std::vector<LoDTensor> *out) {
+  std::lock_guard<std::mutex> lock(mu_);
   PADDLE_ENFORCE_EQ(status_, ReaderStatus::kRunning);
   ReadNextImpl(out);
 }
 
 void ReaderBase::Shutdown() {
+  std::lock_guard<std::mutex> lock(mu_);
   if (status_ != ReaderStatus::kStopped) {
     ShutdownImpl();
     status_ = ReaderStatus::kStopped;
   }
 }
 
 void ReaderBase::Start() {
+  std::lock_guard<std::mutex> lock(mu_);
   if (status_ != ReaderStatus::kRunning) {
     StartImpl();
     status_ = ReaderStatus::kRunning;
diff --git a/paddle/fluid/framework/reader.h b/paddle/fluid/framework/reader.h
@@ -44,6 +44,8 @@ class ReaderBase {
   virtual void StartImpl() = 0;
 
   ReaderStatus status_{kRunning};
+
+  mutable std::mutex mu_;
 };
 
 class DecoratedReader : public ReaderBase {
diff --git a/paddle/fluid/operators/reader/CMakeLists.txt b/paddle/fluid/operators/reader/CMakeLists.txt
@@ -22,7 +22,6 @@ reader_library(create_batch_reader_op SRCS create_batch_reader_op.cc)
 reader_library(create_recordio_file_reader_op SRCS create_recordio_file_reader_op.cc)
 reader_library(create_double_buffer_reader_op SRCS create_double_buffer_reader_op.cc)
 reader_library(create_multi_pass_reader_op SRCS create_multi_pass_reader_op.cc)
-reader_library(create_threaded_reader_op SRCS create_threaded_reader_op.cc)
 reader_library(create_custom_reader_op SRCS create_custom_reader_op.cc)
 reader_library(create_py_reader_op SRCS create_py_reader_op.cc)
 
diff --git a/paddle/fluid/operators/reader/create_threaded_reader_op.cc b/paddle/fluid/operators/reader/create_threaded_reader_op.cc
diff --git a/python/paddle/fluid/layers/io.py b/python/paddle/fluid/layers/io.py
@@ -529,9 +529,6 @@ def open_files(filenames,
         main_prog_reader = multi_pass(
             reader=main_prog_reader, pass_num=pass_num)
 
-    if for_parallel:
-        main_prog_reader = parallel(reader=main_prog_reader)
-
     return monkey_patch_reader_methods(main_prog_reader)
 
 
@@ -647,11 +644,6 @@ def multi_pass(reader, pass_num):
         'create_multi_pass_reader', reader, {'pass_num': int(pass_num)})
 
 
-def parallel(reader):
-    return __create_shared_decorated_reader__('create_threaded_reader', reader,
-                                              {})
-
-
 def read_file(reader):
     """
     Execute the given reader and get data via it.

Original file line number	Diff line number	Diff line change
`@@ -18,18 +18,21 @@ namespace paddle {`
`18`	`18`	`namespace framework {`
`19`	`19`
`20`	`20`	`void ReaderBase::ReadNext(std::vector<LoDTensor> *out) {`
	`21`	`+ std::lock_guard<std::mutex> lock(mu_);`
`21`	`22`	`PADDLE_ENFORCE_EQ(status_, ReaderStatus::kRunning);`
`22`	`23`	`ReadNextImpl(out);`
`23`	`24`	`}`
`24`	`25`
`25`	`26`	`void ReaderBase::Shutdown() {`
	`27`	`+ std::lock_guard<std::mutex> lock(mu_);`
`26`	`28`	`if (status_ != ReaderStatus::kStopped) {`
`27`	`29`	`ShutdownImpl();`
`28`	`30`	`status_ = ReaderStatus::kStopped;`
`29`	`31`	`}`
`30`	`32`	`}`
`31`	`33`
`32`	`34`	`void ReaderBase::Start() {`
	`35`	`+ std::lock_guard<std::mutex> lock(mu_);`
`33`	`36`	`if (status_ != ReaderStatus::kRunning) {`
`34`	`37`	`StartImpl();`
`35`	`38`	`status_ = ReaderStatus::kRunning;`