[AIRFLOW-5663] Switch to real-time logging in PythonVirtualenvOperator (#6389)

Co-Authored-By: Kamil Breguła <mik-laj@users.noreply.github.com>

airflow/operators/python_operator.py

@@ -330,16 +330,20 @@ class PythonVirtualenvOperator(PythonOperator):
         return len(self.op_args) + len(self.op_kwargs) > 0
 
     def _execute_in_subprocess(self, cmd):
-        try:
+        self.log.info("Executing cmd\n{}".format(cmd))
-            self.log.info("Executing cmd\n%s", cmd)
+        self._sp = subprocess.Popen(cmd,
-            output = subprocess.check_output(cmd,
+                                    stdout=subprocess.PIPE,
-                                             stderr=subprocess.STDOUT,
+                                    stderr=subprocess.STDOUT,
-                                             close_fds=True)
+                                    bufsize=0,
-            if output:
+                                    close_fds=True)
-                self.log.info("Got output\n%s", output)
+        self.log.info("Got output")
-        except subprocess.CalledProcessError as e:
+        with self._sp.stdout:
-            self.log.info("Got error output\n%s", e.output)
+            for line in iter(self._sp.stdout.readline, b''):
-            raise
+                self.log.info("%s", line.decode().rstrip())
+
+        returncode = self._sp.wait()
+        if returncode != 0:
+            raise subprocess.CalledProcessError(returncode, cmd)
 
     def _write_string_args(self, filename):
         # writes string_args to a file, which are read line by line