121 строка
4.7 KiB
Python
121 строка
4.7 KiB
Python
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import logging
|
|
from typing import Any, Dict, Iterator, List, Optional, Tuple
|
|
|
|
from cached_property import cached_property
|
|
|
|
from airflow.configuration import conf
|
|
from airflow.models import TaskInstance
|
|
from airflow.utils.helpers import render_log_filename
|
|
from airflow.utils.log.logging_mixin import ExternalLoggingMixin
|
|
|
|
|
|
class TaskLogReader:
|
|
"""Task log reader"""
|
|
|
|
def read_log_chunks(
|
|
self, ti: TaskInstance, try_number: Optional[int], metadata
|
|
) -> Tuple[List[str], Dict[str, Any]]:
|
|
"""
|
|
Reads chunks of Task Instance logs.
|
|
|
|
:param ti: The taskInstance
|
|
:type ti: TaskInstance
|
|
:param try_number: If provided, logs for the given try will be returned.
|
|
Otherwise, logs from all attempts are returned.
|
|
:type try_number: Optional[int]
|
|
:param metadata: A dictionary containing information about how to read the task log
|
|
:type metadata: dict
|
|
:rtype: Tuple[List[str], Dict[str, Any]]
|
|
|
|
The following is an example of how to use this method to read log:
|
|
|
|
.. code-block:: python
|
|
|
|
logs, metadata = task_log_reader.read_log_chunks(ti, try_number, metadata)
|
|
logs = logs[0] if try_number is not None else logs
|
|
|
|
where task_log_reader is an instance of TaskLogReader. The metadata will always
|
|
contain information about the task log which can enable you read logs to the
|
|
end.
|
|
"""
|
|
logs, metadatas = self.log_handler.read(ti, try_number, metadata=metadata)
|
|
metadata = metadatas[0]
|
|
return logs, metadata
|
|
|
|
def read_log_stream(self, ti: TaskInstance, try_number: Optional[int], metadata: dict) -> Iterator[str]:
|
|
"""
|
|
Used to continuously read log to the end
|
|
|
|
:param ti: The Task Instance
|
|
:type ti: TaskInstance
|
|
:param try_number: the task try number
|
|
:type try_number: Optional[int]
|
|
:param metadata: A dictionary containing information about how to read the task log
|
|
:type metadata: dict
|
|
:rtype: Iterator[str]
|
|
"""
|
|
if try_number is None:
|
|
next_try = ti.next_try_number
|
|
try_numbers = list(range(1, next_try))
|
|
else:
|
|
try_numbers = [try_number]
|
|
for current_try_number in try_numbers:
|
|
metadata.pop('end_of_log', None)
|
|
metadata.pop('max_offset', None)
|
|
metadata.pop('offset', None)
|
|
while 'end_of_log' not in metadata or not metadata['end_of_log']:
|
|
logs, metadata = self.read_log_chunks(ti, current_try_number, metadata)
|
|
for host, log in logs[0]:
|
|
yield "\n".join([host, log]) + "\n"
|
|
|
|
@cached_property
|
|
def log_handler(self):
|
|
"""Log handler, which is configured to read logs."""
|
|
logger = logging.getLogger('airflow.task')
|
|
task_log_reader = conf.get('logging', 'task_log_reader')
|
|
handler = next((handler for handler in logger.handlers if handler.name == task_log_reader), None)
|
|
return handler
|
|
|
|
@property
|
|
def supports_read(self):
|
|
"""Checks if a read operation is supported by a current log handler."""
|
|
return hasattr(self.log_handler, 'read')
|
|
|
|
@property
|
|
def supports_external_link(self):
|
|
"""Check if the logging handler supports external links (e.g. to Elasticsearch, Stackdriver, etc)."""
|
|
return isinstance(self.log_handler, ExternalLoggingMixin)
|
|
|
|
def render_log_filename(self, ti: TaskInstance, try_number: Optional[int] = None):
|
|
"""
|
|
Renders the log attachment filename
|
|
|
|
:param ti: The task instance
|
|
:type ti: TaskInstance
|
|
:param try_number: The task try number
|
|
:type try_number: Optional[int]
|
|
:rtype: str
|
|
"""
|
|
filename_template = conf.get('logging', 'LOG_FILENAME_TEMPLATE')
|
|
attachment_filename = render_log_filename(
|
|
ti=ti, try_number="all" if try_number is None else try_number, filename_template=filename_template
|
|
)
|
|
return attachment_filename
|