mozilla
/
addons-server
зеркало из https://github.com/mozilla/addons-server.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

make setting up logs explicit and allow us to break it down and pull the bits we want into services (bug 725828)

This commit is contained in:
Andy McKay 2012-02-09 16:46:00 -08:00
Родитель f0a62f99ec
Коммит f2324c6148
7 изменённых файлов: 170 добавлений и 142 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

181
lib/log_settings_base.py
Просмотреть файл

@ -11,101 +11,108 @@ base_fmt = ('%(name)s:%(levelname)s %(message)s '
error_fmt = ('%(name)s:%(levelname)s %(request_path)s %(message)s '
':%(pathname)s:%(lineno)s')
formatters = {
'debug': {
'()': commonware.log.Formatter,
'datefmt': '%H:%M:%S',
'format': '%(asctime)s ' + base_fmt,
},
'prod': {
'()': commonware.log.Formatter,
'datefmt': '%H:%M:%S',
'format': ('%s: [%%(USERNAME)s][%%(REMOTE_ADDR)s] %s'
% (settings.SYSLOG_TAG, base_fmt)),
},
'prod2': {
'()': commonware.log.Formatter,
'datefmt': '%H:%M:%S',
'format': ('%s: [%%(USERNAME)s][%%(REMOTE_ADDR)s] %s'
% (settings.SYSLOG_TAG2, base_fmt)),
},
'error': {
'()': commonware.log.Formatter,
'datefmt': '%H:%M:%S',
'format': ('%s: [%%(USERNAME)s][%%(REMOTE_ADDR)s] %s'
% (settings.SYSLOG_TAG, error_fmt)),
},
}
handlers = {
'console': {
'()': logging.StreamHandler,
'formatter': 'debug',
},
'syslog': {
'class': 'lib.misc.unicode_log.UnicodeHandler',
'facility': logging.handlers.SysLogHandler.LOG_LOCAL7,
'formatter': 'prod',
},
'syslog2': {
'class': 'lib.misc.unicode_log.UnicodeHandler',
'facility': logging.handlers.SysLogHandler.LOG_LOCAL7,
'formatter': 'prod2',
},
'null': {
'class': 'lib.misc.admin_log.NullHandler',
},
'mail_admins': {
'level': 'ERROR',
'class': 'lib.misc.admin_log.AdminEmailHandler'
},
'statsd': {
'level': 'ERROR',
'class': 'lib.misc.admin_log.StatsdHandler',
},
'arecibo': {
'level': 'ERROR',
'class': 'lib.misc.admin_log.AreciboHandler',
},
'errortype_syslog': {
'class': 'lib.misc.admin_log.ErrorSyslogHandler',
'facility': logging.handlers.SysLogHandler.LOG_LOCAL7,
'formatter': 'error',
},
}
loggers = {
'z': {},
'django.request': {
# Note these handlers will choose what they want to emit and when.
'handlers': ['mail_admins', 'errortype_syslog',
'statsd', 'arecibo'],
'level': 'ERROR',
'propagate': True,
},
}
cfg = {
'version': 1,
'filters': {},
'formatters': {
'debug': {
'()': commonware.log.Formatter,
'datefmt': '%H:%M:%S',
'format': '%(asctime)s ' + base_fmt,
},
'prod': {
'()': commonware.log.Formatter,
'datefmt': '%H:%M:%S',
'format': ('%s: [%%(USERNAME)s][%%(REMOTE_ADDR)s] %s'
% (settings.SYSLOG_TAG, base_fmt)),
},
'prod2': {
'()': commonware.log.Formatter,
'datefmt': '%H:%M:%S',
'format': ('%s: [%%(USERNAME)s][%%(REMOTE_ADDR)s] %s'
% (settings.SYSLOG_TAG2, base_fmt)),
},
'error': {
'()': commonware.log.Formatter,
'datefmt': '%H:%M:%S',
'format': ('%s: [%%(USERNAME)s][%%(REMOTE_ADDR)s] %s'
% (settings.SYSLOG_TAG, error_fmt)),
},
},
'handlers': {
'console': {
'()': logging.StreamHandler,
'formatter': 'debug',
},
'syslog': {
'class': 'lib.misc.admin_log.UnicodeHandler',
'facility': logging.handlers.SysLogHandler.LOG_LOCAL7,
'formatter': 'prod',
},
'syslog2': {
'class': 'lib.misc.admin_log.UnicodeHandler',
'facility': logging.handlers.SysLogHandler.LOG_LOCAL7,
'formatter': 'prod2',
},
'null': {
'class': 'lib.misc.admin_log.NullHandler',
},
'mail_admins': {
'level': 'ERROR',
'class': 'lib.misc.admin_log.AdminEmailHandler'
},
# 'statsd': {
# 'level': 'ERROR',
# 'class': 'lib.misc.admin_log.StatsdHandler',
# },
# 'arecibo': {
# 'level': 'ERROR',
# 'class': 'lib.misc.admin_log.AreciboHandler',
# },
'errortype_syslog': {
'class': 'lib.misc.admin_log.ErrorSyslogHandler',
'facility': logging.handlers.SysLogHandler.LOG_LOCAL7,
'formatter': 'error',
},
},
'loggers': {
'z': {},
'django.request': {
# Note these handlers will choose what they want to emit and when.
'handlers': ['mail_admins', 'errortype_syslog'],#,
# 'statsd', 'arecibo'],
'level': 'ERROR',
'propagate': True,
},
},
'formatters': formatters,
'handlers': handlers,
'loggers': loggers,
'root': {},
}
for key, value in settings.LOGGING.items():
cfg[key].update(value)
def log_configure():
"""You have to explicitly call this to configure logging."""
for key, value in settings.LOGGING.items():
cfg[key].update(value)
USE_SYSLOG = settings.HAS_SYSLOG and not settings.DEBUG
USE_SYSLOG = settings.HAS_SYSLOG and not settings.DEBUG
if USE_SYSLOG:
cfg['loggers']['z.timer'] = {'handlers': ['syslog2']}
# Set the level and handlers for all loggers.
for logger in cfg['loggers'].values() + [cfg['root']]:
if 'handlers' not in logger:
logger['handlers'] = ['syslog' if USE_SYSLOG else 'console']
if 'level' not in logger:
logger['level'] = settings.LOG_LEVEL
if logger is not cfg['root'] and 'propagate' not in logger:
logger['propagate'] = False
if USE_SYSLOG:
cfg['loggers']['z.timer'] = {'handlers': ['syslog2']}
# Set the level and handlers for all loggers.
for logger in cfg['loggers'].values() + [cfg['root']]:
if 'handlers' not in logger:
logger['handlers'] = ['syslog' if USE_SYSLOG else 'console']
if 'level' not in logger:
logger['level'] = settings.LOG_LEVEL
if logger is not cfg['root'] and 'propagate' not in logger:
logger['propagate'] = False
dictconfig.dictConfig(cfg)
dictconfig.dictConfig(cfg)

73
lib/misc/admin_log.py
Просмотреть файл

@ -20,15 +20,14 @@ NO_EMAIL_PATTERNS = {
}
import logging
import socket
import traceback
from django.conf import settings
from django.core import mail
#from django_arecibo.tasks import post
#from django_statsd.clients import statsd
from django_arecibo.tasks import post
from django_statsd.clients import statsd
from unicode_log import UnicodeHandler
getLogger = logging.getLogger
@ -46,30 +45,6 @@ if not logger.handlers:
logger.addHandler(NullHandler())
class UnicodeHandler(logging.handlers.SysLogHandler):
def emit(self, record):
msg = self.format(record) + '\000'
prio = '<%d>' % self.encodePriority(self.facility,
self.mapPriority(record.levelname))
if type(msg) is unicode:
msg = msg.encode('utf-8')
msg = prio + msg
try:
if self.unixsocket:
try:
self.socket.send(msg)
except socket.error:
self._connect_unixsocket(self.address)
self.socket.send(msg)
else:
self.socket.sendto(msg, self.address)
except (KeyboardInterrupt, SystemExit):
raise
except:
self.handleError(record)
class ErrorTypeHandler(logging.Handler):
"""A base class for a logging handler that examines the error."""
@ -98,27 +73,27 @@ class ErrorTypeHandler(logging.Handler):
pass
#class StatsdHandler(ErrorTypeHandler):
# """Send error to statsd, we'll send this every time."""
#
# def emit(self, record):
# if not record.exc_info:
# return
#
# statsd.incr('error.%s' % record.exc_info[0].__name__.lower())
# self.emitted(self.__class__.__name__.lower())
#
#
#class AreciboHandler(ErrorTypeHandler):
# """Send error to Arecibo, only if we are also emailing it."""
#
# def emit(self, record):
# arecibo = getattr(settings, 'ARECIBO_SERVER_URL', '')
# if not self.should_email(record) or not arecibo:
# return
#
# post(record.request, 500)
# self.emitted(self.__class__.__name__.lower())
class StatsdHandler(ErrorTypeHandler):
"""Send error to statsd, we'll send this every time."""
def emit(self, record):
if not record.exc_info:
return
statsd.incr('error.%s' % record.exc_info[0].__name__.lower())
self.emitted(self.__class__.__name__.lower())
class AreciboHandler(ErrorTypeHandler):
"""Send error to Arecibo, only if we are also emailing it."""
def emit(self, record):
arecibo = getattr(settings, 'ARECIBO_SERVER_URL', '')
if not self.should_email(record) or not arecibo:
return
post(record.request, 500)
self.emitted(self.__class__.__name__.lower())
class ErrorSyslogHandler(UnicodeHandler, ErrorTypeHandler):

26
lib/misc/unicode_log.py Normal file
Просмотреть файл

@ -0,0 +1,26 @@
import logging.handlers
import socket
class UnicodeHandler(logging.handlers.SysLogHandler):
def emit(self, record):
msg = self.format(record) + '\000'
prio = '<%d>' % self.encodePriority(self.facility,
self.mapPriority(record.levelname))
if type(msg) is unicode:
msg = msg.encode('utf-8')
msg = prio + msg
try:
if self.unixsocket:
try:
self.socket.send(msg)
except socket.error:
self._connect_unixsocket(self.address)
self.socket.send(msg)
else:
self.socket.sendto(msg, self.address)
except (KeyboardInterrupt, SystemExit):
raise
except:
self.handleError(record)

3
manage.py
Просмотреть файл

@ -100,7 +100,8 @@ djcelery.setup_loader()
# Import for side-effect: configures our logging handlers.
# pylint: disable-msg=W0611
from lib import log_settings_base
from lib.log_settings_base import log_configure
log_configure()
from lib.misc import safe_signals
safe_signals.start_the_machine()

6
services/update.py
Просмотреть файл

@ -15,7 +15,6 @@ from django.utils.http import urlencode
import settings_local as settings
setup_environ(settings)
from lib import log_settings_base
# This has to be imported after the settings so statsd knows where to log to.
from statsd import statsd
@ -25,8 +24,11 @@ except ImportError:
from apps.versions.compare import version_int
from constants import base
from utils import get_mirror, APP_GUIDS, PLATFORMS, STATUSES_PUBLIC
from utils import (get_mirror, log_configure, APP_GUIDS, PLATFORMS,
STATUSES_PUBLIC)
# Go configure the log.
log_configure()
good_rdf = """<?xml version="1.0"?>
<RDF:RDF xmlns:RDF="http://www.w3.org/1999/02/22-rdf-syntax-ns#"

18
services/utils.py
Просмотреть файл

@ -1,4 +1,6 @@
from datetime import datetime, timedelta
import dictconfig
import logging
import settings_local as settings
import posixpath
import re
@ -13,7 +15,7 @@ import commonware.log
# Pyflakes will complain about these, but they are required for setup.
import settings_local as settings
setup_environ(settings)
from lib import log_settings_base
from lib.log_settings_base import formatters, handlers, loggers
# Ugh. But this avoids any zamboni or django imports at all.
# Perhaps we can import these without any problems and we can
@ -83,13 +85,25 @@ def getconn():
mypool = pool.QueuePool(getconn, max_overflow=10, pool_size=5, recycle=300)
error_log = commonware.log.getLogger('z.services')
def log_configure():
"""You have to call this to explicity configure logging."""
cfg = {
'version': 1,
'filters': {},
'formatters': dict(prod2=formatters['prod2']),
'handlers': dict(syslog2=handlers['syslog2']),
'loggers': {},
'root': {},
}
dictconfig.dictConfig(cfg)
def log_exception(data):
(typ, value, discard) = sys.exc_info()
error_log = logging.getLogger('z.services')
error_log.error(u'Type: %s, %s. Data: %s' % (typ, value, data))
def log_info(data, msg):
error_log = logging.getLogger('z.services')
error_log.info(u'Msg: %s, Data: %s' % (msg, data))

5
services/verify.py
Просмотреть файл

@ -4,9 +4,12 @@ import re
from time import time
from urlparse import parse_qsl
from utils import (log_exception, log_info, mypool, settings,
from utils import (log_configure, log_exception, log_info, mypool, settings,
CONTRIB_CHARGEBACK, CONTRIB_PURCHASE, CONTRIB_REFUND)
# Go configure the log.
log_configure()
import jwt
import M2Crypto
# This has to be imported after the settings (utils).