This commit is contained in:
Roberto Agostino Vitillo 2013-12-02 14:42:15 +00:00
Родитель 80ee2dba05
Коммит a91161d734
2 изменённых файлов: 10 добавлений и 4 удалений

Просмотреть файл

@ -3,8 +3,9 @@ import lzma
import sys
import json
import argparse
import telemetry.util.timer as timer
from telemetry.persist import StorageLayout
from datetime import datetime
from multiprocessing import Process, Queue, cpu_count
from pymongo import MongoClient
@ -43,7 +44,7 @@ class MongoImporter:
def _enqueue_filenames(self, input_directory):
for root, _, files in os.walk(input_directory):
for f in files:
if not f.endswith("lzma"):
if not f.endswith(StorageLayout.COMPRESSED_SUFFIX):
continue
fullpath = os.path.join(root, f)
@ -70,6 +71,8 @@ class MongoImporter:
content = f.readlines()
for line in content:
payload = json.loads(line[37:].decode("utf-8"))
# Field names cannot contain dots
# http://docs.mongodb.org/manual/reference/limits/#Restrictions%20on%20Field%20Names
self._replace_dots(payload)
payloads.append(payload)

Просмотреть файл

@ -12,7 +12,10 @@ try:
import simplejson as json
except ImportError:
import json
from telemetry_schema import TelemetrySchema
try:
from telemetry_schema import TelemetrySchema
except ImportError:
from telemetry.telemetry_schema import TelemetrySchema
import time
import logging
@ -64,7 +67,7 @@ class StorageLayout:
if not os.path.exists(dirname):
try:
os.makedirs(dirname)
except OSError, e:
except OSError as e:
# errno 17 means "directory exists". This is a race condition
# in a multi-process environment, and can safely be ignored.
if e.errno != 17: