зеркало из https://github.com/microsoft/AirSim.git
Code merge for Technion https://github.com/Microsoft/AirSim/pull/1410
This commit is contained in:
Shital Shah
Родитель
5f7893099b
Коммит
a57da5204e
|
|
@ -0,0 +1,225 @@
|
|||
import random
|
||||
import csv
|
||||
from PIL import Image
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import sys
|
||||
import os
|
||||
import errno
|
||||
from collections import OrderedDict
|
||||
import h5py
|
||||
from pathlib import Path
|
||||
import copy
|
||||
import re
|
||||
|
||||
|
||||
# This constant is used as an upper bound for normalizing the car's speed to be between 0 and 1
|
||||
MAX_SPEED = 70.0
|
||||
|
||||
|
||||
|
||||
def checkAndCreateDir(full_path):
|
||||
"""Checks if a given path exists and if not, creates the needed directories.
|
||||
Inputs:
|
||||
full_path: path to be checked
|
||||
"""
|
||||
if not os.path.exists(os.path.dirname(full_path)):
|
||||
try:
|
||||
os.makedirs(os.path.dirname(full_path))
|
||||
except OSError as exc: # Guard against race condition
|
||||
if exc.errno != errno.EEXIST:
|
||||
raise
|
||||
|
||||
def readImagesFromPath(image_names):
|
||||
""" Takes in a path and a list of image file names to be loaded and returns a list of all loaded images after resize.
|
||||
Inputs:
|
||||
image_names: list of image names
|
||||
Returns:
|
||||
List of all loaded and resized images
|
||||
"""
|
||||
returnValue = []
|
||||
for image_name in image_names:
|
||||
im = Image.open(image_name)
|
||||
imArr = np.asarray(im)
|
||||
|
||||
#Remove alpha channel if exists
|
||||
if len(imArr.shape) == 3 and imArr.shape[2] == 4:
|
||||
if (np.all(imArr[:, :, 3] == imArr[0, 0, 3])):
|
||||
imArr = imArr[:,:,0:3]
|
||||
if len(imArr.shape) != 3 or imArr.shape[2] != 3:
|
||||
print('Error: Image', image_name, 'is not RGB.')
|
||||
sys.exit()
|
||||
|
||||
returnIm = np.asarray(imArr)
|
||||
|
||||
returnValue.append(returnIm)
|
||||
return returnValue
|
||||
|
||||
|
||||
|
||||
def splitTrainValidationAndTestData(all_data_mappings, split_ratio=(0.7, 0.2, 0.1)):
|
||||
"""Simple function to create train, validation and test splits on the data.
|
||||
Inputs:
|
||||
all_data_mappings: mappings from the entire dataset
|
||||
split_ratio: (train, validation, test) split ratio
|
||||
|
||||
Returns:
|
||||
train_data_mappings: mappings for training data
|
||||
validation_data_mappings: mappings for validation data
|
||||
test_data_mappings: mappings for test data
|
||||
|
||||
"""
|
||||
if round(sum(split_ratio), 5) != 1.0:
|
||||
print("Error: Your splitting ratio should add up to 1")
|
||||
sys.exit()
|
||||
|
||||
train_split = int(len(all_data_mappings) * split_ratio[0])
|
||||
val_split = train_split + int(len(all_data_mappings) * split_ratio[1])
|
||||
|
||||
train_data_mappings = all_data_mappings[0:train_split]
|
||||
validation_data_mappings = all_data_mappings[train_split:val_split]
|
||||
test_data_mappings = all_data_mappings[val_split:]
|
||||
|
||||
return [train_data_mappings, validation_data_mappings, test_data_mappings]
|
||||
|
||||
def generateDataMapAirSim(folders):
|
||||
""" Data map generator for simulator(AirSim) data. Reads the driving_log csv file and returns a list of 'center camera image name - label(s)' tuples
|
||||
Inputs:
|
||||
folders: list of folders to collect data from
|
||||
|
||||
Returns:
|
||||
mappings: All data mappings as a dictionary. Key is the image filepath, the values are a 2-tuple:
|
||||
0 -> label(s) as a list of double
|
||||
1 -> previous state as a list of double
|
||||
"""
|
||||
|
||||
all_mappings = {}
|
||||
for folder in folders:
|
||||
print('Reading data from {0}...'.format(folder))
|
||||
current_df = pd.read_csv(os.path.join(folder, 'airsim_rec.txt'), sep='\t')
|
||||
|
||||
for i in range(1, current_df.shape[0] - 1):
|
||||
|
||||
|
||||
if current_df.iloc[i-1]['Brake'] != 0: # Consider only training examples without breaks
|
||||
continue
|
||||
|
||||
|
||||
norm_steering = [ (float(current_df.iloc[i-1][['Steering']]) + 1) / 2.0 ] # Normalize steering: between 0 and 1
|
||||
norm_throttle = [ float(current_df.iloc[i-1][['Throttle']]) ]
|
||||
norm_speed = [ float(current_df.iloc[i-1][['Speed (kmph)']]) / MAX_SPEED ] # Normalize speed: between 0 and 1
|
||||
|
||||
|
||||
previous_state = norm_steering + norm_throttle + norm_speed # Append lists
|
||||
|
||||
|
||||
#compute average steering over 3 consecutive recorded images, this will serve as the label
|
||||
|
||||
norm_steering0 = (float(current_df.iloc[i][['Steering']]) + 1) / 2.0
|
||||
norm_steering1 = (float(current_df.iloc[i+1][['Steering']]) + 1) / 2.0
|
||||
|
||||
temp_sum_steering = norm_steering[0] + norm_steering0 + norm_steering1
|
||||
average_steering = temp_sum_steering / 3.0
|
||||
|
||||
current_label = [average_steering]
|
||||
|
||||
image_filepath = os.path.join(os.path.join(folder, 'images'), current_df.iloc[i]['ImageName']).replace('\\', '/')
|
||||
|
||||
if (image_filepath in all_mappings):
|
||||
print('Error: attempting to add image {0} twice.'.format(image_filepath))
|
||||
|
||||
all_mappings[image_filepath] = (current_label, previous_state)
|
||||
|
||||
mappings = [(key, all_mappings[key]) for key in all_mappings]
|
||||
|
||||
random.shuffle(mappings)
|
||||
|
||||
return mappings
|
||||
|
||||
|
||||
def generatorForH5py(data_mappings, chunk_size=32):
|
||||
"""
|
||||
This function batches the data for saving to the H5 file
|
||||
"""
|
||||
for chunk_id in range(0, len(data_mappings), chunk_size):
|
||||
# Data is expected to be a dict of <image: (label, previousious_state)>
|
||||
data_chunk = data_mappings[chunk_id:chunk_id + chunk_size]
|
||||
if (len(data_chunk) == chunk_size):
|
||||
image_names_chunk = [a for (a, b) in data_chunk]
|
||||
labels_chunk = np.asarray([b[0] for (a, b) in data_chunk])
|
||||
previous_state_chunk = np.asarray([b[1] for (a, b) in data_chunk])
|
||||
|
||||
#Flatten and yield as tuple
|
||||
yield (image_names_chunk, labels_chunk.astype(float), previous_state_chunk.astype(float))
|
||||
if chunk_id + chunk_size > len(data_mappings):
|
||||
raise StopIteration
|
||||
raise StopIteration
|
||||
|
||||
|
||||
def saveH5pyData(data_mappings, target_file_path, chunk_size):
|
||||
"""
|
||||
Saves H5 data to file
|
||||
"""
|
||||
gen = generatorForH5py(data_mappings,chunk_size)
|
||||
|
||||
image_names_chunk, labels_chunk, previous_state_chunk = next(gen)
|
||||
images_chunk = np.asarray(readImagesFromPath(image_names_chunk))
|
||||
row_count = images_chunk.shape[0]
|
||||
|
||||
checkAndCreateDir(target_file_path)
|
||||
with h5py.File(target_file_path, 'w') as f:
|
||||
|
||||
# Initialize a resizable dataset to hold the output
|
||||
images_chunk_maxshape = (None,) + images_chunk.shape[1:]
|
||||
labels_chunk_maxshape = (None,) + labels_chunk.shape[1:]
|
||||
previous_state_maxshape = (None,) + previous_state_chunk.shape[1:]
|
||||
|
||||
dset_images = f.create_dataset('image', shape=images_chunk.shape, maxshape=images_chunk_maxshape,
|
||||
chunks=images_chunk.shape, dtype=images_chunk.dtype)
|
||||
|
||||
dset_labels = f.create_dataset('label', shape=labels_chunk.shape, maxshape=labels_chunk_maxshape,
|
||||
chunks=labels_chunk.shape, dtype=labels_chunk.dtype)
|
||||
|
||||
dset_previous_state = f.create_dataset('previous_state', shape=previous_state_chunk.shape, maxshape=previous_state_maxshape,
|
||||
chunks=previous_state_chunk.shape, dtype=previous_state_chunk.dtype)
|
||||
|
||||
dset_images[:] = images_chunk
|
||||
dset_labels[:] = labels_chunk
|
||||
dset_previous_state[:] = previous_state_chunk
|
||||
|
||||
for image_names_chunk, label_chunk, previous_state_chunk in gen:
|
||||
image_chunk = np.asarray(readImagesFromPath(image_names_chunk))
|
||||
|
||||
# Resize the dataset to accommodate the next chunk of rows
|
||||
dset_images.resize(row_count + image_chunk.shape[0], axis=0)
|
||||
dset_labels.resize(row_count + label_chunk.shape[0], axis=0)
|
||||
dset_previous_state.resize(row_count + previous_state_chunk.shape[0], axis=0)
|
||||
# Create the next chunk
|
||||
dset_images[row_count:] = image_chunk
|
||||
dset_labels[row_count:] = label_chunk
|
||||
dset_previous_state[row_count:] = previous_state_chunk
|
||||
|
||||
# Increment the row count
|
||||
row_count += image_chunk.shape[0]
|
||||
|
||||
|
||||
def cook(folders, output_directory, train_eval_test_split, chunk_size):
|
||||
""" Primary function for data pre-processing. Reads and saves all data as h5 files.
|
||||
Inputs:
|
||||
folders: a list of all data folders
|
||||
output_directory: location for saving h5 files
|
||||
train_eval_test_split: dataset split ratio
|
||||
"""
|
||||
output_files = [os.path.join(output_directory, f) for f in ['train.h5', 'eval.h5', 'test.h5']]
|
||||
if (any([os.path.isfile(f) for f in output_files])):
|
||||
print("Preprocessed data already exists at: {0}. Skipping preprocessing.".format(output_directory))
|
||||
|
||||
else:
|
||||
all_data_mappings = generateDataMapAirSim(folders)
|
||||
|
||||
split_mappings = splitTrainValidationAndTestData(all_data_mappings, split_ratio=train_eval_test_split)
|
||||
|
||||
for i in range(0, len(split_mappings)-1, 1):
|
||||
print('Processing {0}...'.format(output_files[i]))
|
||||
saveH5pyData(split_mappings[i], output_files[i], chunk_size)
|
||||
print('Finished saving {0}.'.format(output_files[i]))
|
||||
|
|
@ -0,0 +1,342 @@
|
|||
from keras.preprocessing import image
|
||||
import numpy as np
|
||||
import keras.backend as K
|
||||
import os
|
||||
import cv2
|
||||
import PIL
|
||||
from PIL import Image
|
||||
from PIL import ImageChops
|
||||
import cv2
|
||||
|
||||
|
||||
class DriveDataGenerator(image.ImageDataGenerator):
|
||||
def __init__(self,
|
||||
featurewise_center=False,
|
||||
samplewise_center=False,
|
||||
featurewise_std_normalization=False,
|
||||
samplewise_std_normalization=False,
|
||||
zca_whitening=False,
|
||||
zca_epsilon=1e-6,
|
||||
rotation_range=0.,
|
||||
width_shift_range=0.,
|
||||
height_shift_range=0.,
|
||||
shear_range=0.,
|
||||
zoom_range=0.,
|
||||
channel_shift_range=0.,
|
||||
fill_mode='nearest',
|
||||
cval=0.,
|
||||
horizontal_flip=False,
|
||||
vertical_flip=False,
|
||||
rescale=None,
|
||||
preprocessing_function=None,
|
||||
data_format=None,
|
||||
brighten_range=0):
|
||||
super(DriveDataGenerator, self).__init__(featurewise_center,
|
||||
samplewise_center,
|
||||
featurewise_std_normalization,
|
||||
samplewise_std_normalization,
|
||||
zca_whitening,
|
||||
zca_epsilon,
|
||||
rotation_range,
|
||||
width_shift_range,
|
||||
height_shift_range,
|
||||
shear_range,
|
||||
zoom_range,
|
||||
channel_shift_range,
|
||||
fill_mode,
|
||||
cval,
|
||||
horizontal_flip,
|
||||
vertical_flip,
|
||||
rescale,
|
||||
preprocessing_function,
|
||||
data_format)
|
||||
self.brighten_range = brighten_range
|
||||
|
||||
def flow(self, x_images, x_prev_states = None, y=None, batch_size=32, shuffle=True, seed=None,
|
||||
save_to_dir=None, save_prefix='', save_format='png', zero_drop_percentage=0.5, roi=None):
|
||||
return DriveIterator(
|
||||
x_images, x_prev_states, y, self,
|
||||
batch_size=batch_size,
|
||||
shuffle=shuffle,
|
||||
seed=seed,
|
||||
data_format=self.data_format,
|
||||
save_to_dir=save_to_dir,
|
||||
save_prefix=save_prefix,
|
||||
save_format=save_format,
|
||||
zero_drop_percentage=zero_drop_percentage,
|
||||
roi=roi)
|
||||
|
||||
def random_transform_with_states(self, x, seed=None):
|
||||
"""Randomly augment a single image tensor.
|
||||
# Arguments
|
||||
x: 3D tensor, single image.
|
||||
seed: random seed.
|
||||
# Returns
|
||||
A tuple. 0 -> randomly transformed version of the input (same shape). 1 -> true if image was horizontally flipped, false otherwise
|
||||
"""
|
||||
img_row_axis = self.row_axis
|
||||
img_col_axis = self.col_axis
|
||||
img_channel_axis = self.channel_axis
|
||||
|
||||
is_image_horizontally_flipped = False
|
||||
|
||||
# use composition of homographies
|
||||
# to generate final transform that needs to be applied
|
||||
if self.rotation_range:
|
||||
theta = np.pi / 180 * np.random.uniform(-self.rotation_range, self.rotation_range)
|
||||
else:
|
||||
theta = 0
|
||||
|
||||
if self.height_shift_range:
|
||||
tx = np.random.uniform(-self.height_shift_range, self.height_shift_range) * x.shape[img_row_axis]
|
||||
else:
|
||||
tx = 0
|
||||
|
||||
if self.width_shift_range:
|
||||
ty = np.random.uniform(-self.width_shift_range, self.width_shift_range) * x.shape[img_col_axis]
|
||||
else:
|
||||
ty = 0
|
||||
|
||||
if self.shear_range:
|
||||
shear = np.random.uniform(-self.shear_range, self.shear_range)
|
||||
else:
|
||||
shear = 0
|
||||
|
||||
if self.zoom_range[0] == 1 and self.zoom_range[1] == 1:
|
||||
zx, zy = 1, 1
|
||||
else:
|
||||
zx, zy = np.random.uniform(self.zoom_range[0], self.zoom_range[1], 2)
|
||||
|
||||
transform_matrix = None
|
||||
if theta != 0:
|
||||
rotation_matrix = np.array([[np.cos(theta), -np.sin(theta), 0],
|
||||
[np.sin(theta), np.cos(theta), 0],
|
||||
[0, 0, 1]])
|
||||
transform_matrix = rotation_matrix
|
||||
|
||||
if tx != 0 or ty != 0:
|
||||
shift_matrix = np.array([[1, 0, tx],
|
||||
[0, 1, ty],
|
||||
[0, 0, 1]])
|
||||
transform_matrix = shift_matrix if transform_matrix is None else np.dot(transform_matrix, shift_matrix)
|
||||
|
||||
if shear != 0:
|
||||
shear_matrix = np.array([[1, -np.sin(shear), 0],
|
||||
[0, np.cos(shear), 0],
|
||||
[0, 0, 1]])
|
||||
transform_matrix = shear_matrix if transform_matrix is None else np.dot(transform_matrix, shear_matrix)
|
||||
|
||||
if zx != 1 or zy != 1:
|
||||
zoom_matrix = np.array([[zx, 0, 0],
|
||||
[0, zy, 0],
|
||||
[0, 0, 1]])
|
||||
transform_matrix = zoom_matrix if transform_matrix is None else np.dot(transform_matrix, zoom_matrix)
|
||||
|
||||
if transform_matrix is not None:
|
||||
h, w = x.shape[img_row_axis], x.shape[img_col_axis]
|
||||
transform_matrix = image.transform_matrix_offset_center(transform_matrix, h, w)
|
||||
x = image.apply_transform(x, transform_matrix, img_channel_axis,
|
||||
fill_mode=self.fill_mode, cval=self.cval)
|
||||
|
||||
if self.channel_shift_range != 0:
|
||||
x = image.random_channel_shift(x,
|
||||
self.channel_shift_range,
|
||||
img_channel_axis)
|
||||
if self.horizontal_flip:
|
||||
if np.random.random() < 0.5:
|
||||
x = image.flip_axis(x, img_col_axis)
|
||||
is_image_horizontally_flipped = True
|
||||
|
||||
if self.vertical_flip:
|
||||
if np.random.random() < 0.5:
|
||||
x = image.flip_axis(x, img_row_axis)
|
||||
|
||||
if self.brighten_range != 0:
|
||||
random_bright = np.random.uniform(low = 1.0-self.brighten_range, high=1.0+self.brighten_range)
|
||||
|
||||
img = cv2.cvtColor(x, cv2.COLOR_RGB2HSV)
|
||||
img[:, :, 2] = np.clip(img[:, :, 2] * random_bright, 0, 255)
|
||||
x = cv2.cvtColor(img, cv2.COLOR_HSV2RGB)
|
||||
|
||||
return (x, is_image_horizontally_flipped)
|
||||
|
||||
|
||||
|
||||
class DriveIterator(image.Iterator):
|
||||
"""Iterator yielding data from a Numpy array.
|
||||
|
||||
# Arguments
|
||||
x: Numpy array of input data.
|
||||
y: Numpy array of targets data.
|
||||
image_data_generator: Instance of `ImageDataGenerator`
|
||||
to use for random transformations and normalization.
|
||||
batch_size: Integer, size of a batch.
|
||||
shuffle: Boolean, whether to shuffle the data between epochs.
|
||||
seed: Random seed for data shuffling.
|
||||
data_format: String, one of `channels_first`, `channels_last`.
|
||||
save_to_dir: Optional directory where to save the pictures
|
||||
being yielded, in a viewable format. This is useful
|
||||
for visualizing the random transformations being
|
||||
applied, for debugging purposes.
|
||||
save_prefix: String prefix to use for saving sample
|
||||
images (if `save_to_dir` is set).
|
||||
save_format: Format to use for saving sample images
|
||||
(if `save_to_dir` is set).
|
||||
"""
|
||||
|
||||
def __init__(self, x_images, x_prev_states, y, image_data_generator,
|
||||
batch_size=32, shuffle=False, seed=None,
|
||||
data_format=None,
|
||||
save_to_dir=None, save_prefix='', save_format='png', zero_drop_percentage = 0.5, roi = None):
|
||||
if y is not None and len(x_images) != len(y):
|
||||
raise ValueError('X (images tensor) and y (labels) '
|
||||
'should have the same length. '
|
||||
'Found: X.shape = %s, y.shape = %s' %
|
||||
(np.asarray(x_images).shape, np.asarray(y).shape))
|
||||
|
||||
if data_format is None:
|
||||
data_format = K.image_data_format()
|
||||
|
||||
self.x_images = x_images
|
||||
|
||||
self.zero_drop_percentage = zero_drop_percentage
|
||||
self.roi = roi
|
||||
|
||||
if self.x_images.ndim != 4:
|
||||
raise ValueError('Input data in `NumpyArrayIterator` '
|
||||
'should ave rank 4. You passed an array '
|
||||
'with shape', self.x_images.shape)
|
||||
channels_axis = 3 if data_format == 'channels_last' else 1
|
||||
if self.x_images.shape[channels_axis] not in {1, 3, 4}:
|
||||
raise ValueError('NumpyArrayIterator is set to use the '
|
||||
'data format convention "' + data_format + '" '
|
||||
'(channels on axis ' + str(channels_axis) + '), i.e. expected '
|
||||
'either 1, 3 or 4 channels on axis ' + str(channels_axis) + '. '
|
||||
'However, it was passed an array with shape ' + str(self.x_images.shape) +
|
||||
' (' + str(self.x_images.shape[channels_axis]) + ' channels).')
|
||||
if x_prev_states is not None:
|
||||
self.x_prev_states = x_prev_states
|
||||
else:
|
||||
self.x_prev_states = None
|
||||
|
||||
if y is not None:
|
||||
self.y = y
|
||||
else:
|
||||
self.y = None
|
||||
self.image_data_generator = image_data_generator
|
||||
self.data_format = data_format
|
||||
self.save_to_dir = save_to_dir
|
||||
self.save_prefix = save_prefix
|
||||
self.save_format = save_format
|
||||
self.batch_size = batch_size
|
||||
super(DriveIterator, self).__init__(x_images.shape[0], batch_size, shuffle, seed)
|
||||
|
||||
def next(self):
|
||||
"""For python 2.x.
|
||||
|
||||
# Returns
|
||||
The next batch.
|
||||
"""
|
||||
# Keeps under lock only the mechanism which advances
|
||||
# the indexing of each batch.
|
||||
with self.lock:
|
||||
index_array = next(self.index_generator)
|
||||
# The transformation of images is not under thread lock
|
||||
# so it can be done in parallel
|
||||
|
||||
return self.__get_indexes(index_array)
|
||||
|
||||
def __get_indexes(self, index_array):
|
||||
index_array = sorted(index_array)
|
||||
if self.x_prev_states is not None:
|
||||
batch_x_images = np.zeros(tuple([self.batch_size]+ list(self.x_images.shape)[1:]),
|
||||
dtype=K.floatx())
|
||||
batch_x_prev_states = np.zeros(tuple([self.batch_size]+list(self.x_prev_states.shape)[1:]), dtype=K.floatx())
|
||||
else:
|
||||
batch_x_images = np.zeros(tuple([self.batch_size] + list(self.x_images.shape)[1:]), dtype=K.floatx())
|
||||
|
||||
if self.roi is not None:
|
||||
batch_x_images = batch_x_images[:, self.roi[0]:self.roi[1], self.roi[2]:self.roi[3], :]
|
||||
|
||||
used_indexes = []
|
||||
is_horiz_flipped = []
|
||||
for i, j in enumerate(index_array):
|
||||
x_images = self.x_images[j]
|
||||
|
||||
if self.roi is not None:
|
||||
x_images = x_images[self.roi[0]:self.roi[1], self.roi[2]:self.roi[3], :]
|
||||
|
||||
|
||||
transformed = self.image_data_generator.random_transform_with_states(x_images.astype(K.floatx()))
|
||||
x_images = transformed[0]
|
||||
is_horiz_flipped.append(transformed[1])
|
||||
x_images = self.image_data_generator.standardize(x_images)
|
||||
batch_x_images[i] = x_images
|
||||
|
||||
if self.x_prev_states is not None:
|
||||
x_prev_states = self.x_prev_states[j]
|
||||
|
||||
if (transformed[1]):
|
||||
x_prev_states[0] *= -1.0
|
||||
|
||||
batch_x_prev_states[i] = x_prev_states
|
||||
|
||||
used_indexes.append(j)
|
||||
|
||||
if self.x_prev_states is not None:
|
||||
batch_x = [np.asarray(batch_x_images)]
|
||||
else:
|
||||
batch_x = np.asarray(batch_x_images)
|
||||
|
||||
if self.save_to_dir:
|
||||
for i in range(0, self.batch_size, 1):
|
||||
hash = np.random.randint(1e4)
|
||||
|
||||
img = image.array_to_img(batch_x_images[i], self.data_format, scale=True)
|
||||
fname = '{prefix}_{index}_{hash}.{format}'.format(prefix=self.save_prefix,
|
||||
index=1,
|
||||
hash=hash,
|
||||
format=self.save_format)
|
||||
img.save(os.path.join(self.save_to_dir, fname))
|
||||
|
||||
batch_y = self.y[list(sorted(used_indexes))]
|
||||
idx = []
|
||||
num_of_close_samples = 0
|
||||
num_of_non_close_samples = 0
|
||||
for i in range(0, len(is_horiz_flipped), 1):
|
||||
if batch_y.shape[1] == 1:
|
||||
|
||||
if (is_horiz_flipped[i]):
|
||||
batch_y[i] *= -1
|
||||
|
||||
if (np.isclose(batch_y[i], 0.5, rtol=0.005, atol=0.005)):
|
||||
num_of_close_samples += 1
|
||||
if (np.random.uniform(low=0, high=1) > self.zero_drop_percentage):
|
||||
idx.append(True)
|
||||
else:
|
||||
idx.append(False)
|
||||
else:
|
||||
num_of_non_close_samples += 1
|
||||
idx.append(True)
|
||||
else:
|
||||
|
||||
if (batch_y[i][int(len(batch_y[i])/2)] == 1):
|
||||
if (np.random.uniform(low=0, high=1) > self.zero_drop_percentage):
|
||||
idx.append(True)
|
||||
else:
|
||||
idx.append(False)
|
||||
else:
|
||||
idx.append(True)
|
||||
|
||||
if (is_horiz_flipped[i]):
|
||||
batch_y[i] = batch_y[i][::-1]
|
||||
|
||||
batch_y = batch_y[idx]
|
||||
batch_x[0] = batch_x[0][idx]
|
||||
|
||||
|
||||
return batch_x, batch_y
|
||||
|
||||
def _get_batches_of_transformed_samples(self, index_array):
|
||||
return self.__get_indexes(index_array)
|
||||
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
# Imitation Learning
|
||||
|
||||
This section is about training a model to steer our Formula car using imitation learning.
|
||||
The code in this section is based on the [Autonomous Driving Cookbook](https://github.com/Microsoft/AutonomousDrivingCookbook/tree/master/AirSimE2EDeepLearning) from Airsim and it's highly recommended to read the tutorial first.
|
||||
|
||||
## Prerequisites
|
||||
* Operating system: Windows 10
|
||||
* GPU: Nvidia GTX 1080 or higher (recommended)
|
||||
* Software: Unreal Engine 4.18 and Visual Studio 2017 (see [upgrade instructions](../../docs/unreal_upgrade.md))
|
||||
* Development: CUDA 9.0 and python 3.5.
|
||||
* Python libraries: Keras 2.1.2, TensorFlow 1.6.0.
|
||||
* Note: Newer versions of keras or tensorflow are recommended but can cause syntax errors.
|
||||
|
||||
## What's inside
|
||||
|
||||
|
||||
*Driving in simulation using trained imitation learning model, based on recorded data*
|
||||
|
||||
Imitation learning includes the usage of labeled data as input to a training algorithm with the purpose of having the algorithm imitate the actions of people who recorded the data.
|
||||
|
||||
|
||||
|
||||
This diagram is represented by these files:
|
||||
|
||||
**cook_data.py**
|
||||
This file is responsible for preparing .h5 dataset files for the training procedure.
|
||||
The code rely on having two adjacent folders:
|
||||
'raw_data' - contains folders of recorded data by airsim's recording method.
|
||||
'cooked_data' - empty folder to store the .h5 files.
|
||||
|
||||
The flag "COOK_ALL_DATA" gives the option to choose all subfolders, or exclude some of them.
|
||||
|
||||
**train_model.py**
|
||||
This file is responsible to train a model using the .h5 dataset files.
|
||||
The code rely on having two adjacent folders:
|
||||
'cooked_data' - contains the .h5 dataset files.
|
||||
'models' - empty folder to store the generated models.
|
||||
|
||||
The file will preprocess the data, add augmentations and create a neural network model that predicts the next steering angle.
|
||||
|
||||
**drive_model.py**
|
||||
This file connects to the simulation in order to upload a trained model and drive using it.
|
||||
By using the predicted steering value, the code calculates related control parameters and maintain driving with steady velocities.
|
||||
|
||||
## Training Tips
|
||||
We recommend on using augmentation and recording techniques.
|
||||
We give here an example for two methods:
|
||||
- [CycleLight](../../docs/graphic_features.md) - Animation of a day light cycle in a changeable, potentially very short period of time.
|
||||
- Shifted images - Altering the camera’s position to the right or the left of the car, so that it can record images in extreme conditions. To simulate driving back to the center from those extreme situations, post-process the recorded angle of the steering accordingly (manually).
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
#%matplotlib inline
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import h5py
|
||||
from matplotlib import use
|
||||
use("TkAgg")
|
||||
import matplotlib.pyplot as plt
|
||||
from PIL import Image, ImageDraw
|
||||
import os
|
||||
import Cooking
|
||||
|
||||
# chunk size for training batches
|
||||
chunk_size = 32
|
||||
|
||||
# No test set needed, since testing in our case is running the model on an unseen map in AirSim
|
||||
train_eval_test_split = [0.8, 0.2, 0.0]
|
||||
|
||||
# Point this to the directory containing the raw data
|
||||
RAW_DATA_DIR = './raw_data/'
|
||||
|
||||
# Point this to the desired output directory for the cooked (.h5) data
|
||||
COOKED_DATA_DIR = './cooked_data/'
|
||||
|
||||
# Choose The folders to search for data under RAW_DATA_DIR
|
||||
COOK_ALL_DATA = True
|
||||
|
||||
data_folders = []
|
||||
|
||||
#if COOK_ALL_DATA is set to False, append your desired data folders here
|
||||
# data_folder.append('folder_name1')
|
||||
# data_folder.append('folder_name2')
|
||||
# ...
|
||||
if COOK_ALL_DATA:
|
||||
data_folders = [name for name in os.listdir(RAW_DATA_DIR)]
|
||||
|
||||
|
||||
full_path_raw_folders = [os.path.join(RAW_DATA_DIR, f) for f in data_folders]
|
||||
Cooking.cook(full_path_raw_folders, COOKED_DATA_DIR, train_eval_test_split, chunk_size)
|
||||
|
|
@ -0,0 +1,76 @@
|
|||
import os
|
||||
os.environ['TF_CPP_MIN_LOG_LEVEL']='2'
|
||||
import tensorflow as tf
|
||||
from keras.models import load_model
|
||||
import sys
|
||||
import time
|
||||
import numpy as np
|
||||
|
||||
import airsim
|
||||
|
||||
import keras.backend as K
|
||||
from keras.preprocessing import image
|
||||
from PIL import Image, ImageDraw
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
# Trained model path
|
||||
MODEL_PATH = './models/example_model.h5'
|
||||
|
||||
model = load_model(MODEL_PATH)
|
||||
|
||||
# Connect to AirSim
|
||||
client = airsim.CarClient()
|
||||
client.confirmConnection()
|
||||
client.enableApiControl(True)
|
||||
car_controls = airsim.CarControls()
|
||||
|
||||
# Start driving
|
||||
car_controls.steering = 0
|
||||
car_controls.throttle = 0
|
||||
car_controls.brake = 0
|
||||
client.setCarControls(car_controls)
|
||||
|
||||
# Initialize image buffer
|
||||
image_buf = np.zeros((1, 66, 200, 3))
|
||||
|
||||
def get_image():
|
||||
"""
|
||||
Get image from AirSim client
|
||||
"""
|
||||
image_response = client.simGetImages([airsim.ImageRequest("0", airsim.ImageType.Scene, False, False)])[0]
|
||||
image1d = np.fromstring(image_response.image_data_uint8, dtype=np.uint8)
|
||||
image_rgba = image1d.reshape(image_response.height, image_response.width, 4)
|
||||
return image_rgba[78:144,27:227,0:3].astype(float)
|
||||
|
||||
while True:
|
||||
# Update throttle value according to steering angle
|
||||
if abs(car_controls.steering) <= 1.0:
|
||||
car_controls.throttle = 0.8-(0.4*abs(car_controls.steering))
|
||||
else:
|
||||
car_controls.throttle = 0.4
|
||||
|
||||
image_buf[0] = get_image()
|
||||
image_buf[0] /= 255 # Normalization
|
||||
|
||||
start_time = time.time()
|
||||
|
||||
# Prediction
|
||||
model_output = model.predict([image_buf])
|
||||
|
||||
end_time = time.time()
|
||||
received_output = model_output[0][0]
|
||||
|
||||
# Rescale prediction to [-1,1] and factor by 0.82 for drive smoothness
|
||||
car_controls.steering = round((0.82*(float((model_output[0][0]*2.0)-1))), 2)
|
||||
|
||||
# Print progress
|
||||
print('Sending steering = {0}, throttle = {1}, prediction time = {2}'.format(received_output, car_controls.throttle,str(end_time-start_time)))
|
||||
|
||||
# Update next car state
|
||||
client.setCarControls(car_controls)
|
||||
|
||||
# Wait a bit between iterations
|
||||
time.sleep(0.05)
|
||||
|
||||
|
||||
client.enableApiControl(False)
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
# Import this module to automatically setup path to local airsim module
|
||||
# This module first tries to see if airsim module is installed via pip
|
||||
# If it does then we don't do anything else
|
||||
# Else we look up grand-parent folder to see if it has airsim folder
|
||||
# and if it does then we add that in sys.path
|
||||
|
||||
import os,sys,inspect,logging
|
||||
|
||||
#this class simply tries to see if airsim
|
||||
class SetupPath:
|
||||
@staticmethod
|
||||
def getDirLevels(path):
|
||||
path_norm = os.path.normpath(path)
|
||||
return len(path_norm.split(os.sep))
|
||||
|
||||
@staticmethod
|
||||
def getCurrentPath():
|
||||
cur_filepath = os.path.abspath(inspect.getfile(inspect.currentframe()))
|
||||
return os.path.dirname(cur_filepath)
|
||||
|
||||
@staticmethod
|
||||
def getGrandParentDir():
|
||||
cur_path = SetupPath.getCurrentPath()
|
||||
if SetupPath.getDirLevels(cur_path) >= 2:
|
||||
return os.path.dirname(os.path.dirname(cur_path))
|
||||
return ''
|
||||
|
||||
@staticmethod
|
||||
def getParentDir():
|
||||
cur_path = SetupPath.getCurrentPath()
|
||||
if SetupPath.getDirLevels(cur_path) >= 1:
|
||||
return os.path.dirname(cur_path)
|
||||
return ''
|
||||
|
||||
@staticmethod
|
||||
def addAirSimModulePath():
|
||||
# if airsim module is installed then don't do anything else
|
||||
#import pkgutil
|
||||
#airsim_loader = pkgutil.find_loader('airsim')
|
||||
#if airsim_loader is not None:
|
||||
# return
|
||||
|
||||
parent = SetupPath.getParentDir()
|
||||
if parent != '':
|
||||
airsim_path = os.path.join(parent, 'airsim')
|
||||
client_path = os.path.join(airsim_path, 'client.py')
|
||||
if os.path.exists(client_path):
|
||||
sys.path.insert(0, parent)
|
||||
else:
|
||||
logging.warning("airsim module not found in parent folder. Using installed package (pip install airsim).")
|
||||
|
||||
SetupPath.addAirSimModulePath()
|
||||
|
|
@ -0,0 +1,94 @@
|
|||
import os
|
||||
os.environ['TF_CPP_MIN_LOG_LEVEL']='2'
|
||||
import tensorflow as tf
|
||||
from keras.preprocessing.image import ImageDataGenerator
|
||||
from keras.models import Sequential, Model
|
||||
from keras.layers.convolutional import Convolution2D
|
||||
from keras.layers import Conv2D, MaxPooling2D, Dropout, Flatten, Dense, Lambda, Input, concatenate
|
||||
from keras.layers.core import Activation
|
||||
from keras.layers.normalization import BatchNormalization
|
||||
from keras.layers.advanced_activations import ELU, LeakyReLU
|
||||
from keras.optimizers import Adam, SGD, Adamax, Nadam
|
||||
from keras.callbacks import ReduceLROnPlateau, ModelCheckpoint, CSVLogger, EarlyStopping
|
||||
import keras.backend as K
|
||||
from keras.preprocessing import image
|
||||
from keras_tqdm import TQDMNotebookCallback
|
||||
import json
|
||||
import os
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from Generator import DriveDataGenerator
|
||||
import h5py
|
||||
import math
|
||||
|
||||
# Hyper-parameters
|
||||
batch_size = 32
|
||||
learning_rate = 0.0001
|
||||
number_of_epochs = 500
|
||||
|
||||
# Activation functions
|
||||
activation = 'relu'
|
||||
out_activation = 'sigmoid'
|
||||
|
||||
#Stop training if in the last 20 epochs, there was no change of the best recorded validation loss
|
||||
training_patience = 20
|
||||
|
||||
|
||||
# << The directory containing the cooked data from the previous step >>
|
||||
COOKED_DATA_DIR = './cooked_data/'
|
||||
|
||||
# << The directory in which the model output will be placed >>
|
||||
MODEL_OUTPUT_DIR = './models/'
|
||||
|
||||
train_dataset = h5py.File(os.path.join(COOKED_DATA_DIR, 'train.h5'), 'r')
|
||||
eval_dataset = h5py.File(os.path.join(COOKED_DATA_DIR, 'eval.h5'), 'r')
|
||||
|
||||
num_train_examples = train_dataset['image'].shape[0]
|
||||
num_eval_examples = eval_dataset['image'].shape[0]
|
||||
|
||||
# Use ROI of [78,144,27,227] for FOV 60 with Formula car
|
||||
data_generator = DriveDataGenerator(rescale=1./255., horizontal_flip=False, brighten_range=0.4)
|
||||
train_generator = data_generator.flow\
|
||||
(train_dataset['image'], train_dataset['previous_state'], train_dataset['label'], batch_size=batch_size, zero_drop_percentage=0.95, roi=[78,144,27,227])
|
||||
eval_generator = data_generator.flow\
|
||||
(eval_dataset['image'], eval_dataset['previous_state'], eval_dataset['label'], batch_size=batch_size, zero_drop_percentage=0.95, roi=[78,144,27,227])
|
||||
|
||||
[sample_batch_train_data, sample_batch_test_data] = next(train_generator)
|
||||
|
||||
image_input_shape = sample_batch_train_data[0].shape[1:]
|
||||
|
||||
pic_input = Input(shape=image_input_shape)
|
||||
|
||||
#Network definition
|
||||
img_stack = Conv2D(24, (5, 5), name="conv1", strides=(2, 2), padding="valid", activation=activation, kernel_initializer="he_normal")(pic_input)
|
||||
img_stack = Conv2D(36, (5, 5), name="conv2", strides=(2, 2), padding="valid", activation=activation, kernel_initializer="he_normal")(img_stack)
|
||||
img_stack = Conv2D(48, (5, 5), name="conv3", strides=(2, 2), padding="valid", activation=activation, kernel_initializer="he_normal")(img_stack)
|
||||
|
||||
img_stack = Dropout(0.5)(img_stack)
|
||||
|
||||
img_stack = Conv2D(64, (3, 3), name="conv4", strides=(1, 1), padding="valid", activation=activation, kernel_initializer="he_normal")(img_stack)
|
||||
img_stack = Conv2D(64, (3, 3), name="conv5", strides=(1, 1), padding="valid", activation=activation, kernel_initializer="he_normal")(img_stack)
|
||||
|
||||
img_stack = Flatten(name = 'flatten')(img_stack)
|
||||
|
||||
img_stack = Dense(100, name="fc2", activation=activation,kernel_initializer="he_normal")(img_stack)
|
||||
img_stack = Dense(50, name="fc3", activation=activation, kernel_initializer="he_normal")(img_stack)
|
||||
img_stack = Dense(10, name="fc4", activation=activation, kernel_initializer="he_normal")(img_stack)
|
||||
img_stack = Dense(1, name="output", activation = out_activation, kernel_initializer="he_normal")(img_stack)
|
||||
|
||||
adam = Adam(lr=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0)
|
||||
|
||||
model = Model(inputs=[pic_input], outputs=img_stack)
|
||||
model.compile(optimizer=adam, loss='mse')
|
||||
|
||||
model.summary()
|
||||
|
||||
plateau_callback = ReduceLROnPlateau(monitor="val_loss", factor=0.5, patience=3, min_lr=learning_rate, verbose=1)
|
||||
csv_callback = CSVLogger(os.path.join(MODEL_OUTPUT_DIR, 'training_log.csv'))
|
||||
checkpoint_filepath = os.path.join(MODEL_OUTPUT_DIR, 'fresh_models', '{0}_model.{1}-{2}.h5'.format('model', '{epoch:02d}', '{val_loss:.7f}'))
|
||||
checkpoint_callback = ModelCheckpoint(checkpoint_filepath, save_best_only=True, verbose=1)
|
||||
early_stopping_callback = EarlyStopping(monitor="val_loss", patience=training_patience, verbose=1)
|
||||
callbacks=[plateau_callback, csv_callback, checkpoint_callback, early_stopping_callback, TQDMNotebookCallback()]
|
||||
|
||||
history = model.fit_generator(train_generator, steps_per_epoch=num_train_examples//batch_size, epochs=number_of_epochs, callbacks=callbacks,\
|
||||
validation_data=eval_generator, validation_steps=num_eval_examples//batch_size, verbose=2)
|
||||
Загрузка…
Ссылка в новой задаче