added perspective depth

AirLib/include/controllers/VehicleCameraBase.hpp

@@ -14,7 +14,8 @@ namespace msr { namespace airlib {
         public: //types
             enum class ImageType : uint { //this indexes to array
                 Scene = 0, 
-                DepthMeters, 
+                DepthPlanner, 
+                DepthPerspective,
                 DepthVis, 
                 DisparityNormalized,
                 Segmentation,

HelloCar/main.cpp

@@ -34,7 +34,7 @@ int main()
         client.confirmConnection();
 
         std::cout << "Press Enter to get FPV image" << std::endl; std::cin.get();
-        vector<ImageRequest> request = { ImageRequest(0, ImageType::Scene), ImageRequest(1, ImageType::DepthMeters, true) };
+        vector<ImageRequest> request = { ImageRequest(0, ImageType::Scene), ImageRequest(1, ImageType::DepthPlanner, true) };
         const vector<ImageResponse>& response = client.simGetImages(request);
         std::cout << "# of images recieved: " << response.size() << std::endl;
 

HelloDrone/main.cpp

@@ -30,7 +30,7 @@ int main()
         client.confirmConnection();
 
         std::cout << "Press Enter to get FPV image" << std::endl; std::cin.get();
-        vector<ImageRequest> request = { ImageRequest(0, ImageType::Scene), ImageRequest(1, ImageType::DepthMeters, true) };
+        vector<ImageRequest> request = { ImageRequest(0, ImageType::Scene), ImageRequest(1, ImageType::DepthPlanner, true) };
         const vector<ImageResponse>& response = client.simGetImages(request);
         std::cout << "# of images recieved: " << response.size() << std::endl;
 

PythonClient/PythonClient.py

@@ -21,11 +21,12 @@ class MsgpackMixin:
 
 class AirSimImageType:    
     Scene = 0
-    DepthMeters = 1
-    DepthVis = 2
-    DisparityNormalized = 3
-    Segmentation = 4
-    SurfaceNormals = 5
+    DepthPlanner = 1
+    DepthPerspective = 2
+    DepthVis = 3
+    DisparityNormalized = 4
+    Segmentation = 5
+    SurfaceNormals = 6
 
 class DrivetrainType:
     MaxDegreeOfFreedom = 0

PythonClient/cv_mode.py

@@ -13,7 +13,7 @@ for x in range(3): # do 5 times
 
     responses = client.simGetImages([
         ImageRequest(0, AirSimImageType.DepthVis),
-        ImageRequest(1, AirSimImageType.DepthMeters, True),
+        ImageRequest(1, AirSimImageType.DepthPerspective, True),
         ImageRequest(0, AirSimImageType.Segmentation),
         ImageRequest(0, AirSimImageType.Scene),
         ImageRequest(0, AirSimImageType.DisparityNormalized),

PythonClient/hello_car.py

@@ -46,7 +46,7 @@ while True:
     # get camera images from the car
     responses = client.simGetImages([
         ImageRequest(0, AirSimImageType.DepthVis),
-        ImageRequest(1, AirSimImageType.DepthMeters, True)]) 
+        ImageRequest(1, AirSimImageType.DepthPerspective, True)]) 
     print('Retrieved images: %d', len(responses))
 
     for response in responses:

PythonClient/hello_drone.py

@@ -15,7 +15,7 @@ client.moveToPosition(-10, 10, -10, 5)
 MultirotorClient.wait_key('Press any key to take images')
 responses = client.simGetImages([
     ImageRequest(0, AirSimImageType.DepthVis), 
-    ImageRequest(1, AirSimImageType.DepthMeters, True)])
+    ImageRequest(1, AirSimImageType.DepthPlanner, True)])
 print('Retrieved images: %d', len(responses))
 
 for response in responses:

Unreal/Plugins/AirSim/Source/Car/CarPawn.cpp

@@ -285,6 +285,13 @@ void ACarPawn::initializeForBeginPlay(bool enable_rpc, const std::string& api_se
     startApiServer(enable_rpc, api_server_address);
 }
 
+void ACarPawn::reset()
+{
+    this->getVehiclePawnWrapper()->reset();
+    controller_->setCarControls(CarController::CarControls());
+    api_control_enabled_ = false;
+}
+
 void ACarPawn::enableApiControl(bool is_enabled)
 {
     api_control_enabled_ = is_enabled;

Unreal/Plugins/AirSim/Source/Car/CarPawn.h

@@ -79,6 +79,7 @@ public:
 
     void enableApiControl(bool is_enabled);
     bool isApiControlEnabled();
+    void reset();
 
     // Begin Actor interface
     virtual void Tick(float Delta) override;

Unreal/Plugins/AirSim/Source/Car/SimModeCar.cpp

@@ -124,7 +124,7 @@ void ASimModeCar::reset()
         {
             //initialize each vehicle pawn we found
             TVehiclePawn* vehicle_pawn = static_cast<TVehiclePawn*>(pawn);
-            vehicle_pawn->getVehiclePawnWrapper()->reset();
+            vehicle_pawn->reset();
         }
     }
 

Unreal/Plugins/AirSim/Source/PIPCamera.cpp

@@ -22,8 +22,10 @@ void APIPCamera::PostInitializeComponents()
 
     captures_[Utils::toNumeric(ImageType::Scene)] = 
         UAirBlueprintLib::GetActorComponent<USceneCaptureComponent2D>(this, TEXT("SceneCaptureComponent"));
-    captures_[Utils::toNumeric(ImageType::DepthMeters)] = 
-        UAirBlueprintLib::GetActorComponent<USceneCaptureComponent2D>(this, TEXT("DepthMetersCaptureComponent"));
+    captures_[Utils::toNumeric(ImageType::DepthPlanner)] = 
+        UAirBlueprintLib::GetActorComponent<USceneCaptureComponent2D>(this, TEXT("DepthPlannerCaptureComponent"));
+    captures_[Utils::toNumeric(ImageType::DepthPerspective)] =
+        UAirBlueprintLib::GetActorComponent<USceneCaptureComponent2D>(this, TEXT("DepthPerspectiveCaptureComponent"));
     captures_[Utils::toNumeric(ImageType::DepthVis)] = 
         UAirBlueprintLib::GetActorComponent<USceneCaptureComponent2D>(this, TEXT("DepthVisCaptureComponent"));
     captures_[Utils::toNumeric(ImageType::DisparityNormalized)] = 

docs/image_apis.md

@@ -40,7 +40,7 @@ int getStereoAndDepthImages()
     vector<ImageRequest> request = { 
         ImageRequest(0, ImageType_::Scene), 
         ImageRequest(1, ImageType_::Scene),        
-        ImageRequest(1, ImageType_::DepthMeters, true) 
+        ImageRequest(1, ImageType_::DepthPlanner, true) 
     };
     const vector<ImageResponse>& response = client.simGetImages(request);
     //do something with response which contains image data, pose, timestamp etc
@@ -96,7 +96,7 @@ To change resolution, FOV etc, you can use [settings.json](settings.md). For exa
 
 ## What Does Pixel Values Mean in Different Image Types?
 ### Depth Image
-You normally want retrieve depth image as float (i.e. set `pixels_as_float = true` and specify `ImageType = DepthMeters` in `ImageRequest`) in which case each pixel is depth in *camera plane* in meters. This way ground truth depth image is directly compatible with estimated depth image generated by stereo algorithms. There is no need to convert depth to planner value like in old versions.
+You normally want retrieve depth image as float (i.e. set `pixels_as_float = true`) and specify `ImageType = DepthPlanner` or `ImageType = DepthPerspective` in `ImageRequest`. For `ImageType = DepthPlanner`, you get depth in camera plan, i.e., all points that are in plan parallel to camera have same depth. For `ImageType = DepthPerspective`, you get depth from camera using a project ray that hits that pixel. Depending on your use case, planner depth or perspective depth may be the ground truth image that you want. For example, you may be able to feed perspective depth to ROS package such as `depth_image_proc` to generate point cloud. Or planner depth may be more compatible with estimated depth image generated by stereo algorithms such as SGM.
 
 ### Depth Visualization Image
 When you specify `ImageType = DepthVis` in `ImageRequest`, you get image that helps depth visualization. In this case, each pixel value is interpolated from red to green depending on depth in camera plane in meters. The pixels with pure green means depth of 100m or more while pure red means depth of 0 meters.

docs/python.md

@@ -35,7 +35,7 @@ client.moveToPosition(-10, 10, -10, 5)
 MultirotorClient.wait_key('Press any key to take images')
 responses = client.simGetImages([
     ImageRequest(0, AirSimImageType.DepthVis), 
-    ImageRequest(1, AirSimImageType.DepthMeters, True)])
+    ImageRequest(1, AirSimImageType.DepthPlanner, True)])
 print('Retrieved images: %d', len(responses))
 
 for response in responses:
@@ -76,7 +76,7 @@ while True:
     # get camera images from the car
     responses = client.simGetImages([
         ImageRequest(0, AirSimImageType.DepthVis),
-        ImageRequest(1, AirSimImageType.DepthMeters, True)]) 
+        ImageRequest(1, AirSimImageType.DepthPlanner, True)]) 
     print('Retrieved images: %d', len(responses))
 
     for response in responses:

docs/settings.md

@@ -99,11 +99,12 @@ The `ImageType` element determines which image type the settings applies to. By
 
 ```
 Scene = 0, 
-DepthMeters = 1, 
-DepthVis = 2, 
-DisparityNormalized = 3,
-Segmentation = 4,
-SurfaceNormals = 5
+DepthPlanner = 1,
+DepthPerspective = 2,
+DepthVis = 3, 
+DisparityNormalized = 4,
+Segmentation = 5,
+SurfaceNormals = 6
 ```
 
 Note that `CaptureSettings` element is json array so you can add settings for multiple image types easily.