Support of ... syntax in image reader

Source/Readers/ImageReader/ByteReader.h

@@ -30,8 +30,13 @@ public:
 class FileByteReader : public ByteReader
 {
 public:
+    FileByteReader(const std::string& expandDirectory) : m_expandDirectory(expandDirectory)
+    {}
+
     void Register(const std::map<std::string, size_t>&) override {}
     cv::Mat Read(size_t seqId, const std::string& path, bool grayscale) override;
+
+    std::string m_expandDirectory;
 };
 
 #ifdef USE_ZIP

Source/Readers/ImageReader/ImageDataDeserializer.cpp

@@ -251,6 +251,10 @@ void ImageDataDeserializer::CreateSequenceDescriptions(CorpusDescriptorPtr corpu
         RuntimeError("Could not open %s for reading.", mapPath.c_str());
     }
 
+    // Creating the default reader with expanded directory to the map file.
+    auto mapFileDirectory = ExtractDirectory(mapPath);
+    m_defaultReader = make_unique<FileByteReader>(mapFileDirectory);
+
     size_t itemsPerLine = isMultiCrop ? 10 : 1;
     size_t curId = 0;
     std::string line;
@@ -315,7 +319,7 @@ void ImageDataDeserializer::CreateSequenceDescriptions(CorpusDescriptorPtr corpu
 
             m_keyToSequence[description.m_key.m_sequence] = m_imageSequences.size();
             m_imageSequences.push_back(description);
-            RegisterByteReader(description.m_id, description.m_path, knownReaders, readerSequences);
+            RegisterByteReader(description.m_id, description.m_path, knownReaders, readerSequences, mapFileDirectory);
         }
     }
 
@@ -337,9 +341,11 @@ ChunkPtr ImageDataDeserializer::GetChunk(ChunkIdType chunkId)
     return std::make_shared<ImageChunk>(sequenceDescription, *this);
 }
 
-void ImageDataDeserializer::RegisterByteReader(size_t seqId, const std::string& path, PathReaderMap& knownReaders, ReaderSequenceMap& readerSequences)
+void ImageDataDeserializer::RegisterByteReader(size_t seqId, const std::string& seqPath, PathReaderMap& knownReaders, ReaderSequenceMap& readerSequences, const std::string& expandDirectory)
 {
-    assert(!path.empty());
+    assert(!seqPath.empty());
+
+    auto path = Expand3Dots(seqPath, expandDirectory);
 
     auto atPos = path.find_first_of('@');
     // Is it container or plain image file?
@@ -383,13 +389,14 @@ cv::Mat ImageDataDeserializer::ReadImage(size_t seqId, const std::string& path,
 
     ImageDataDeserializer::SeqReaderMap::const_iterator r;
     if (m_readers.empty() || (r = m_readers.find(seqId)) == m_readers.end())
-        return m_defaultReader.Read(seqId, path, grayscale);
+        return m_defaultReader->Read(seqId, path, grayscale);
     return (*r).second->Read(seqId, path, grayscale);
 }
 
-cv::Mat FileByteReader::Read(size_t, const std::string& path, bool grayscale)
+cv::Mat FileByteReader::Read(size_t, const std::string& seqPath, bool grayscale)
 {
-    assert(!path.empty());
+    assert(!seqPath.empty());
+    auto path = Expand3Dots(seqPath, m_expandDirectory);
 
     return cv::imread(path, grayscale ? cv::IMREAD_GRAYSCALE : cv::IMREAD_COLOR);
 }

Source/Readers/ImageReader/ImageDataDeserializer.h

@@ -73,14 +73,14 @@ private:
     // Not using nocase_compare here as it's not correct on Linux.
     using PathReaderMap = std::unordered_map<std::string, std::shared_ptr<ByteReader>>;
     using ReaderSequenceMap = std::map<std::string, std::map<std::string, size_t>>;
-    void RegisterByteReader(size_t seqId, const std::string& path, PathReaderMap& knownReaders, ReaderSequenceMap& readerSequences);
+    void RegisterByteReader(size_t seqId, const std::string& path, PathReaderMap& knownReaders, ReaderSequenceMap& readerSequences, const std::string& expandDirectory);
     cv::Mat ReadImage(size_t seqId, const std::string& path, bool grayscale);
 
     // REVIEW alexeyk: can potentially use vector instead of map. Need to handle default reader and resizing though.
     using SeqReaderMap = std::unordered_map<size_t, std::shared_ptr<ByteReader>>;
     SeqReaderMap m_readers;
 
-    FileByteReader m_defaultReader;
+    std::unique_ptr<FileByteReader> m_defaultReader;
     int m_verbosity;
 };
 

Source/Readers/ReaderLib/ConfigUtil.h

@@ -37,4 +37,50 @@ inline std::vector<std::string> GetSectionsWithParameter(const std::string& read
     return result;
 }
 
+template<class T>
+struct DirectoryExpansion
+{
+    static T*  Delimiters() { return "/\\"; }
+    static T*  Pattern() { return "..."; }
+};
+
+template<>
+struct DirectoryExpansion<wchar_t>
+{
+    static wchar_t*  Delimiters() { return L"/\\"; }
+    static wchar_t*  Pattern() { return L"..."; }
+};
+
+// Extracts the directory from the absolute path.
+template<class TString>
+TString ExtractDirectory(const TString& absoluteFilePath)
+{
+    static_assert(std::is_same<TString, std::string>::value || std::is_same<TString, std::wstring>::value, "Only string types are supported");
+    using Char = typename TString::value_type;
+
+    const auto delim = DirectoryExpansion<Char>::Delimiters();
+    auto result = absoluteFilePath;
+    auto pos = result.find_last_of(delim);
+    if (pos != result.npos)
+        result.resize(pos);
+    return result;
+}
+
+// Expands the filePath using the directory if the filePath starts with ...
+template<class TString>
+TString Expand3Dots(const TString& filePath, const TString& directoryExpansion)
+{
+    static_assert(std::is_same<TString, std::string>::value || std::is_same<TString, std::wstring>::value, "Only string types are supported");
+    using Char = typename TString::value_type;
+
+    const auto extensionPattern = DirectoryExpansion<Char>::Pattern();
+    size_t pos = filePath.find(extensionPattern);
+    if (pos == filePath.npos)
+        return filePath;
+
+    if (pos != 0)
+        RuntimeError("Invalid path containing '...'");
+    return directoryExpansion + filePath.substr(pos + 3);
+}
+
 }}}

Tests/UnitTests/ReaderTests/Config/ImageDeserializers.cntk

@@ -1,19 +1,9 @@
 RootDir = .
-ModelDir = "models"
-command = "Simple_Test"
-
 precision = "float"
 
-modelPath = "$ModelDir$/ImageReaderSimple_Model.dnn"
+MapFile="testMustSubstituteThis"
 
-# deviceId = -1 for CPU, >= 0 for GPU devices
-deviceId = -1
-
-outputNodeNames = "Dummy"
-traceLevel = 1
-frameMode = false
-
-Simple_Test = [
+ImageAndImageReaderSimple_Test = [
 
 # Parameter values for the reader
 reader = [
@@ -24,7 +14,7 @@ reader = [
             [
                 type = "ImageDeserializer"
                 module = "ImageReader"
-                file = "$RootDir$/ImageReaderSimple_map.txt"
+                file = "$MapFile$"
 
                 input = [
                     features1 = [
@@ -82,3 +72,18 @@ reader = [
         )
     ]
 ]
+
+3DotsExpansionTest = {
+    reader = {
+        deserializers = ({
+            type = "ImageDeserializer"
+            module = "ImageReader"
+            file = "$MapFile$"
+
+            input = {
+                features = { transforms = () }
+                labels = { labelDim = 4 }               
+            }
+        })
+    }
+}

Tests/UnitTests/ReaderTests/Control/ImageReader3DotsSyntaxInMapFile_Control.txt

@@ -0,0 +1,4 @@
+0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
+0 0 254 0 0 254 0 0 254 0 0 254 0 0 254 0 0 254 0 0 254 0 0 254 0 0 254 0 0 254 0 0 254 0 0 254 0 0 254 0 0 254 0 0 254 0 0 254 0 0 254 0 0 254 0 0 254 0 0 254 0 0 254 0 0 254 0 0 254 0 0 254 0 0 254 0 0 254 0 0 254 0 0 254 0 0 254 0 0 254 0 0 254 0 0 254
+1 0 0 0
+0 0 0 1

Tests/UnitTests/ReaderTests/Data/ImageReader3Dots_map.txt

@@ -0,0 +1,2 @@
+.../images/simple.zip@/chunk0/black.jpg	0
+.../images/red.jpg	3

Tests/UnitTests/ReaderTests/ImageReaderTests.cpp

@@ -324,4 +324,41 @@ BOOST_AUTO_TEST_CASE(ImageReaderMissingScaleTransforms)
 }
 
 BOOST_AUTO_TEST_SUITE_END()
-} } } }
+
+namespace
+{
+    // Test with not set data directory.
+    struct EmptyDataDirFixture : ReaderFixture
+    {
+        EmptyDataDirFixture() : ReaderFixture("/.") { }
+    };
+
+    BOOST_FIXTURE_TEST_SUITE(ReaderTestSuite, EmptyDataDirFixture)
+
+    BOOST_AUTO_TEST_CASE(ImageReader3DotsSyntaxInMapFile)
+    {
+        auto testDir = testDataPath();
+        std::wstring mapFileLocaton = std::wstring(testDir.begin(), testDir.end()) + L"/Data/ImageReader3Dots_map.txt";
+        HelperRunReaderTest<float>(
+            testDataPath() + "/Config/ImageDeserializers.cntk",
+            testDataPath() + "/Control/ImageReader3DotsSyntaxInMapFile_Control.txt",
+            testDataPath() + "/Control/ImageReader3DotsSyntaxInMapFile_Output.txt",
+            "3DotsExpansionTest",
+            "reader",
+            1,
+            2,
+            1,
+            1,
+            1,
+            0,
+            1,
+            false,
+            true,
+            true,
+            { L"MapFile=\"" + mapFileLocaton + L"\"" });
+    }
+
+    BOOST_AUTO_TEST_SUITE_END()
+}
+
+}}}}