From 575339add86da2a6a047c4c94f62d0e7363bd1a3 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Sun, 2 Nov 2014 19:18:53 -0800 Subject: [PATCH] changed GCC version of fileutil to use ftello/fseeko instead of fgetpos/fsetpos (expertimental probably not needed); fixed some C++11 insanity with types declared inside a base class that has a template parameter (Label{Id}Type in IData{Read,Writ}er), now compiles with GCC --- Common/DataReader.cpp | 6 +++--- Common/DataWriter.cpp | 4 ++-- Common/Include/DataReader.h | 12 +++++++----- Common/Include/DataWriter.h | 4 +++- Common/fileutil.cpp | 27 ++++++++++++++++++++++----- makefile | 2 +- 6 files changed, 38 insertions(+), 17 deletions(-) diff --git a/Common/DataReader.cpp b/Common/DataReader.cpp index 936a31e46..86c08b12c 100644 --- a/Common/DataReader.cpp +++ b/Common/DataReader.cpp @@ -117,7 +117,7 @@ void DataReader::SetSentenceEndInBatch(std::vector &sentenceEn // GetLabelMapping - Gets the label mapping from integer index to label type // returns - a map from numeric datatype to native label type template -const map::LabelIdType, typename IDataReader::LabelType>& DataReader::GetLabelMapping(const std::wstring& sectionName) +const std::map::LabelIdType, typename DataReader::LabelType>& DataReader::GetLabelMapping(const std::wstring& sectionName) { return m_dataReader->GetLabelMapping(sectionName); } @@ -126,7 +126,7 @@ const map::LabelIdType, typename IDataReader -void DataReader::SetLabelMapping(const std::wstring& sectionName, const std::map::LabelIdType, typename IDataReader::LabelType>& labelMapping) +void DataReader::SetLabelMapping(const std::wstring& sectionName, const std::map& labelMapping) { m_dataReader->SetLabelMapping(sectionName, labelMapping); } @@ -155,4 +155,4 @@ bool DataReader::DataEnd(EndDataType endDataType) template class DataReader; template class DataReader; -}}} \ No newline at end of file +}}} diff --git a/Common/DataWriter.cpp b/Common/DataWriter.cpp index 712e1d354..eb7d9939f 100644 --- a/Common/DataWriter.cpp +++ b/Common/DataWriter.cpp @@ -111,7 +111,7 @@ bool DataWriter::SaveData(size_t recordStart, const std::map -void DataWriter::SaveMapping(std::wstring saveId, const std::map& labelMapping) +void DataWriter::SaveMapping(std::wstring saveId, const std::map& labelMapping) { m_dataWriter->SaveMapping(saveId, labelMapping); } @@ -120,4 +120,4 @@ void DataWriter::SaveMapping(std::wstring saveId, const std::map; template class DataWriter; -}}} \ No newline at end of file +}}} diff --git a/Common/Include/DataReader.h b/Common/Include/DataReader.h index 1f0596ea8..69a53c4bd 100644 --- a/Common/Include/DataReader.h +++ b/Common/Include/DataReader.h @@ -45,7 +45,7 @@ class DATAREADER_API IDataReader { public: typedef std::string LabelType; - typedef unsigned LabelIdType; + typedef unsigned int LabelIdType; virtual void Init(const ConfigParameters& config) = 0; virtual void Destroy() = 0; @@ -53,8 +53,8 @@ public: virtual bool GetMinibatch(std::map*>& matrices) = 0; virtual size_t NumberSlicesInEachRecurrentIter() = 0; virtual void SetNbrSlicesEachRecurrentIter(const size_t) = 0; - virtual const std::map& GetLabelMapping(const std::wstring& sectionName) = 0; - virtual void SetLabelMapping(const std::wstring& sectionName, const std::map& labelMapping) = 0; + virtual const std::map& GetLabelMapping(const std::wstring& sectionName) = 0; + virtual void SetLabelMapping(const std::wstring& sectionName, const std::map& labelMapping) = 0; virtual bool GetData(const std::wstring& sectionName, size_t numRecords, void* data, size_t& dataBufferSize, size_t recordStart) = 0; virtual bool DataEnd(EndDataType endDataType) = 0; virtual void SetSentenceEndInBatch(vector &sentenceEnd) = 0; @@ -74,6 +74,8 @@ extern "C" DATAREADER_API void GetReaderD(IDataReader** preader); template class DataReader : public IDataReader, public Plugin { + typedef typename IDataReader::LabelType LabelType; + typedef typename IDataReader::LabelIdType LabelIdType; private: IDataReader *m_dataReader; // reader @@ -132,12 +134,12 @@ public: // GetLabelMapping - Gets the label mapping from integer index to label type // returns - a map from numeric datatype to native label type - virtual const std::map& GetLabelMapping(const std::wstring& sectionName); + virtual const std::map& GetLabelMapping(const std::wstring& sectionName); // SetLabelMapping - Sets the label mapping from integer index to label // labelMapping - mapping table from label values to IDs (must be 0-n) // note: for tasks with labels, the mapping table must be the same between a training run and a testing run - virtual void SetLabelMapping(const std::wstring& sectionName, const std::map& labelMapping); + virtual void SetLabelMapping(const std::wstring& sectionName, const std::map& labelMapping); // GetData - Gets metadata from the specified section (into CPU memory) // sectionName - section name to retrieve data from diff --git a/Common/Include/DataWriter.h b/Common/Include/DataWriter.h index 6478ef4a3..e29ea5fce 100644 --- a/Common/Include/DataWriter.h +++ b/Common/Include/DataWriter.h @@ -47,7 +47,7 @@ class DATAWRITER_API IDataWriter { public: typedef std::string LabelType; - typedef unsigned LabelIdType; + typedef unsigned int LabelIdType; virtual void Init(const ConfigParameters& config) = 0; virtual void Destroy() = 0; @@ -71,6 +71,8 @@ extern "C" DATAWRITER_API void GetWriterD(IDataWriter** pwriter); template class DataWriter : public IDataWriter, public Plugin { + typedef typename IDataWriter::LabelType LabelType; + typedef typename IDataWriter::LabelIdType LabelIdType; private: IDataWriter *m_dataWriter; // writer diff --git a/Common/fileutil.cpp b/Common/fileutil.cpp index e05147270..a1ae32c94 100644 --- a/Common/fileutil.cpp +++ b/Common/fileutil.cpp @@ -7,7 +7,7 @@ #define _CRT_SECURE_NO_WARNINGS // "secure" CRT not available on all platforms --add this at the top of all CPP files that give "function or variable may be unsafe" warnings #define _CRT_NONSTDC_NO_DEPRECATE // make VS accept POSIX functions without _ #pragma warning (disable: 4996) // ^^ this does not seem to work--TODO: make it work -#define _FILE_OFFSET_BITS 64 // for ftell64() in Linux +#define _FILE_OFFSET_BITS 64 // to force fseeko() and ftello() 64 bit in Linux #ifndef UNDER_CE // fixed-buffer overloads not available for wince #ifdef _CRT_SECURE_CPP_OVERLOAD_STANDARD_NAMES // fixed-buffer overloads for strcpy() etc. @@ -331,16 +331,26 @@ int64_t filesize64 (const wchar_t * pathname) uint64_t fgetpos (FILE * f) { +#ifdef _MSC_VER // standard does not allow to cast between fpos_t and integer numbers, so use ftello() instead fpos_t post; int rc = ::fgetpos (f, &post); if (rc != 0) RuntimeError ("error getting file position: %s", strerror (errno)); return post; +#else + off_t post = ftello (f); + uint64_t pos = (uint64_t) post; + static_assert (sizeof(off_t) >= sizeof(pos), "64-bit file offsets not enabled"); + if ((decltype (pos)) post != pos) + LogicError("64-bit file offsets not enabled"); + int rc = fseeko(f, post, SEEK_SET); +#endif } void fsetpos (FILE * f, uint64_t reqpos) { -#ifdef _MSC_VER +#ifdef _MSC_VER // standard does not allow to cast between fpos_t and integer numbers, so use fseeko() instead +#ifdef _MSC_VER // special hack for VS CRT // Visual Studio's ::fsetpos() flushes the read buffer. This conflicts with a situation where // we generally read linearly but skip a few bytes or KB occasionally, as is // the case in speech recognition tools. This requires a number of optimizations. @@ -363,13 +373,20 @@ void fsetpos (FILE * f, uint64_t reqpos) if (curpos != fgetpos (f) || curpos + f->_cnt != cureob) break; // oops } -#endif +#endif // end special hack for VS CRT // actually perform the seek fpos_t post = reqpos; - int rc = ::fsetpos (f, &post); + int rc = ::fsetpos(f, &post); +#else // assuming __unix__ + off_t post = (off_t) reqpos; + static_assert (sizeof (off_t) >= sizeof (reqpos), "64-bit file offsets not enabled"); + if ((decltype (reqpos)) post != reqpos) + LogicError("64-bit file offsets not enabled"); + int rc = fseeko(f, post, SEEK_SET); +#endif if (rc != 0) - RuntimeError ("error setting file position: %s", strerror (errno)); + RuntimeError("error setting file position: %s", strerror(errno)); } // ---------------------------------------------------------------------------- diff --git a/makefile b/makefile index dc12f6b71..3e2e30ef7 100644 --- a/makefile +++ b/makefile @@ -39,7 +39,7 @@ SRC = $(MATH_SRC) $(COMMON_SRC) all: ${SRC:.cpp=.obj} -CFLAGS = -std=c++0x -std=c++11 -DCPUONLY -D_POSIX_SOURCE -D_XOPEN_SOURCE=600 -fpermissive +CFLAGS = -std=c++0x -std=c++11 -DCPUONLY -D_POSIX_SOURCE -D_XOPEN_SOURCE=600 -D__USE_XOPEN2K -fpermissive %.obj: %.cpp gcc -c -o $@ $(CPPFLAGS) $(CFLAGS) $(INCFLAGS) -MD -MP -MF ${@:.obj=.dep} $<