From 8666a2c0628d85433eb68f8b5a77570472976335 Mon Sep 17 00:00:00 2001 From: feiga Date: Tue, 5 Jan 2016 12:04:45 +0800 Subject: [PATCH] add vs infer project for windows, remove pthread --- inference/infer.cpp | 40 +++++++++++++++++++++++----------------- inference/inferer.cpp | 18 ++++++++++++------ inference/inferer.h | 13 +++++++++---- windows/lightlda.sln | 15 ++++++++++++++- 4 files changed, 58 insertions(+), 28 deletions(-) diff --git a/inference/infer.cpp b/inference/infer.cpp index 685b989..a0d3a60 100644 --- a/inference/infer.cpp +++ b/inference/infer.cpp @@ -9,7 +9,9 @@ #include "inferer.h" #include #include -#include +#include +// #include +#include namespace multiverso { namespace lightlda { @@ -18,7 +20,7 @@ namespace multiverso { namespace lightlda public: static void Run(int argc, char** argv) { - Log::ResetLogFile("LightLDA." + std::to_string(clock()) + ".log"); + Log::ResetLogFile("LightLDA_infer." + std::to_string(clock()) + ".log"); Config::Init(argc, argv); //init meta meta.Init(); @@ -32,8 +34,9 @@ namespace multiverso { namespace lightlda AliasTable* alias_table = new AliasTable(); //init inferers std::vector inferers; - pthread_barrier_t barrier; - pthread_barrier_init(&barrier, nullptr, Config::num_local_workers); + Barrier barrier(Config::num_local_workers); + // pthread_barrier_t barrier; + // pthread_barrier_init(&barrier, nullptr, Config::num_local_workers); for (int32_t i = 0; i < Config::num_local_workers; ++i) { inferers.push_back(new Inferer(alias_table, data_stream, @@ -53,7 +56,7 @@ namespace multiverso { namespace lightlda delete inferer; inferer = nullptr; } - pthread_barrier_destroy(&barrier); + // pthread_barrier_destroy(&barrier); delete data_stream; delete alias_table; delete model; @@ -61,23 +64,26 @@ namespace multiverso { namespace lightlda private: static void Inference(std::vector& inferers) { - pthread_t * threads = new pthread_t[Config::num_local_workers]; - if(nullptr == threads) + //pthread_t * threads = new pthread_t[Config::num_local_workers]; + //if(nullptr == threads) + //{ + // Log::Fatal("failed to allocate space for worker threads"); + //} + std::vector threads; + for(int32_t i = 0; i < Config::num_local_workers; ++i) { - Log::Fatal("failed to allocate space for worker threads"); + threads.push_back(std::thread(&InferenceThread, inferers[i])); + //if(pthread_create(threads + i, nullptr, InferenceThread, inferers[i])) + //{ + // Log::Fatal("failed to create worker threads"); + //} } for(int32_t i = 0; i < Config::num_local_workers; ++i) { - if(pthread_create(threads + i, nullptr, InferenceThread, inferers[i])) - { - Log::Fatal("failed to create worker threads"); - } + // pthread_join(threads[i], nullptr); + threads[i].join(); } - for(int32_t i = 0; i < Config::num_local_workers; ++i) - { - pthread_join(threads[i], nullptr); - } - delete [] threads; + // delete [] threads; } static void* InferenceThread(void* arg) diff --git a/inference/inferer.cpp b/inference/inferer.cpp index d31b79e..a9981da 100644 --- a/inference/inferer.cpp +++ b/inference/inferer.cpp @@ -9,13 +9,14 @@ #include "data_stream.h" #include #include +#include namespace multiverso { namespace lightlda { Inferer::Inferer(AliasTable* alias_table, IDataStream * data_stream, Meta* meta, LocalModel * model, - pthread_barrier_t* barrier, + Barrier* barrier, int32_t id, int32_t thread_num): alias_(alias_table), data_stream_(data_stream), meta_(meta), model_(model), @@ -36,7 +37,8 @@ namespace multiverso { namespace lightlda { //get data block if(id_ == 0) data_stream_->BeforeDataAccess(); - pthread_barrier_wait(barrier_); + // pthread_barrier_wait(barrier_); + barrier_->Wait(); DataBlock& data = data_stream_->CurrDataBlock(); data.set_meta(&(meta_->local_vocab(block))); lda_data_block_->set_data(&data); @@ -47,7 +49,8 @@ namespace multiverso { namespace lightlda const LocalVocab& local_vocab = data.meta(); //determin alias index if (id_ == 0) alias_->Init(meta_->alias_index(block, 0)); - pthread_barrier_wait(barrier_); + // pthread_barrier_wait(barrier_); + barrier_->Wait(); // build alias table for (const int32_t* pword = local_vocab.begin(0) + id_; pword < local_vocab.end(0); @@ -56,7 +59,8 @@ namespace multiverso { namespace lightlda alias_->Build(*pword, model_); } if (id_ == 0) alias_->Build(-1, model_); - pthread_barrier_wait(barrier_); + // pthread_barrier_wait(barrier_); + barrier_->Wait(); if (id_ == 0) { Log::Info("Alias Time used: %.2f s \n", watch.ElapsedSeconds()); @@ -75,7 +79,8 @@ namespace multiverso { namespace lightlda Log::Info("Iter = %d, Block = %d\n", iter, block); } // wait for all threads - pthread_barrier_wait(barrier_); + // pthread_barrier_wait(barrier_); + barrier_->Wait(); // Inference with lightlda sampler for (int32_t doc_id = id_; doc_id < data.Size(); doc_id += thread_num_) { @@ -86,7 +91,8 @@ namespace multiverso { namespace lightlda void Inferer::EndIteration() { - pthread_barrier_wait(barrier_); + // pthread_barrier_wait(barrier_); + barrier_->Wait(); if(id_ == 0) { data_stream_->EndDataAccess(); diff --git a/inference/inferer.h b/inference/inferer.h index cd40ebb..df2d871 100644 --- a/inference/inferer.h +++ b/inference/inferer.h @@ -5,11 +5,16 @@ #ifndef LIGHTLDA_INFERER_H_ #define LIGHTLDA_INFERER_H_ -#include +// #include #include #include +#include -namespace multiverso { namespace lightlda +namespace multiverso +{ + class Barrier; + +namespace lightlda { class AliasTable; class LDADataBlock; @@ -24,7 +29,7 @@ namespace multiverso { namespace lightlda Inferer(AliasTable* alias_table, IDataStream * data_stream, Meta* meta, LocalModel * model, - pthread_barrier_t* barrier, + Barrier* barrier, int32_t id, int32_t thread_num); ~Inferer(); @@ -36,7 +41,7 @@ namespace multiverso { namespace lightlda IDataStream * data_stream_; Meta* meta_; LocalModel * model_; - pthread_barrier_t* barrier_; + Barrier* barrier_; int32_t id_; int32_t thread_num_; LightDocSampler* sampler_; diff --git a/windows/lightlda.sln b/windows/lightlda.sln index 0897c90..34894ae 100755 --- a/windows/lightlda.sln +++ b/windows/lightlda.sln @@ -1,12 +1,14 @@  Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio 2013 -VisualStudioVersion = 12.0.21005.1 +VisualStudioVersion = 12.0.40629.0 MinimumVisualStudioVersion = 10.0.40219.1 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "lightlda", "lightlda\lightlda.vcxproj", "{9A6EEB61-6B68-49A3-B83A-E044F6678D6A}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "dump_binary", "dump_binary\dump_binary.vcxproj", "{FFD24CAD-825A-4732-8186-4361D3E1438F}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "infer", "infer\infer.vcxproj", "{3CF22D68-9F4B-46B9-B0D4-7129467DD759}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Mixed Platforms = Debug|Mixed Platforms @@ -40,6 +42,17 @@ Global {FFD24CAD-825A-4732-8186-4361D3E1438F}.Release|Win32.Build.0 = Release|Win32 {FFD24CAD-825A-4732-8186-4361D3E1438F}.Release|x64.ActiveCfg = Release|x64 {FFD24CAD-825A-4732-8186-4361D3E1438F}.Release|x64.Build.0 = Release|x64 + {3CF22D68-9F4B-46B9-B0D4-7129467DD759}.Debug|Mixed Platforms.ActiveCfg = Debug|Win32 + {3CF22D68-9F4B-46B9-B0D4-7129467DD759}.Debug|Mixed Platforms.Build.0 = Debug|Win32 + {3CF22D68-9F4B-46B9-B0D4-7129467DD759}.Debug|Win32.ActiveCfg = Debug|Win32 + {3CF22D68-9F4B-46B9-B0D4-7129467DD759}.Debug|Win32.Build.0 = Debug|Win32 + {3CF22D68-9F4B-46B9-B0D4-7129467DD759}.Debug|x64.ActiveCfg = Debug|Win32 + {3CF22D68-9F4B-46B9-B0D4-7129467DD759}.Release|Mixed Platforms.ActiveCfg = Release|Win32 + {3CF22D68-9F4B-46B9-B0D4-7129467DD759}.Release|Mixed Platforms.Build.0 = Release|Win32 + {3CF22D68-9F4B-46B9-B0D4-7129467DD759}.Release|Win32.ActiveCfg = Release|Win32 + {3CF22D68-9F4B-46B9-B0D4-7129467DD759}.Release|Win32.Build.0 = Release|Win32 + {3CF22D68-9F4B-46B9-B0D4-7129467DD759}.Release|x64.ActiveCfg = Release|x64 + {3CF22D68-9F4B-46B9-B0D4-7129467DD759}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE