microsoft
/
LightLDA
зеркало из https://github.com/microsoft/LightLDA.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

add vs infer project for windows, remove pthread

This commit is contained in:
feiga 2016-01-05 12:04:45 +08:00
Родитель 15ee80712f
Коммит 8666a2c062
4 изменённых файлов: 58 добавлений и 28 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

40
inference/infer.cpp
Просмотреть файл

@ -9,7 +9,9 @@
#include "inferer.h"
#include <vector>
#include <iostream>
#include <pthread.h>
#include <thread>
// #include <pthread.h>
#include <multiverso/barrier.h>
namespace multiverso { namespace lightlda
{
@ -18,7 +20,7 @@ namespace multiverso { namespace lightlda
public:
static void Run(int argc, char** argv)
{
Log::ResetLogFile("LightLDA." + std::to_string(clock()) + ".log");
Log::ResetLogFile("LightLDA_infer." + std::to_string(clock()) + ".log");
Config::Init(argc, argv);
//init meta
meta.Init();
@ -32,8 +34,9 @@ namespace multiverso { namespace lightlda
AliasTable* alias_table = new AliasTable();
//init inferers
std::vector<Inferer*> inferers;
pthread_barrier_t barrier;
pthread_barrier_init(&barrier, nullptr, Config::num_local_workers);
Barrier barrier(Config::num_local_workers);
// pthread_barrier_t barrier;
// pthread_barrier_init(&barrier, nullptr, Config::num_local_workers);
for (int32_t i = 0; i < Config::num_local_workers; ++i)
{
inferers.push_back(new Inferer(alias_table, data_stream,
@ -53,7 +56,7 @@ namespace multiverso { namespace lightlda
delete inferer;
inferer = nullptr;
}
pthread_barrier_destroy(&barrier);
// pthread_barrier_destroy(&barrier);
delete data_stream;
delete alias_table;
delete model;
@ -61,23 +64,26 @@ namespace multiverso { namespace lightlda
private:
static void Inference(std::vector<Inferer*>& inferers)
{
pthread_t * threads = new pthread_t[Config::num_local_workers];
if(nullptr == threads)
//pthread_t * threads = new pthread_t[Config::num_local_workers];
//if(nullptr == threads)
//{
// Log::Fatal("failed to allocate space for worker threads");
//}
std::vector<std::thread> threads;
for(int32_t i = 0; i < Config::num_local_workers; ++i)
{
Log::Fatal("failed to allocate space for worker threads");
threads.push_back(std::thread(&InferenceThread, inferers[i]));
//if(pthread_create(threads + i, nullptr, InferenceThread, inferers[i]))
//{
// Log::Fatal("failed to create worker threads");
//}
}
for(int32_t i = 0; i < Config::num_local_workers; ++i)
{
if(pthread_create(threads + i, nullptr, InferenceThread, inferers[i]))
{
Log::Fatal("failed to create worker threads");
}
// pthread_join(threads[i], nullptr);
threads[i].join();
}
for(int32_t i = 0; i < Config::num_local_workers; ++i)
{
pthread_join(threads[i], nullptr);
}
delete [] threads;
// delete [] threads;
}
static void* InferenceThread(void* arg)

18
inference/inferer.cpp
Просмотреть файл

@ -9,13 +9,14 @@
#include "data_stream.h"
#include <multiverso/stop_watch.h>
#include <multiverso/log.h>
#include <multiverso/barrier.h>
namespace multiverso { namespace lightlda
{
Inferer::Inferer(AliasTable* alias_table,
IDataStream * data_stream,
Meta* meta, LocalModel * model,
pthread_barrier_t* barrier,
Barrier* barrier,
int32_t id, int32_t thread_num):
alias_(alias_table), data_stream_(data_stream),
meta_(meta), model_(model),
@ -36,7 +37,8 @@ namespace multiverso { namespace lightlda
{
//get data block
if(id_ == 0) data_stream_->BeforeDataAccess();
pthread_barrier_wait(barrier_);
// pthread_barrier_wait(barrier_);
barrier_->Wait();
DataBlock& data = data_stream_->CurrDataBlock();
data.set_meta(&(meta_->local_vocab(block)));
lda_data_block_->set_data(&data);
@ -47,7 +49,8 @@ namespace multiverso { namespace lightlda
const LocalVocab& local_vocab = data.meta();
//determin alias index
if (id_ == 0) alias_->Init(meta_->alias_index(block, 0));
pthread_barrier_wait(barrier_);
// pthread_barrier_wait(barrier_);
barrier_->Wait();
// build alias table
for (const int32_t* pword = local_vocab.begin(0) + id_;
pword < local_vocab.end(0);
@ -56,7 +59,8 @@ namespace multiverso { namespace lightlda
alias_->Build(*pword, model_);
}
if (id_ == 0) alias_->Build(-1, model_);
pthread_barrier_wait(barrier_);
// pthread_barrier_wait(barrier_);
barrier_->Wait();
if (id_ == 0)
{
Log::Info("Alias Time used: %.2f s \n", watch.ElapsedSeconds());
@ -75,7 +79,8 @@ namespace multiverso { namespace lightlda
Log::Info("Iter = %d, Block = %d\n", iter, block);
}
// wait for all threads
pthread_barrier_wait(barrier_);
// pthread_barrier_wait(barrier_);
barrier_->Wait();
// Inference with lightlda sampler
for (int32_t doc_id = id_; doc_id < data.Size(); doc_id += thread_num_)
{
@ -86,7 +91,8 @@ namespace multiverso { namespace lightlda
void Inferer::EndIteration()
{
pthread_barrier_wait(barrier_);
// pthread_barrier_wait(barrier_);
barrier_->Wait();
if(id_ == 0)
{
data_stream_->EndDataAccess();

13
inference/inferer.h
Просмотреть файл

@ -5,11 +5,16 @@
#ifndef LIGHTLDA_INFERER_H_
#define LIGHTLDA_INFERER_H_
#include <pthread.h>
// #include <pthread.h>
#include <multiverso/multiverso.h>
#include <multiverso/log.h>
#include <multiverso/barrier.h>
namespace multiverso { namespace lightlda
namespace multiverso
{
class Barrier;
namespace lightlda
{
class AliasTable;
class LDADataBlock;
@ -24,7 +29,7 @@ namespace multiverso { namespace lightlda
Inferer(AliasTable* alias_table,
IDataStream * data_stream,
Meta* meta, LocalModel * model,
pthread_barrier_t* barrier,
Barrier* barrier,
int32_t id, int32_t thread_num);
~Inferer();
@ -36,7 +41,7 @@ namespace multiverso { namespace lightlda
IDataStream * data_stream_;
Meta* meta_;
LocalModel * model_;
pthread_barrier_t* barrier_;
Barrier* barrier_;
int32_t id_;
int32_t thread_num_;
LightDocSampler* sampler_;

15
windows/lightlda.sln
Просмотреть файл

@ -1,12 +1,14 @@

Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 2013
VisualStudioVersion = 12.0.21005.1
VisualStudioVersion = 12.0.40629.0
MinimumVisualStudioVersion = 10.0.40219.1
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "lightlda", "lightlda\lightlda.vcxproj", "{9A6EEB61-6B68-49A3-B83A-E044F6678D6A}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "dump_binary", "dump_binary\dump_binary.vcxproj", "{FFD24CAD-825A-4732-8186-4361D3E1438F}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "infer", "infer\infer.vcxproj", "{3CF22D68-9F4B-46B9-B0D4-7129467DD759}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Mixed Platforms = Debug|Mixed Platforms
@ -40,6 +42,17 @@ Global
{FFD24CAD-825A-4732-8186-4361D3E1438F}.Release|Win32.Build.0 = Release|Win32
{FFD24CAD-825A-4732-8186-4361D3E1438F}.Release|x64.ActiveCfg = Release|x64
{FFD24CAD-825A-4732-8186-4361D3E1438F}.Release|x64.Build.0 = Release|x64
{3CF22D68-9F4B-46B9-B0D4-7129467DD759}.Debug|Mixed Platforms.ActiveCfg = Debug|Win32
{3CF22D68-9F4B-46B9-B0D4-7129467DD759}.Debug|Mixed Platforms.Build.0 = Debug|Win32
{3CF22D68-9F4B-46B9-B0D4-7129467DD759}.Debug|Win32.ActiveCfg = Debug|Win32
{3CF22D68-9F4B-46B9-B0D4-7129467DD759}.Debug|Win32.Build.0 = Debug|Win32
{3CF22D68-9F4B-46B9-B0D4-7129467DD759}.Debug|x64.ActiveCfg = Debug|Win32
{3CF22D68-9F4B-46B9-B0D4-7129467DD759}.Release|Mixed Platforms.ActiveCfg = Release|Win32
{3CF22D68-9F4B-46B9-B0D4-7129467DD759}.Release|Mixed Platforms.Build.0 = Release|Win32
{3CF22D68-9F4B-46B9-B0D4-7129467DD759}.Release|Win32.ActiveCfg = Release|Win32
{3CF22D68-9F4B-46B9-B0D4-7129467DD759}.Release|Win32.Build.0 = Release|Win32
{3CF22D68-9F4B-46B9-B0D4-7129467DD759}.Release|x64.ActiveCfg = Release|x64
{3CF22D68-9F4B-46B9-B0D4-7129467DD759}.Release|x64.Build.0 = Release|x64
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE