This commit is contained in:
chenqi 2022-03-22 11:56:18 +00:00
Родитель 067e5d267e
Коммит 6529a77c38
9 изменённых файлов: 178 добавлений и 29 удалений

Просмотреть файл

@ -165,6 +165,7 @@ namespace SPTAG
exit(-1);
}
LOG(Helper::LogLevel::LL_Info, "Begin to generate truth for query(%d,%d) and doc(%d,%d)...\n", querySet->Count(), querySet->Dimension(), vectorSet->Count(), vectorSet->Dimension());
std::vector< std::vector<SPTAG::SizeType> > truthset(querySet->Count(), std::vector<SPTAG::SizeType>(K, 0));
std::vector< std::vector<float> > distset(querySet->Count(), std::vector<float>(K, 0));
#pragma omp parallel for
@ -185,7 +186,7 @@ namespace SPTAG
}
}
LOG(Helper::LogLevel::LL_Info, "Start to write truth file...\n");
writeTruthFile(truthFile, querySet->Count(), K, truthset, distset, p_truthFileType);
auto ptr = SPTAG::f_createIO();

Просмотреть файл

@ -990,12 +990,6 @@ void buildGraph(SPTAG::VectorIndex* index, int m_iGraphSize, int m_iNeighborhood
int m_iFeatureDim = index->GetFeatureDim();
int m_disttype = (int)index->GetDistCalcMethod();
// Make sure that neighborhood size is a power of 2
if(m_iNeighborhoodSize == 0 || (m_iNeighborhoodSize & (m_iNeighborhoodSize-1)) != 0) {
LOG(SPTAG::Helper::LogLevel::LL_Error, "NeighborhoodSize (with scaling factor applied) is %d but must be a power of 2 for GPU construction.\n", m_iNeighborhoodSize);
exit(1);
}
// Have to give compiler-time known bounds on dimensions so that we can store points in registers
// This significantly speeds up distance comparisons.
// Create other options here for other commonly-used dimension values.
@ -1016,6 +1010,15 @@ void buildGraph(SPTAG::VectorIndex* index, int m_iGraphSize, int m_iNeighborhood
else if (m_iFeatureDim <= 768) {
buildGraphGPU_Batch<T, float, 768>(index, (size_t)m_iGraphSize, (size_t)m_iNeighborhoodSize, trees, results, graph, leafSize, NUM_GPUS, balanceFactor);
}
else if (m_iFeatureDim <= 1024) {
buildGraphGPU_Batch<T, float, 1024>(index, (size_t)m_iGraphSize, (size_t)m_iNeighborhoodSize, trees, results, graph, leafSize, NUM_GPUS, balanceFactor);
}
else if (m_iFeatureDim <= 2048) {
buildGraphGPU_Batch<T, float, 2048>(index, (size_t)m_iGraphSize, (size_t)m_iNeighborhoodSize, trees, results, graph, leafSize, NUM_GPUS, balanceFactor);
}
else if (m_iFeatureDim <= 4096) {
buildGraphGPU_Batch<T, float, 4096>(index, (size_t)m_iGraphSize, (size_t)m_iNeighborhoodSize, trees, results, graph, leafSize, NUM_GPUS, balanceFactor);
}
else {
LOG(SPTAG::Helper::LogLevel::LL_Error, "%d dimensions not currently supported for GPU construction.\n");
exit(1);
@ -1036,7 +1039,17 @@ void buildGraph(SPTAG::VectorIndex* index, int m_iGraphSize, int m_iNeighborhood
}
else if (m_iFeatureDim <= 768) {
buildGraphGPU_Batch<T, int32_t, 768>(index, (size_t)m_iGraphSize, (size_t)m_iNeighborhoodSize, trees, results, graph, leafSize, NUM_GPUS, balanceFactor);
} else {
}
else if (m_iFeatureDim <= 1024) {
buildGraphGPU_Batch<T, int32_t, 1024>(index, (size_t)m_iGraphSize, (size_t)m_iNeighborhoodSize, trees, results, graph, leafSize, NUM_GPUS, balanceFactor);
}
else if (m_iFeatureDim <= 2048) {
buildGraphGPU_Batch<T, int32_t, 2048>(index, (size_t)m_iGraphSize, (size_t)m_iNeighborhoodSize, trees, results, graph, leafSize, NUM_GPUS, balanceFactor);
}
else if (m_iFeatureDim <= 4096) {
buildGraphGPU_Batch<T, int32_t, 4096>(index, (size_t)m_iGraphSize, (size_t)m_iNeighborhoodSize, trees, results, graph, leafSize, NUM_GPUS, balanceFactor);
}
else {
LOG(SPTAG::Helper::LogLevel::LL_Error, "%d dimensions not currently supported for GPU construction.\n");
exit(1);
}

Просмотреть файл

@ -822,11 +822,23 @@ void VectorIndex::ApproximateRNG(std::shared_ptr<VectorSet>& fullVectors, std::u
#define DefineVectorValueType(Name, Type) \
case VectorValueType::Name: \
if(fullVectors->Dimension() <= 64) { \
getTailNeighborsTPT<Type, float, SUMTYPE, 64>((Type*)fullVectors->GetData(), fullVectors->Count(), this, exceptIDS, 64, replicaCount, numThreads, numTrees, leafSize, metric, numGPUs, selections); \
getTailNeighborsTPT<Type, float, SUMTYPE, 64>((Type*)fullVectors->GetData(), fullVectors->Count(), this, exceptIDS, fullVectors->Dimension(), replicaCount, numThreads, numTrees, leafSize, metric, numGPUs, selections); \
} else if (fullVectors->Dimension() <= 100) { \
getTailNeighborsTPT<Type, float, SUMTYPE, 100>((Type*)fullVectors->GetData(), fullVectors->Count(), this, exceptIDS, 100, replicaCount, numThreads, numTrees, leafSize, metric, numGPUs, selections); \
getTailNeighborsTPT<Type, float, SUMTYPE, 100>((Type*)fullVectors->GetData(), fullVectors->Count(), this, exceptIDS, fullVectors->Dimension(), replicaCount, numThreads, numTrees, leafSize, metric, numGPUs, selections); \
} else if (fullVectors->Dimension() <= 128) { \
getTailNeighborsTPT<Type, float, SUMTYPE, 128>((Type*)fullVectors->GetData(), fullVectors->Count(), this, exceptIDS, fullVectors->Dimension(), replicaCount, numThreads, numTrees, leafSize, metric, numGPUs, selections); \
} else if (fullVectors->Dimension() <= 200) { \
getTailNeighborsTPT<Type, float, SUMTYPE, 200>((Type*)fullVectors->GetData(), fullVectors->Count(), this, exceptIDS, fullVectors->Dimension(), replicaCount, numThreads, numTrees, leafSize, metric, numGPUs, selections); \
} else if (fullVectors->Dimension() <= 768) { \
getTailNeighborsTPT<Type, float, SUMTYPE, 768>((Type*)fullVectors->GetData(), fullVectors->Count(), this, exceptIDS, fullVectors->Dimension(), replicaCount, numThreads, numTrees, leafSize, metric, numGPUs, selections); \
} else if (fullVectors->Dimension() <= 1024) { \
getTailNeighborsTPT<Type, float, SUMTYPE, 1024>((Type*)fullVectors->GetData(), fullVectors->Count(), this, exceptIDS, fullVectors->Dimension(), replicaCount, numThreads, numTrees, leafSize, metric, numGPUs, selections); \
} else if (fullVectors->Dimension() <= 2048) { \
getTailNeighborsTPT<Type, float, SUMTYPE, 2048>((Type*)fullVectors->GetData(), fullVectors->Count(), this, exceptIDS, fullVectors->Dimension(), replicaCount, numThreads, numTrees, leafSize, metric, numGPUs, selections); \
} else if (fullVectors->Dimension() <= 4096) { \
getTailNeighborsTPT<Type, float, SUMTYPE, 4096>((Type*)fullVectors->GetData(), fullVectors->Count(), this, exceptIDS, fullVectors->Dimension(), replicaCount, numThreads, numTrees, leafSize, metric, numGPUs, selections); \
} else { \
LOG(Helper::LogLevel::LL_Error, "Datasets of >100 dimensions not currently supported for GPU Index build\n"); \
LOG(Helper::LogLevel::LL_Error, "Datasets of >768 dimensions not currently supported for GPU Index build\n"); \
exit(1); \
} \
break; \
@ -841,13 +853,31 @@ void VectorIndex::ApproximateRNG(std::shared_ptr<VectorSet>& fullVectors, std::u
typedef float SUMTYPE;
if (fullVectors->Dimension() <= 64) {
getTailNeighborsTPT<float, float, SUMTYPE, 64>((float*)fullVectors->GetData(), fullVectors->Count(), this, exceptIDS, 64, replicaCount, numThreads, numTrees, leafSize, metric, numGPUs, selections);
getTailNeighborsTPT<float, float, SUMTYPE, 64>((float*)fullVectors->GetData(), fullVectors->Count(), this, exceptIDS, fullVectors->Dimension(), replicaCount, numThreads, numTrees, leafSize, metric, numGPUs, selections);
}
else if (fullVectors->Dimension() <= 100) {
getTailNeighborsTPT<float, float, SUMTYPE, 100>((float*)fullVectors->GetData(), fullVectors->Count(), this, exceptIDS, 100, replicaCount, numThreads, numTrees, leafSize, metric, numGPUs, selections);
getTailNeighborsTPT<float, float, SUMTYPE, 100>((float*)fullVectors->GetData(), fullVectors->Count(), this, exceptIDS, fullVectors->Dimension(), replicaCount, numThreads, numTrees, leafSize, metric, numGPUs, selections);
}
else if (fullVectors->Dimension() <= 128) {
getTailNeighborsTPT<float, float, SUMTYPE, 128>((float*)fullVectors->GetData(), fullVectors->Count(), this, exceptIDS, fullVectors->Dimension(), replicaCount, numThreads, numTrees, leafSize, metric, numGPUs, selections);
}
else if (fullVectors->Dimension() <= 200) {
getTailNeighborsTPT<float, float, SUMTYPE, 200>((float*)fullVectors->GetData(), fullVectors->Count(), this, exceptIDS, fullVectors->Dimension(), replicaCount, numThreads, numTrees, leafSize, metric, numGPUs, selections);
}
else if (fullVectors->Dimension() <= 768) {
getTailNeighborsTPT<float, float, SUMTYPE, 768>((float*)fullVectors->GetData(), fullVectors->Count(), this, exceptIDS, fullVectors->Dimension(), replicaCount, numThreads, numTrees, leafSize, metric, numGPUs, selections);
}
else if (fullVectors->Dimension() <= 1024) {
getTailNeighborsTPT<float, float, SUMTYPE, 1024>((float*)fullVectors->GetData(), fullVectors->Count(), this, exceptIDS, fullVectors->Dimension(), replicaCount, numThreads, numTrees, leafSize, metric, numGPUs, selections);
}
else if (fullVectors->Dimension() <= 2048) {
getTailNeighborsTPT<float, float, SUMTYPE, 2048>((float*)fullVectors->GetData(), fullVectors->Count(), this, exceptIDS, fullVectors->Dimension(), replicaCount, numThreads, numTrees, leafSize, metric, numGPUs, selections);
}
else if (fullVectors->Dimension() <= 4096) {
getTailNeighborsTPT<float, float, SUMTYPE, 4096>((float*)fullVectors->GetData(), fullVectors->Count(), this, exceptIDS, fullVectors->Dimension(), replicaCount, numThreads, numTrees, leafSize, metric, numGPUs, selections);
}
else {
LOG(Helper::LogLevel::LL_Error, "Datasets of >100 dimensions not currently supported for GPU Index build\n");
LOG(Helper::LogLevel::LL_Error, "Datasets of >768 dimensions not currently supported for GPU Index build\n");
exit(1);
}
}

1
MANIFEST.in Normal file
Просмотреть файл

@ -0,0 +1 @@
recursive-include sptag *.py _SPTAG* Server.exe server

Просмотреть файл

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\packages\python.3.10.1\build\native\python.props" Condition="Exists('..\packages\python.3.10.1\build\native\python.props')" />
<Import Project="..\packages\python.3.9.11.1\build\native\python.props" Condition="Exists('..\packages\python.3.9.11.1\build\native\python.props')" />
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
<Configuration>Debug</Configuration>
@ -83,14 +83,9 @@
</PropertyGroup>
<ItemDefinitionGroup>
<Link>
<AdditionalDependencies>CoreLibrary.lib;SocketLib.lib;$(SolutionDir)packages\python.3.10.1\tools\libs\python310.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalDependencies>CoreLibrary.lib;SocketLib.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup>
<ClCompile>
<AdditionalIncludeDirectories>$(SolutionDir)packages\python.3.10.1\tools\include\;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
</ClCompile>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
@ -123,7 +118,6 @@
<PreprocessorDefinitions>_WINDLL;_SCL_SECURE_NO_WARNINGS;SWIG_PYTHON_INTERPRETER_NO_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<ControlFlowGuard>Guard</ControlFlowGuard>
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
<RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
</ClCompile>
<Link>
<AdditionalOptions>/guard:cf %(AdditionalOptions)</AdditionalOptions>
@ -181,11 +175,11 @@
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\packages\python.3.10.1\tools\python.exe')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\python.3.10.1\tools\python.exe'))" />
<Error Condition="!Exists('..\packages\swigwin.3.0.9\tools\swigwin-3.0.9\swig.exe')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\swigwin.3.0.9\tools\swigwin-3.0.9\swig.exe'))" />
<Error Condition="!Exists('..\packages\boost.1.72.0.0\build\boost.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\boost.1.72.0.0\build\boost.targets'))" />
<Error Condition="!Exists('..\packages\boost_system-vc142.1.72.0.0\build\boost_system-vc142.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\boost_system-vc142.1.72.0.0\build\boost_system-vc142.targets'))" />
<Error Condition="!Exists('..\packages\boost_date_time-vc142.1.72.0.0\build\boost_date_time-vc142.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\boost_date_time-vc142.1.72.0.0\build\boost_date_time-vc142.targets'))" />
<Error Condition="!Exists('..\packages\boost_regex-vc142.1.72.0.0\build\boost_regex-vc142.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\boost_regex-vc142.1.72.0.0\build\boost_regex-vc142.targets'))" />
<Error Condition="!Exists('..\packages\python.3.9.11.1\build\native\python.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\python.3.9.11.1\build\native\python.props'))" />
</Target>
</Project>

Просмотреть файл

@ -1,5 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\packages\python.3.9.11.1\build\native\python.props" Condition="Exists('..\packages\python.3.9.11.1\build\native\python.props')" />
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
<Configuration>Debug</Configuration>
@ -82,13 +83,12 @@
</PropertyGroup>
<ItemDefinitionGroup>
<Link>
<AdditionalDependencies>CoreLibrary.lib;$(SolutionDir)packages\python.3.10.1\tools\libs\python310.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalDependencies>CoreLibrary.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup>
<ClCompile>
<PreprocessorDefinitions Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">_WINDLL;SWIG_PYTHON_INTERPRETER_NO_DEBUG;_SCL_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>$(SolutionDir)packages\python.3.10.1\tools\include\;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<ControlFlowGuard Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">Guard</ControlFlowGuard>
<DebugInformationFormat Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">ProgramDatabase</DebugInformationFormat>
<PreprocessorDefinitions Condition="'$(Configuration)|$(Platform)'=='Release|x64'">_WINDLL;_SCL_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
@ -113,8 +113,7 @@
<None Include="packages.config" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
<ImportGroup Label="ExtensionTargets" />
<Target Name="BeforeBuild" BeforeTargets="PrepareForBuild">
<MakeDir Directories="$(IntDir)" />
<Exec Command="$(SolutionDir)packages\swigwin.3.0.9\tools\swigwin-3.0.9\swig.exe -python -c++ -I$(IntDir) -outdir $(IntDir) -o $(IntDir)CoreInterface_pwrap.cpp inc\PythonCore.i" />
@ -126,7 +125,7 @@
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\packages\python.3.10.1\tools\python.exe')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\python.3.10.1\tools\python.exe'))" />
<Error Condition="!Exists('..\packages\swigwin.3.0.9\tools\swigwin-3.0.9\swig.exe')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\swigwin.3.0.9\tools\swigwin-3.0.9\swig.exe'))" />
<Error Condition="!Exists('..\packages\python.3.9.11.1\build\native\python.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\python.3.9.11.1\build\native\python.props'))" />
</Target>
</Project>

Просмотреть файл

@ -7,6 +7,6 @@
<package id="boost_system-vc142" version="1.72.0.0" targetFramework="native" />
<package id="boost_thread-vc142" version="1.72.0.0" targetFramework="native" />
<package id="boost_wserialization-vc142" version="1.72.0.0" targetFramework="native" />
<package id="python" version="3.10.1" targetFramework="native" />
<package id="python" version="3.9.11.1" targetFramework="native" />
<package id="swigwin" version="3.0.9" targetFramework="native" />
</packages>

108
setup.py Normal file
Просмотреть файл

@ -0,0 +1,108 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT license.
"""
Script for installation and distribution.
You can use environment variable `SPTAG_RELEASE` to set release version.
If release version is not set, default to a development build whose version string will be `0.0.0.dev`.
## Prepare Environment ##
Install dependencies:
$ pip install -U -r setup.txt
## Development ##
Build and install for development:
$ python setup.py develop
Uninstall:
$ pip uninstall sptag
Remove generated files: (use "--all" to remove toolchain and built wheel)
$ python setup.py clean [--all]
## Release ##
Build wheel package:
$ SPTAG_RELEASE=1.0 python setup.py bdist_wheel -p win_amd64
Where "1.0" is version string and "win_amd64" is platform.
The platform may also be "manylinux1_x86_64".
"""
from distutils.cmd import Command
from distutils.command.build import build
from distutils.command.clean import clean
import glob
import os
import shutil
import sys
import setuptools
from setuptools.command.develop import develop
release = os.environ.get('SPTAG_RELEASE')
python_version = "%d.%d" % (sys.version_info.major, sys.version_info.minor)
print ("Python version:%s" % python_version)
def _setup():
setuptools.setup(
name = 'sptag',
version = release or '0.0.0.dev',
description = 'SPTAG: A library for fast approximate nearest neighbor search',
long_description = open('README.md', encoding='utf-8').read(),
long_description_content_type = 'text/markdown',
url = 'https://github.com/Microsoft/SPTAG',
author = 'Microsoft SPTAG Team',
author_email = 'cheqi@microsoft.com',
license = 'MIT',
include_package_data=True,
classifiers = [
'License :: OSI Approved :: MIT License',
'Operating System :: Microsoft :: Windows :: Windows 10',
'Operating System :: POSIX :: Linux',
'Programming Language :: Python :: 3',
'Intended Audience :: Science/Research',
],
packages = _find_python_packages(),
python_requires = '>=3.7',
install_requires = ['numpy'],
cmdclass = {
'build': Build,
'clean': Clean,
'develop': Develop,
}
)
def _find_python_packages():
if os.path.exists('sptag'): shutil.rmtree('sptag')
if os.path.exists('Release'):
shutil.copytree('Release', 'sptag')
elif os.path.exists(os.path.join('x64', 'Release')):
shutil.copytree(os.path.join('x64', 'Release'), 'sptag')
f = open(os.path.join('sptag', '__init__.py'), 'w')
f.close()
return ['sptag']
class Build(build):
def run(self):
if not release:
sys.exit('Please set environment variable "SPTAG_RELEASE=<release_version>"')
open('sptag/version.py', 'w').write(f"__version__ = '{release}'")
super().run()
class Develop(develop):
def run(self):
open('sptag/version.py', 'w').write("__version__ = '0.0.0.dev'")
super().run()
class Clean(clean):
def finalize_options(self):
self._all = self.all
self.all = True # always use `clean --all`
super().finalize_options()
def run(self):
super().run()
shutil.rmtree('sptag.egg-info', ignore_errors=True)
if self._all:
shutil.rmtree('dist', ignore_errors=True)
if __name__ == '__main__':
_setup()

3
setup.txt Normal file
Просмотреть файл

@ -0,0 +1,3 @@
pip < 21.4
setuptools < 61
wheel < 0.38