[demo] Allow chunk sizes to be configurable when deploying models (#126)
Co-authored-by: vaibhavarora102 <aroravaibhav102@gmail.com> Co-authored-by: Justin D. Harris <>
This commit is contained in:
Justin D. Harris
GitHub
Родитель
abbd3edade
Коммит
af3eab182d
|
|
@ -1,3 +1,4 @@
|
|||
.idea/
|
||||
|
||||
.vscode/
|
||||
|
||||
.venv/
|
||||
|
|
|
|||
|
|
@ -10,9 +10,9 @@ const { convertData, convertNum } = require('../float-utils-node')
|
|||
|
||||
const _toFloat = 1E9
|
||||
|
||||
async function deployDensePerceptron(model, web3, toFloat) {
|
||||
async function deployDensePerceptron(model, web3, options) {
|
||||
const { toFloat, initialChunkSize = 450, chunkSize = 450 } = options
|
||||
let gasUsed = 0
|
||||
const weightChunkSize = 450
|
||||
const { classifications } = model
|
||||
const weights = convertData(model.weights, web3, toFloat)
|
||||
const intercept = convertNum(model.intercept || model.bias, web3, toFloat)
|
||||
|
|
@ -23,15 +23,15 @@ async function deployDensePerceptron(model, web3, toFloat) {
|
|||
throw new Error("featureIndices are not supported yet.")
|
||||
}
|
||||
|
||||
console.log(` Deploying Dense Perceptron classifier with first ${Math.min(weights.length, weightChunkSize)} weights.`)
|
||||
const classifierContract = await DensePerceptron.new(classifications, weights.slice(0, weightChunkSize), intercept, learningRate)
|
||||
console.log(` Deploying Dense Perceptron classifier with first ${Math.min(weights.length, initialChunkSize)} weights.`)
|
||||
const classifierContract = await DensePerceptron.new(classifications, weights.slice(0, initialChunkSize), intercept, learningRate)
|
||||
gasUsed += (await web3.eth.getTransactionReceipt(classifierContract.transactionHash)).gasUsed
|
||||
|
||||
// Add remaining weights.
|
||||
for (let i = weightChunkSize; i < weights.length; i += weightChunkSize) {
|
||||
for (let i = initialChunkSize; i < weights.length; i += chunkSize) {
|
||||
// Do not parallelize so that weights are set in order.
|
||||
const r = await classifierContract.initializeWeights(weights.slice(i, i + weightChunkSize))
|
||||
console.debug(` Added classifier weights [${i}, ${Math.min(i + weightChunkSize, weights.length)}). gasUsed: ${r.receipt.gasUsed}`)
|
||||
const r = await classifierContract.initializeWeights(weights.slice(i, i + chunkSize))
|
||||
console.debug(` Added classifier weights [${i}, ${Math.min(i + chunkSize, weights.length)}). gasUsed: ${r.receipt.gasUsed}`)
|
||||
gasUsed += r.receipt.gasUsed
|
||||
}
|
||||
|
||||
|
|
@ -43,8 +43,8 @@ async function deployDensePerceptron(model, web3, toFloat) {
|
|||
}
|
||||
}
|
||||
|
||||
async function deploySparsePerceptron(model, web3, toFloat) {
|
||||
const weightChunkSize = 300
|
||||
async function deploySparsePerceptron(model, web3, options) {
|
||||
const { toFloat, initialChunkSize = 300, chunkSize = 300 } = options
|
||||
const { classifications } = model
|
||||
const weights = convertData(model.weights, web3, toFloat)
|
||||
const intercept = convertNum(model.intercept || model.bias, web3, toFloat)
|
||||
|
|
@ -63,19 +63,19 @@ async function deploySparsePerceptron(model, web3, toFloat) {
|
|||
}
|
||||
}
|
||||
|
||||
console.log(` Deploying Sparse Perceptron classifier with first ${Math.min(weights.length, weightChunkSize)} weights...`)
|
||||
const classifierContract = await SparsePerceptron.new(classifications, weights.slice(0, weightChunkSize), intercept, learningRate)
|
||||
console.log(` Deploying Sparse Perceptron classifier with first ${Math.min(weights.length, initialChunkSize)} weights...`)
|
||||
const classifierContract = await SparsePerceptron.new(classifications, weights.slice(0, initialChunkSize), intercept, learningRate)
|
||||
let gasUsed = (await web3.eth.getTransactionReceipt(classifierContract.transactionHash)).gasUsed
|
||||
console.log(` Deployed Sparse Perceptron classifier with first ${Math.min(weights.length, weightChunkSize)} weights. gasUsed: ${gasUsed}`)
|
||||
console.log(` Deployed Sparse Perceptron classifier with first ${Math.min(weights.length, initialChunkSize)} weights. gasUsed: ${gasUsed}`)
|
||||
|
||||
// Add remaining weights.
|
||||
for (let i = weightChunkSize; i < weights.length; i += weightChunkSize) {
|
||||
const r = await classifierContract.initializeWeights(i, weights.slice(i, i + weightChunkSize))
|
||||
console.debug(` Added classifier weights [${i}, ${Math.min(i + weightChunkSize, weights.length)}) gasUsed: ${r.receipt.gasUsed}`)
|
||||
for (let i = initialChunkSize; i < weights.length; i += chunkSize) {
|
||||
const r = await classifierContract.initializeWeights(i, weights.slice(i, i + chunkSize))
|
||||
console.debug(` Added classifier weights [${i}, ${Math.min(i + chunkSize, weights.length)}) gasUsed: ${r.receipt.gasUsed}`)
|
||||
gasUsed += r.receipt.gasUsed
|
||||
}
|
||||
|
||||
const sparseWeightsChunkSize = Math.round(weightChunkSize / 2)
|
||||
const sparseWeightsChunkSize = Math.round(chunkSize / 2)
|
||||
for (let i = 0; i < sparseWeights.length; i += sparseWeightsChunkSize) {
|
||||
const r = await classifierContract.initializeSparseWeights(
|
||||
sparseWeights.slice(i, i + sparseWeightsChunkSize))
|
||||
|
|
@ -91,7 +91,8 @@ async function deploySparsePerceptron(model, web3, toFloat) {
|
|||
}
|
||||
}
|
||||
|
||||
async function deployNearestCentroidClassifier(model, web3, toFloat) {
|
||||
async function deployNearestCentroidClassifier(model, web3, options) {
|
||||
const { toFloat } = options
|
||||
let gasUsed = 0
|
||||
const classifications = []
|
||||
const centroids = []
|
||||
|
|
@ -135,10 +136,9 @@ async function deployNearestCentroidClassifier(model, web3, toFloat) {
|
|||
})
|
||||
}
|
||||
|
||||
exports.deploySparseNearestCentroidClassifier = async function (model, web3, toFloat) {
|
||||
exports.deploySparseNearestCentroidClassifier = async function (model, web3, options) {
|
||||
const { toFloat, initialChunkSize = 200, chunkSize = 250 } = options
|
||||
let gasUsed = 0
|
||||
const initialChunkSize = 200
|
||||
const chunkSize = 250
|
||||
const classifications = []
|
||||
const centroids = []
|
||||
const dataCounts = []
|
||||
|
|
@ -190,20 +190,19 @@ exports.deploySparseNearestCentroidClassifier = async function (model, web3, toF
|
|||
})
|
||||
}
|
||||
|
||||
async function deployNaiveBayes(model, web3, toFloat) {
|
||||
async function deployNaiveBayes(model, web3, options) {
|
||||
const { toFloat, initialChunkSize = 150, chunkSize = 350 } = options
|
||||
let gasUsed = 0
|
||||
const initialFeatureChunkSize = 150
|
||||
const featureChunkSize = 350
|
||||
const { classifications, classCounts, featureCounts, totalNumFeatures } = model
|
||||
const smoothingFactor = convertNum(model.smoothingFactor, web3, toFloat)
|
||||
console.log(` Deploying Naive Bayes classifier.`)
|
||||
const classifierContract = await NaiveBayesClassifier.new([classifications[0]], [classCounts[0]], [featureCounts[0].slice(0, initialFeatureChunkSize)], totalNumFeatures, smoothingFactor)
|
||||
const classifierContract = await NaiveBayesClassifier.new([classifications[0]], [classCounts[0]], [featureCounts[0].slice(0, initialChunkSize)], totalNumFeatures, smoothingFactor)
|
||||
gasUsed += (await web3.eth.getTransactionReceipt(classifierContract.transactionHash)).gasUsed
|
||||
|
||||
const addClassPromises = []
|
||||
for (let i = 1; i < classifications.length; ++i) {
|
||||
addClassPromises.push(classifierContract.addClass(
|
||||
classCounts[i], featureCounts[i].slice(0, initialFeatureChunkSize), classifications[i]
|
||||
classCounts[i], featureCounts[i].slice(0, initialChunkSize), classifications[i]
|
||||
).then(r => {
|
||||
console.debug(` Added class ${i}. gasUsed: ${r.receipt.gasUsed}`)
|
||||
return r
|
||||
|
|
@ -216,11 +215,11 @@ async function deployNaiveBayes(model, web3, toFloat) {
|
|||
// Add remaining feature counts.
|
||||
// Tried with promises but got weird unhelpful errors from Truffle (some were like network timeout errors).
|
||||
for (let classification = 0; classification < classifications.length; ++classification) {
|
||||
for (let j = initialFeatureChunkSize; j < featureCounts[classification].length; j += featureChunkSize) {
|
||||
for (let j = initialChunkSize; j < featureCounts[classification].length; j += chunkSize) {
|
||||
const r = await classifierContract.initializeCounts(
|
||||
featureCounts[classification].slice(j, j + featureChunkSize), classification
|
||||
featureCounts[classification].slice(j, j + chunkSize), classification
|
||||
)
|
||||
console.debug(` Added features [${j}, ${Math.min(j + featureChunkSize, featureCounts[classification].length)}) for class ${classification}. gasUsed: ${r.receipt.gasUsed}`)
|
||||
console.debug(` Added features [${j}, ${Math.min(j + chunkSize, featureCounts[classification].length)}) for class ${classification}. gasUsed: ${r.receipt.gasUsed}`)
|
||||
gasUsed += r.receipt.gasUsed
|
||||
}
|
||||
}
|
||||
|
|
@ -237,22 +236,25 @@ async function deployNaiveBayes(model, web3, toFloat) {
|
|||
* @returns The contract for the model, an instance of `Classifier64`
|
||||
* along with the the total amount of gas used to deploy the model.
|
||||
*/
|
||||
exports.deployModel = async function (model, web3, toFloat = _toFloat) {
|
||||
exports.deployModel = async function (model, web3, options) {
|
||||
if (typeof model === 'string') {
|
||||
model = JSON.parse(fs.readFileSync(model, 'utf8'))
|
||||
}
|
||||
if (options.toFloat === undefined) {
|
||||
options.toFloat = _toFloat
|
||||
}
|
||||
switch (model.type) {
|
||||
case 'dense perceptron':
|
||||
return deployDensePerceptron(model, web3, toFloat)
|
||||
return deployDensePerceptron(model, web3, options)
|
||||
case 'naive bayes':
|
||||
return deployNaiveBayes(model, web3, toFloat)
|
||||
return deployNaiveBayes(model, web3, options)
|
||||
case 'dense nearest centroid classifier':
|
||||
case 'nearest centroid classifier':
|
||||
return deployNearestCentroidClassifier(model, web3, toFloat)
|
||||
return deployNearestCentroidClassifier(model, web3, options)
|
||||
case 'sparse nearest centroid classifier':
|
||||
return exports.deploySparseNearestCentroidClassifier(model, web3, toFloat)
|
||||
return exports.deploySparseNearestCentroidClassifier(model, web3, options)
|
||||
case 'sparse perceptron':
|
||||
return deploySparsePerceptron(model, web3, toFloat)
|
||||
return deploySparsePerceptron(model, web3, options)
|
||||
default:
|
||||
// Should not happen.
|
||||
throw new Error(`Unrecognized model type: "${model.type}"`)
|
||||
|
|
|
|||
|
|
@ -36,11 +36,10 @@ export class ModelDeployer {
|
|||
const { account, toFloat,
|
||||
notify, dismissNotification,
|
||||
saveTransactionHash, saveAddress,
|
||||
initialChunkSize = 150, chunkSize = 350,
|
||||
} = options
|
||||
|
||||
const defaultSmoothingFactor = 1
|
||||
const initialFeatureChunkSize = 150
|
||||
const featureChunkSize = 350
|
||||
const { classifications, classCounts, featureCounts, totalNumFeatures } = model
|
||||
const smoothingFactor = convertNum(model.smoothingFactor || defaultSmoothingFactor, this.web3, toFloat)
|
||||
|
||||
|
|
@ -50,7 +49,7 @@ export class ModelDeployer {
|
|||
|
||||
return contract.deploy({
|
||||
data: ContractInfo.bytecode,
|
||||
arguments: [[classifications[0]], [classCounts[0]], [featureCounts[0].slice(0, initialFeatureChunkSize)], totalNumFeatures, smoothingFactor]
|
||||
arguments: [[classifications[0]], [classCounts[0]], [featureCounts[0].slice(0, initialChunkSize)], totalNumFeatures, smoothingFactor]
|
||||
}).send({
|
||||
from: account,
|
||||
gas: this.gasLimit,
|
||||
|
|
@ -68,7 +67,7 @@ export class ModelDeployer {
|
|||
addClassPromises.push(new Promise((resolve, reject) => {
|
||||
const notification = notify(`Please accept the prompt to create the "${classifications[i]}" class`)
|
||||
newContractInstance.methods.addClass(
|
||||
classCounts[i], featureCounts[i].slice(0, initialFeatureChunkSize), classifications[i]
|
||||
classCounts[i], featureCounts[i].slice(0, initialChunkSize), classifications[i]
|
||||
).send({
|
||||
from: account,
|
||||
// Block gas limit by most miners as of October 2019.
|
||||
|
|
@ -86,14 +85,14 @@ export class ModelDeployer {
|
|||
return Promise.all(addClassPromises).then(async _ => {
|
||||
// Add remaining feature counts.
|
||||
for (let classification = 0; classification < classifications.length; ++classification) {
|
||||
for (let j = initialFeatureChunkSize; j < featureCounts[classification].length; j += featureChunkSize) {
|
||||
const notification = notify(`Please accept the prompt to upload the features [${j},${Math.min(j + featureChunkSize, featureCounts[classification].length)}) for the "${classifications[classification]}" class`)
|
||||
for (let j = initialChunkSize; j < featureCounts[classification].length; j += chunkSize) {
|
||||
const notification = notify(`Please accept the prompt to upload the features [${j},${Math.min(j + chunkSize, featureCounts[classification].length)}) for the "${classifications[classification]}" class`)
|
||||
await newContractInstance.methods.initializeCounts(
|
||||
featureCounts[classification].slice(j, j + featureChunkSize), classification).send().on('transactionHash', () => {
|
||||
featureCounts[classification].slice(j, j + chunkSize), classification).send().on('transactionHash', () => {
|
||||
dismissNotification(notification)
|
||||
}).on('error', (err: any) => {
|
||||
dismissNotification(notification)
|
||||
notify(`Error setting feature indices for [${j},${Math.min(j + featureChunkSize, featureCounts[classification].length)}) for the "${classifications[classification]}" class`, { variant: 'error' })
|
||||
notify(`Error setting feature indices for [${j},${Math.min(j + chunkSize, featureCounts[classification].length)}) for the "${classifications[classification]}" class`, { variant: 'error' })
|
||||
throw err
|
||||
})
|
||||
}
|
||||
|
|
@ -109,9 +108,9 @@ export class ModelDeployer {
|
|||
const { account, toFloat,
|
||||
notify, dismissNotification,
|
||||
saveTransactionHash, saveAddress,
|
||||
initialChunkSize = 200, chunkSize = 250,
|
||||
} = options
|
||||
const initialChunkSize = 200
|
||||
const chunkSize = 250
|
||||
|
||||
const classifications: string[] = []
|
||||
const centroids: number[][] | number[][][] = []
|
||||
const dataCounts: number[] = []
|
||||
|
|
@ -203,9 +202,10 @@ export class ModelDeployer {
|
|||
const { account, toFloat,
|
||||
notify, dismissNotification,
|
||||
saveTransactionHash, saveAddress,
|
||||
chunkSize = 350,
|
||||
} = options
|
||||
const defaultLearningRate = 0.5
|
||||
const weightChunkSize = 300
|
||||
|
||||
const { classifications, featureIndices } = model
|
||||
let weightsArray: any[] = []
|
||||
let sparseWeights: any[][] = []
|
||||
|
|
@ -225,17 +225,16 @@ export class ModelDeployer {
|
|||
}
|
||||
const intercept = convertNum(model.intercept, this.web3, toFloat)
|
||||
const learningRate = convertNum(model.learningRate || defaultLearningRate, this.web3, toFloat)
|
||||
|
||||
if (featureIndices !== undefined && featureIndices.length !== weightsArray.length + sparseWeights.length) {
|
||||
return Promise.reject("The number of features must match the number of weights.")
|
||||
}
|
||||
|
||||
const ContractInfo = ModelDeployer.modelTypes[model.type]
|
||||
const contract = new this.web3.eth.Contract(ContractInfo.abi, undefined, { from: account })
|
||||
const pleaseAcceptKey = notify(`Please accept the prompt to deploy the Perceptron classifier with the first ${Math.min(weightsArray.length, weightChunkSize)} weights`)
|
||||
const pleaseAcceptKey = notify(`Please accept the prompt to deploy the Perceptron classifier with the first ${Math.min(weightsArray.length, chunkSize)} weights`)
|
||||
return contract.deploy({
|
||||
data: ContractInfo.bytecode,
|
||||
arguments: [classifications, weightsArray.slice(0, weightChunkSize), intercept, learningRate],
|
||||
arguments: [classifications, weightsArray.slice(0, chunkSize), intercept, learningRate],
|
||||
}).send({
|
||||
from: account,
|
||||
gas: this.gasLimit,
|
||||
|
|
@ -249,20 +248,20 @@ export class ModelDeployer {
|
|||
console.error(err)
|
||||
}).then(async newContractInstance => {
|
||||
// Add remaining weights.
|
||||
for (let i = weightChunkSize; i < weightsArray.length; i += weightChunkSize) {
|
||||
for (let i = chunkSize; i < weightsArray.length; i += chunkSize) {
|
||||
// Not parallel since order matters for the dense model.
|
||||
// Even for the sparse model, it nice not to be bombarded with many notifications that can look out of order.
|
||||
let transaction: any
|
||||
if (model.type === 'dense perceptron' || model.type === 'perceptron') {
|
||||
transaction = newContractInstance.methods.initializeWeights(weightsArray.slice(i, i + weightChunkSize))
|
||||
transaction = newContractInstance.methods.initializeWeights(weightsArray.slice(i, i + chunkSize))
|
||||
} else if (model.type === 'sparse perceptron') {
|
||||
transaction = newContractInstance.methods.initializeWeights(i, weightsArray.slice(i, i + weightChunkSize))
|
||||
transaction = newContractInstance.methods.initializeWeights(i, weightsArray.slice(i, i + chunkSize))
|
||||
} else {
|
||||
throw new Error(`Unrecognized model type: "${model.type}"`)
|
||||
}
|
||||
// Subtract 1 from the count because the first chunk has already been uploaded.
|
||||
const notification = notify(`Please accept the prompt to upload classifier
|
||||
weights [${i},${Math.min(i + weightChunkSize, weightsArray.length)}) (${i / weightChunkSize}/${Math.ceil(weightsArray.length / weightChunkSize) - 1})`)
|
||||
weights [${i},${Math.min(i + chunkSize, weightsArray.length)}) (${i / chunkSize}/${Math.ceil(weightsArray.length / chunkSize) - 1})`)
|
||||
await transaction.send({
|
||||
from: account,
|
||||
gas: this.gasLimit,
|
||||
|
|
@ -270,28 +269,28 @@ export class ModelDeployer {
|
|||
dismissNotification(notification)
|
||||
}).on('error', (err: any) => {
|
||||
dismissNotification(notification)
|
||||
notify(`Error setting weights classifier weights [${i},${Math.min(i + weightChunkSize, weightsArray.length)})`, { variant: 'error' })
|
||||
notify(`Error setting weights classifier weights [${i},${Math.min(i + chunkSize, weightsArray.length)})`, { variant: 'error' })
|
||||
console.error(err)
|
||||
})
|
||||
}
|
||||
if (featureIndices !== undefined) {
|
||||
// Add feature indices to use.
|
||||
for (let i = 0; i < featureIndices.length; i += weightChunkSize) {
|
||||
const notification = notify(`Please accept the prompt to upload the feature indices [${i},${Math.min(i + weightChunkSize, featureIndices.length)})`)
|
||||
await newContractInstance.methods.addFeatureIndices(featureIndices.slice(i, i + weightChunkSize)).send({
|
||||
for (let i = 0; i < featureIndices.length; i += chunkSize) {
|
||||
const notification = notify(`Please accept the prompt to upload the feature indices [${i},${Math.min(i + chunkSize, featureIndices.length)})`)
|
||||
await newContractInstance.methods.addFeatureIndices(featureIndices.slice(i, i + chunkSize)).send({
|
||||
from: account,
|
||||
gas: this.gasLimit,
|
||||
}).on('transactionHash', () => {
|
||||
dismissNotification(notification)
|
||||
}).on('error', (err: any) => {
|
||||
dismissNotification(notification)
|
||||
notify(`Error setting feature indices for [${i},${Math.min(i + weightChunkSize, featureIndices.length)})`, { variant: 'error' })
|
||||
notify(`Error setting feature indices for [${i},${Math.min(i + chunkSize, featureIndices.length)})`, { variant: 'error' })
|
||||
console.error(err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const sparseWeightsChunkSize = Math.round(weightChunkSize / 2)
|
||||
const sparseWeightsChunkSize = Math.round(chunkSize / 2)
|
||||
for (let i = 0; i < sparseWeights.length; i += sparseWeightsChunkSize) {
|
||||
const notification = notify(`Please accept the prompt to upload sparse classifier weights [${i},${Math.min(i + sparseWeightsChunkSize, sparseWeights.length)}) out of ${sparseWeights.length}`)
|
||||
await newContractInstance.methods.initializeSparseWeights(sparseWeights.slice(i, i + sparseWeightsChunkSize)).send({
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ contract('CheckGasUsage', function (accounts) {
|
|||
gasUsed += (await web3.eth.getTransactionReceipt(dataHandler.transactionHash)).gasUsed
|
||||
console.log(` Deployed data handler to ${dataHandler.address}. Total gasUsed: ${gasUsed}.`)
|
||||
|
||||
const classifierInfo = await deployModel(modelPath, web3, toFloat)
|
||||
const classifierInfo = await deployModel(modelPath, web3, { toFloat })
|
||||
const classifier = classifierInfo.classifierContract
|
||||
gasUsed += classifierInfo.gasUsed
|
||||
console.log(" Deploying Incentive Mechanism.")
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ contract('SparseNearestCentroidClassifier', function (accounts) {
|
|||
}
|
||||
}
|
||||
}
|
||||
classifier = (await deploySparseNearestCentroidClassifier(model, web3, toFloat)).classifierContract
|
||||
classifier = (await deploySparseNearestCentroidClassifier(model, web3, { toFloat })).classifierContract
|
||||
})
|
||||
|
||||
it("...should get the classifications", function () {
|
||||
|
|
|
|||
|
|
@ -273,7 +273,7 @@ contract('CollaborativeTrainer with Perceptron', function (accounts) {
|
|||
intercept: 2,
|
||||
learningRate: 0.5,
|
||||
}
|
||||
const { classifierContract } = await deployModel(model, web3, toFloat)
|
||||
const { classifierContract } = await deployModel(model, web3, { toFloat })
|
||||
assert.equal(await classifierContract.intercept().then(parseFloatBN), model.intercept)
|
||||
assert.equal(await classifierContract.learningRate().then(parseFloatBN), model.learningRate)
|
||||
for (let i = 0; i < model.weights; ++i) {
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче