diff endpoints + added version in ServerInfo (#235)

.circleci/build.sh

@@ -10,7 +10,7 @@ if [[ "$CIRCLE_TAG" =~ ^v.* ]]; then
     IMAGE_VERSION_TAG=$CIRCLE_TAG
 fi
 
-docker build -t $DOCKER_IMAGE_TAG:latest . -f packages/$SPECKLE_SERVER_PACKAGE/Dockerfile
+docker build --build-arg SPECKLE_SERVER_VERSION=$IMAGE_VERSION_TAG -t $DOCKER_IMAGE_TAG:latest . -f packages/$SPECKLE_SERVER_PACKAGE/Dockerfile
 docker tag $DOCKER_IMAGE_TAG:latest $DOCKER_IMAGE_TAG:$IMAGE_VERSION_TAG
 
 echo "$DOCKER_REG_PASS" | docker login -u "$DOCKER_REG_USER" --password-stdin $DOCKER_REG_URL

packages/frontend/nginx/nginx.conf

@@ -7,7 +7,7 @@ server {
     try_files $uri $uri/ /app.html;
   }
 
-  location ~* ^/(graphql|explorer|(auth/.*)|(objects/.*)|(preview/.*)) {
+  location ~* ^/(graphql|explorer|(auth/.*)|(objects/.*)|(preview/.*)|(api/.*)) {
     resolver 127.0.0.11 valid=30s;
     set $upstream_speckle_server speckle-server;
     client_max_body_size 100m;

packages/server/Dockerfile

@@ -9,6 +9,9 @@ RUN chmod +x /wait
 
 ARG NODE_ENV=production
 ENV NODE_ENV=${NODE_ENV}
+
+ARG SPECKLE_SERVER_VERSION=custom
+ENV SPECKLE_SERVER_VERSION=${SPECKLE_SERVER_VERSION}
 WORKDIR /app
 
 COPY packages/server/package*.json ./

packages/server/modules/core/graph/schemas/server.graphql

@@ -15,6 +15,7 @@ type ServerInfo {
   roles: [Role]!
   scopes: [Scope]!
   inviteOnly: Boolean
+  version: String
 }
 
 """

packages/server/modules/core/index.js

@@ -11,6 +11,10 @@ exports.init = async ( app, options ) => {
   require( './rest/upload' )( app )
   require( './rest/download' )( app )
 
+  // Initialises the two diff-based upload/download endpoints
+  require( './rest/diffUpload' )( app )
+  require( './rest/diffDownload' )( app )
+  
   // Register core-based scoeps
   const scopes = require( './scopes.js' )
   for ( let scope of scopes ) {

packages/server/modules/core/rest/authUtils.js

@@ -0,0 +1,55 @@
+'use strict'
+const appRoot = require( 'app-root-path' )
+const { contextMiddleware, validateScopes, authorizeResolver } = require( `${appRoot}/modules/shared` )
+
+const { getStream } = require( '../services/streams' )
+
+module.exports = {
+  async validatePermissionsReadStream( streamId, req ) {
+    const stream = await getStream( { streamId: streamId, userId: req.context.userId } )
+
+    if ( !stream ) {
+      return { result: false, status: 404 }
+    }
+
+    if ( !stream.isPublic && req.context.auth === false ) {
+      return { result: false, status: 401 }
+    }
+
+    if ( !stream.isPublic ) {
+      try {
+        await validateScopes( req.context.scopes, 'streams:read' )
+      } catch ( err ) {
+        return { result: false, status: 401 }
+      }
+
+      try {
+        await authorizeResolver( req.context.userId, streamId, 'stream:reviewer' )
+      } catch ( err ) {
+        return { result: false, status: 401 }
+      }
+    }
+
+    return { result: true, status: 200 }
+  },
+
+  async validatePermissionsWriteStream( streamId, req ) {
+    if ( !req.context || !req.context.auth ) {
+      return { result: false, status: 401 }
+    }
+
+    try {
+      await validateScopes( req.context.scopes, 'streams:write' )
+    } catch ( err ) {
+      return { result: false, status: 401 }
+    }
+
+    try {
+      await authorizeResolver( req.context.userId, streamId, 'stream:contributor' )
+    } catch ( err ) {
+      return { result: false, status: 401 }
+    }
+    
+    return { result: true, status: 200 }
+  }
+}

packages/server/modules/core/rest/diffDownload.js

@@ -0,0 +1,96 @@
+'use strict'
+const zlib = require( 'zlib' )
+const Busboy = require( 'busboy' )
+const debug = require( 'debug' )
+const appRoot = require( 'app-root-path' )
+const cors = require( 'cors' )
+
+const { matomoMiddleware } = require( `${appRoot}/logging/matomoHelper` )
+const { contextMiddleware, validateScopes, authorizeResolver } = require( `${appRoot}/modules/shared` )
+const { validatePermissionsReadStream } = require( './authUtils' )
+
+const { getObjectsStream } = require( '../services/objects' )
+
+module.exports = ( app ) => {
+
+  app.options( '/api/getobjects/:streamId', cors() )
+  app.post( '/api/getobjects/:streamId', cors(), contextMiddleware, matomoMiddleware, async ( req, res ) => {
+    let hasStreamAccess = await validatePermissionsReadStream( req.params.streamId, req )
+    if ( !hasStreamAccess.result ) {
+      return res.status( hasStreamAccess.status ).end()
+    }
+
+    let childrenList = JSON.parse( req.body.objects )
+
+    let simpleText = req.headers.accept === 'text/plain'
+
+    let dbStream = await getObjectsStream( { streamId: req.params.streamId, objectIds: childrenList } )
+
+    let currentChunkSize = 0
+    let maxChunkSize = 50000
+    let chunk = simpleText ? '' : [ ]
+    let isFirst = true
+
+
+    res.writeHead( 200, { 'Content-Encoding': 'gzip', 'Content-Type': simpleText ? 'text/plain' : 'application/json' } )
+
+    const gzip = zlib.createGzip( )
+
+    if ( !simpleText ) gzip.write( '[' )
+
+    // helper func to flush the gzip buffer
+    const writeBuffer = ( addTrailingComma ) => {
+      // console.log( `writing buff ${currentChunkSize}` )
+      if ( simpleText ) {
+        gzip.write( chunk )
+      } else {
+        gzip.write( chunk.join( ',' ) )
+        if ( addTrailingComma ) {
+          gzip.write( ',' )
+        }
+      }
+      gzip.flush( )
+      chunk = simpleText ? '' : [ ]
+    }
+
+    let k = 0
+    let requestDropped = false
+    dbStream.on( 'data', row => {
+      try {
+        let data = JSON.stringify( row.data )
+        currentChunkSize += Buffer.byteLength( data, 'utf8' )
+        if ( simpleText ) {
+          chunk += `${row.data.id}\t${data}\n`
+        } else {
+          chunk.push( data )
+        }
+        if ( currentChunkSize >= maxChunkSize ) {
+          currentChunkSize = 0
+          writeBuffer( true )
+        }
+        k++
+      } catch ( e ) {
+        requestDropped = true
+        debug( 'speckle:error' )( `'Failed to find object, or object is corrupted.' ${req.params.objectId}` )
+        return
+      }
+    } )
+
+    dbStream.on( 'error', err => {
+      debug( 'speckle:error' )( `Error in streaming object children for ${req.params.objectId}: ${err}` )
+      requestDropped = true
+      return
+    } )
+
+    dbStream.on( 'end', ( ) => {
+      if ( currentChunkSize !== 0 ) {
+        writeBuffer( false )
+        if ( !simpleText ) gzip.write( ']' )
+      }
+      gzip.end( )
+    } )
+
+    // 🚬
+    gzip.pipe( res )
+  } )
+}

packages/server/modules/core/rest/diffUpload.js

@@ -0,0 +1,31 @@
+'use strict'
+const zlib = require( 'zlib' )
+const Busboy = require( 'busboy' )
+const debug = require( 'debug' )
+const appRoot = require( 'app-root-path' )
+
+const { matomoMiddleware } = require( `${appRoot}/logging/matomoHelper` )
+const { contextMiddleware } = require( `${appRoot}/modules/shared` )
+const { validatePermissionsWriteStream } = require( './authUtils' )
+
+const { hasObjects } = require( '../services/objects' )
+
+module.exports = ( app ) => {
+  app.post( '/api/diff/:streamId', contextMiddleware, matomoMiddleware, async ( req, res ) => {
+    let hasStreamAccess = await validatePermissionsWriteStream( req.params.streamId, req )
+    if ( !hasStreamAccess.result ) {
+      return res.status( hasStreamAccess.status ).end()
+    }
+    
+    let objectList = JSON.parse( req.body.objects )
+
+    let response = await hasObjects( { streamId: req.params.streamId, objectIds: objectList } )
+    // console.log(response)
+    res.writeHead( 200, { 'Content-Encoding': 'gzip', 'Content-Type': 'application/json' } )
+    const gzip = zlib.createGzip( )
+    gzip.write( JSON.stringify( response ) )
+    gzip.flush( )
+    gzip.end( )
+    gzip.pipe( res )
+  } )
+}

packages/server/modules/core/rest/download.js

@@ -6,38 +6,19 @@ const appRoot = require( 'app-root-path' )
 const cors = require( 'cors' )
 
 const { matomoMiddleware } = require( `${appRoot}/logging/matomoHelper` )
-const { contextMiddleware, validateScopes, authorizeResolver } = require( `${appRoot}/modules/shared` )
+const { contextMiddleware } = require( `${appRoot}/modules/shared` )
+const { validatePermissionsReadStream } = require( './authUtils' )
+
 const { getObject, getObjectChildrenStream } = require( '../services/objects' )
-const { getStream } = require( '../services/streams' )
 
 module.exports = ( app ) => {
 
   app.options( '/objects/:streamId/:objectId', cors() )
 
   app.get( '/objects/:streamId/:objectId', cors(), contextMiddleware, matomoMiddleware, async ( req, res ) => {
-
-    const stream = await getStream( { streamId: req.params.streamId, userId: req.context.userId } )
-
-    if ( !stream ) {
-      return res.status( 404 ).end()
-    }
-
-    if ( !stream.isPublic && req.context.auth === false ) {
-      return res.status( 401 ).end( )
-    }
-
-    if ( !stream.isPublic ) {
-      try {
-        await validateScopes( req.context.scopes, 'streams:read' )
-      } catch ( err ) {
-        return res.status( 401 ).end( )
-      }
-
-      try {
-        await authorizeResolver( req.context.userId, req.params.streamId, 'stream:reviewer' )
-      } catch ( err ) {
-        return res.status( 401 ).end( )
-      }
+    let hasStreamAccess = await validatePermissionsReadStream( req.params.streamId, req )
+    if ( !hasStreamAccess.result ) {
+      return res.status( hasStreamAccess.status ).end()
     }
 
     // Populate first object (the "commit")
@@ -129,12 +110,15 @@ module.exports = ( app ) => {
     gzip.pipe( res )
   } )
 
-  // TODO: is this needed/used?
-  app.get( '/objects/:streamId/:objectId/single', async ( req, res ) => {
-    // TODO: authN & authZ checks
+  app.options( '/objects/:streamId/:objectId/single', cors() )
+  app.get( '/objects/:streamId/:objectId/single', cors(), contextMiddleware, matomoMiddleware, async ( req, res ) => {
+    let hasStreamAccess = await validatePermissionsReadStream( req.params.streamId, req )
+    if ( !hasStreamAccess.result ) {
+      return res.status( hasStreamAccess.status ).end()
+    }
 
-    let obj = await getObject( req.params.streamId, req.params.objectId )
+    let obj = await getObject( { streamId: req.params.streamId, objectId: req.params.objectId } )
 
-    res.send( obj )
+    res.send( obj.data )
   } )
 }

packages/server/modules/core/rest/upload.js

@@ -5,29 +5,17 @@ const debug = require( 'debug' )
 const appRoot = require( 'app-root-path' )
 
 const { matomoMiddleware } = require( `${appRoot}/logging/matomoHelper` )
-const { contextMiddleware, validateScopes, authorizeResolver } = require( `${appRoot}/modules/shared` )
+const { contextMiddleware } = require( `${appRoot}/modules/shared` )
+const { validatePermissionsWriteStream } = require( './authUtils' )
 
 const { createObjects, createObjectsBatched } = require( '../services/objects' )
 
+
 module.exports = ( app ) => {
   app.post( '/objects/:streamId', contextMiddleware, matomoMiddleware, async ( req, res ) => {
-
-    debug( 'speckle:upload-endpoint' )( 'booom upload endpoint' )
-
-    if ( !req.context || !req.context.auth ) {
-      return res.status( 401 ).end( )
-    }
-
-    try {
-      await validateScopes( req.context.scopes, 'streams:write' )
-    } catch ( err ) {
-      return res.status( 401 ).end( )
-    }
-
-    try {
-      await authorizeResolver( req.context.userId, req.params.streamId, 'stream:contributor' )
-    } catch ( err ) {
-      return res.status( 401 ).end( )
+    let hasStreamAccess = await validatePermissionsWriteStream( req.params.streamId, req )
+    if ( !hasStreamAccess.result ) {
+      return res.status( hasStreamAccess.status ).end()
     }
 
     debug( 'speckle:upload-endpoint' )( 'Upload started' )

packages/server/modules/core/services/generic.js

@@ -9,9 +9,9 @@ const Info = ( ) => knex( 'server_config' )
 module.exports = {
 
   async getServerInfo( ) {
-
-    return await Info( ).select( '*' ).first( )
-
+    let serverInfo = await Info( ).select( '*' ).first( )
+    serverInfo.version = process.env.SPECKLE_SERVER_VERSION || 'dev'
+    return serverInfo
   },
 
   async getAllScopes( ) {

packages/server/modules/core/services/objects.js

@@ -436,6 +436,31 @@ module.exports = {
     return res
   },
 
+  async getObjectsStream( { streamId, objectIds } ) {
+    let res = Objects( )
+      .whereIn( 'id', objectIds )
+      .andWhere( 'streamId', streamId )
+      .orderBy( 'id' )
+      .select( 'id', 'speckleType', 'totalChildrenCount', 'totalChildrenCountByDepth', 'createdAt', 'data' )
+    return res.stream( )
+  },
+
+  async hasObjects( { streamId, objectIds } ) {
+    let dbRes = await Objects( )
+      .whereIn( 'id', objectIds )
+      .andWhere( 'streamId', streamId )
+      .select( 'id' )
+
+    let res = {}
+    for ( let i in objectIds ) {
+      res[ objectIds[ i ] ] = false
+    }
+    for ( let i in dbRes ) {
+      res [ dbRes[ i ].id ] = true
+    }
+    return res
+  },
+  
   // NOTE: Derive Object
   async updateObject( ) {
     throw new Error( 'not implemeneted' )

packages/server/modules/core/tests/graph.spec.js

@@ -966,6 +966,7 @@ describe( 'GraphQL API Core @core-api', ( ) => {
             adminContact
             termsOfService
             description
+            version
             roles{
               name
               description

packages/server/modules/core/tests/rest.spec.js

@@ -126,9 +126,9 @@ describe( 'Upload/Download Routes @api-rest', ( ) => {
 
   let parentId
   let numObjs = 5000
+  let objBatches = [ createManyObjects( numObjs ), createManyObjects( numObjs ), createManyObjects( numObjs ) ]
 
   it( 'Should properly upload a bunch of objects', async ( ) => {
-    let objBatches = [ createManyObjects( numObjs ), createManyObjects( numObjs ), createManyObjects( numObjs ) ]
     parentId = objBatches[ 0 ][ 0 ].id
 
     let res =
@@ -208,6 +208,83 @@ describe( 'Upload/Download Routes @api-rest', ( ) => {
 
   } )
 
+  it( 'Should properly download a list of objects', ( done ) => {
+    let objectIds = []
+    for ( let i = 0; i < objBatches[0].length; i++ ) {
+      objectIds.push( objBatches[0][i].id )
+    }
+    let res = request( expressApp )
+      .post( `/api/getobjects/${testStream.id}` )
+      .set( 'Authorization', userA.token )
+      .set( 'Accept', 'text/plain' )
+      .send( { objects: JSON.stringify( objectIds ) } )
+      .buffer( )
+      .parse( ( res, cb ) => {
+        res.data = ''
+        res.on( 'data', chunk => {
+          res.data += chunk.toString( )
+        } )
+        res.on( 'end', ( ) => {
+          cb( null, res.data )
+        } )
+      } )
+      .end( ( err, res ) => {
+        if ( err ) done( err )
+        try {
+          let o = res.body.split( '\n' ).filter( l => l !== '' )
+          expect( o.length ).to.equal( objectIds.length )
+          expect( res ).to.be.text
+          done( )
+        } catch ( err ) {
+          done( err )
+        }
+      } )
+  } )
+
+  it( 'Should properly check if the server has a list of objects', ( done ) => {
+    let objectIds = []
+    for ( let i = 0; i < objBatches[0].length; i++ ) {
+      objectIds.push( objBatches[0][i].id )
+    }
+    let fakeIds = []
+    for ( let i = 0; i < 100; i++ ) {
+      let fakeId = crypto.createHash( 'md5' ).update( 'fakefake' + i ).digest( 'hex' )
+      fakeIds.push( fakeId )
+      objectIds.push( fakeId )
+    }
+
+    let res = request( expressApp )
+      .post( `/api/diff/${testStream.id}` )
+      .set( 'Authorization', userA.token )
+      .send( { objects: JSON.stringify( objectIds ) } )
+      .buffer( )
+      .parse( ( res, cb ) => {
+        res.data = ''
+        res.on( 'data', chunk => {
+          res.data += chunk.toString( )
+        } )
+        res.on( 'end', ( ) => {
+          cb( null, res.data )
+        } )
+      } )
+      .end( ( err, res ) => {
+        if ( err ) done( err )
+        try {
+          let o = JSON.parse( res.body )
+          expect( Object.keys(o).length ).to.equal( objectIds.length )
+          for ( let i = 0; i < objBatches[0].length; i++ ) {
+            assert( o[objBatches[0][i].id] === true, 'Server is missing an object' )
+          }
+          for ( let i = 0; i < fakeIds.length; i++ ) {
+            assert( o[fakeIds[i]] === false, 'Server wrongly reports it has an extra object' )
+          }
+          done( )
+        } catch ( err ) {
+          done( err )
+        }
+      } )
+  } )
+
 } )
 
 function createManyObjects( amount, noise ) {

utils/1click_image_scripts/template-nginx-site.conf

@@ -16,7 +16,7 @@ server {
         proxy_set_header Connection "upgrade";
     }
 
-    location ~* ^/(graphql|explorer|(auth/.*)|(objects/.*)|(preview/.*)) {
+    location ~* ^/(graphql|explorer|(auth/.*)|(objects/.*)|(preview/.*)|(api/.*)) {
         client_max_body_size 100m;
         proxy_set_header Host $host;
         proxy_set_header X-Real-IP $remote_addr;