Merge branch 'main' of github.com:specklesystems/speckle-server into alessandro/web-957-dispatch-webhook-event

This commit is contained in:
Alessandro Magionami 2024-09-17 10:17:48 +02:00
Родитель a7a74ac3ff 9992a9bd1d
Коммит c88c464842
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: EC367516F896CBA4
11 изменённых файлов: 423 добавлений и 216 удалений

Просмотреть файл

@ -3,7 +3,7 @@ import { logger } from '@/observability/logging.js'
import { randomUUID } from 'crypto'
import type { Request } from 'express'
import type { IncomingHttpHeaders, IncomingMessage } from 'http'
import { get } from 'lodash'
import { get } from 'lodash-es'
import { pinoHttp } from 'pino-http'
function determineRequestId(headers: IncomingHttpHeaders, uuidGenerator = randomUUID) {

Просмотреть файл

@ -0,0 +1,42 @@
import { ObjectPreview } from '@/modules/previews/domain/types'
import { Nullable, Optional } from '@speckle/shared'
import express from 'express'
export type GetObjectPreviewInfo = (params: {
streamId: string
objectId: string
}) => Promise<Optional<ObjectPreview>>
export type CreateObjectPreview = (
params: Pick<ObjectPreview, 'streamId' | 'objectId' | 'priority'>
) => Promise<void>
export type GetPreviewImage = (params: {
previewId: string
}) => Promise<Nullable<Buffer>>
export type GetObjectPreviewBufferOrFilepath = (params: {
streamId: string
objectId: string
angle?: string
}) => Promise<
| {
type: 'file'
file: string
error?: true
errorCode?: string
}
| { type: 'buffer'; buffer: Buffer; error?: true; errorCode?: string }
>
export type SendObjectPreview = (
req: express.Request,
res: express.Response,
streamId: string,
objectId: string,
angle: string
) => Promise<void>
export type CheckStreamPermissions = (
req: express.Request
) => Promise<{ hasPermissions: boolean; httpErrorCode: number }>

Просмотреть файл

@ -0,0 +1,15 @@
import { Nullable } from '@speckle/shared'
export type ObjectPreview = {
streamId: string
objectId: string
previewStatus: number
priority: number
lastUpdate: Date
preview: Nullable<Record<string, string>>
}
export type Preview = {
id: string
data: Buffer
}

Просмотреть файл

@ -1,35 +1,38 @@
/* istanbul ignore file */
'use strict'
const { validateScopes, authorizeResolver } = require('@/modules/shared')
const { getStream } = require('../core/services/streams')
const { getObject } = require('../core/services/objects')
const {
getCommitsByStreamId,
getCommitsByBranchName,
getCommitById
} = require('../core/services/commits')
const {
getPreviewImage,
createObjectPreview,
getObjectPreviewInfo
} = require('./services/previews')
const { makeOgImage } = require('./ogImage')
const { moduleLogger, logger } = require('@/logging/logging')
const { moduleLogger } = require('@/logging/logging')
const {
listenForPreviewGenerationUpdates
listenForPreviewGenerationUpdatesFactory
} = require('@/modules/previews/services/resultListener')
const { Scopes, Roles } = require('@speckle/shared')
const httpErrorImage = (httpErrorCode) =>
require.resolve(`#/assets/previews/images/preview_${httpErrorCode}.png`)
const cors = require('cors')
const { db } = require('@/db/knex')
const {
getObjectPreviewBufferOrFilepathFactory,
sendObjectPreviewFactory,
checkStreamPermissionsFactory
} = require('@/modules/previews/services/management')
const { getObject } = require('@/modules/core/services/objects')
const {
getObjectPreviewInfoFactory,
createObjectPreviewFactory,
getPreviewImageFactory
} = require('@/modules/previews/repository/previews')
const { publish } = require('@/modules/shared/utils/subscriptions')
const { getObjectCommitsWithStreamIds } = require('@/modules/core/repositories/commits')
const noPreviewImage = require.resolve('#/assets/previews/images/no_preview.png')
const previewErrorImage = require.resolve('#/assets/previews/images/preview_error.png')
exports.init = (app, isInitial) => {
if (process.env.DISABLE_PREVIEWS) {
@ -38,142 +41,21 @@ exports.init = (app, isInitial) => {
moduleLogger.info('📸 Init object preview module')
}
const DEFAULT_ANGLE = '0'
const getObjectPreviewBufferOrFilepath = async ({ streamId, objectId, angle }) => {
if (process.env.DISABLE_PREVIEWS) {
return {
type: 'file',
file: noPreviewImage
}
}
// Check if objectId is valid
const dbObj = await getObject({ streamId, objectId })
if (!dbObj) {
return {
type: 'file',
file: require.resolve('#/assets/previews/images/preview_404.png'),
error: true,
errorCode: 'OBJECT_NOT_FOUND'
}
}
// Get existing preview metadata
const previewInfo = await getObjectPreviewInfo({ streamId, objectId })
if (!previewInfo) {
await createObjectPreview({ streamId, objectId, priority: 0 })
}
if (!previewInfo || previewInfo.previewStatus !== 2 || !previewInfo.preview) {
return { type: 'file', file: noPreviewImage }
}
const previewImgId = previewInfo.preview[angle]
if (!previewImgId) {
logger.warn(
`Preview angle '${angle}' not found for object ${streamId}:${objectId}`
)
return {
type: 'file',
error: true,
errorCode: 'ANGLE_NOT_FOUND',
file: previewErrorImage
}
}
const previewImg = await getPreviewImage({ previewId: previewImgId })
if (!previewImg) {
logger.warn(`Preview image not found: ${previewImgId}`)
return {
type: 'file',
file: previewErrorImage,
error: true,
errorCode: 'PREVIEW_NOT_FOUND'
}
}
return { type: 'buffer', buffer: previewImg }
}
const sendObjectPreview = async (req, res, streamId, objectId, angle) => {
let previewBufferOrFile = await getObjectPreviewBufferOrFilepath({
streamId,
objectId,
angle
})
if (req.query.postprocess === 'og') {
const stream = await getStream({ streamId: req.params.streamId })
const streamName = stream.name
if (previewBufferOrFile.type === 'file') {
previewBufferOrFile = {
type: 'buffer',
buffer: await makeOgImage(previewBufferOrFile.file, streamName)
}
} else {
previewBufferOrFile = {
type: 'buffer',
buffer: await makeOgImage(previewBufferOrFile.buffer, streamName)
}
}
}
if (previewBufferOrFile.error) {
res.set('X-Preview-Error', 'true')
}
if (previewBufferOrFile.errorCode) {
res.set('X-Preview-Error-Code', previewBufferOrFile.errorCode)
}
if (previewBufferOrFile.type === 'file') {
// we can't cache these cause they may switch to proper buffer previews in a sec
// at least if they're not in the error state which they will not get out of (and thus can be cached in that scenario)
if (previewBufferOrFile.error) {
res.set('Cache-Control', 'private, max-age=604800')
} else {
res.set('Cache-Control', 'no-cache, no-store')
}
res.sendFile(previewBufferOrFile.file)
} else {
res.contentType('image/png')
// If the preview is a buffer, it comes from the DB and can be cached on clients
res.set('Cache-Control', 'private, max-age=604800')
res.send(previewBufferOrFile.buffer)
}
}
const checkStreamPermissions = async (req) => {
const stream = await getStream({
streamId: req.params.streamId,
userId: req.context.userId
})
if (!stream) {
return { hasPermissions: false, httpErrorCode: 404 }
}
if (!stream.isPublic && req.context.auth === false) {
return { hasPermissions: false, httpErrorCode: 401 }
}
if (!stream.isPublic) {
try {
await validateScopes(req.context.scopes, Scopes.Streams.Read)
} catch {
return { hasPermissions: false, httpErrorCode: 401 }
}
try {
await authorizeResolver(
req.context.userId,
req.params.streamId,
Roles.Stream.Reviewer,
req.context.resourceAccessRules
)
} catch {
return { hasPermissions: false, httpErrorCode: 401 }
}
}
return { hasPermissions: true, httpErrorCode: 200 }
}
const getObjectPreviewBufferOrFilepath = getObjectPreviewBufferOrFilepathFactory({
getObject,
getObjectPreviewInfo: getObjectPreviewInfoFactory({ db }),
createObjectPreview: createObjectPreviewFactory({ db }),
getPreviewImage: getPreviewImageFactory({ db })
})
const sendObjectPreview = sendObjectPreviewFactory({
getObject,
getObjectPreviewBufferOrFilepath,
makeOgImage
})
const checkStreamPermissions = checkStreamPermissionsFactory({
validateScopes,
authorizeResolver
})
app.options('/preview/:streamId/:angle?', cors())
app.get('/preview/:streamId/:angle?', cors(), async (req, res) => {
@ -198,7 +80,7 @@ exports.init = (app, isInitial) => {
res,
req.params.streamId,
lastCommit.referencedObject,
req.params.angle || DEFAULT_ANGLE
req.params.angle
)
})
@ -234,7 +116,7 @@ exports.init = (app, isInitial) => {
res,
req.params.streamId,
lastCommit.referencedObject,
req.params.angle || DEFAULT_ANGLE
req.params.angle
)
}
)
@ -258,7 +140,7 @@ exports.init = (app, isInitial) => {
res,
req.params.streamId,
commit.referencedObject,
req.params.angle || DEFAULT_ANGLE
req.params.angle
)
})
@ -274,11 +156,15 @@ exports.init = (app, isInitial) => {
res,
req.params.streamId,
req.params.objectId,
req.params.angle || DEFAULT_ANGLE
req.params.angle
)
})
if (isInitial) {
const listenForPreviewGenerationUpdates = listenForPreviewGenerationUpdatesFactory({
getObjectCommitsWithStreamIds,
publish
})
listenForPreviewGenerationUpdates()
}
}

Просмотреть файл

@ -0,0 +1,74 @@
/* istanbul ignore file */
import knex from '@/db/knex'
import { buildTableHelper } from '@/modules/core/dbSchema'
import {
CreateObjectPreview,
GetObjectPreviewInfo,
GetPreviewImage
} from '@/modules/previews/domain/operations'
import {
ObjectPreview as ObjectPreviewRecord,
Preview
} from '@/modules/previews/domain/types'
import { Knex } from 'knex'
import { SetOptional } from 'type-fest'
const ObjectPreview = buildTableHelper('object_preview', [
'streamId',
'objectId',
'previewStatus',
'priority',
'lastUpdate',
'preview'
])
const Previews = buildTableHelper('previews', ['id', 'data'])
const tables = {
objectPreview: (db: Knex) => db<ObjectPreviewRecord>(ObjectPreview.name),
previews: (db: Knex) => db<Preview>(Previews.name)
}
export const getObjectPreviewInfoFactory =
(deps: { db: Knex }): GetObjectPreviewInfo =>
async ({ streamId, objectId }: { streamId: string; objectId: string }) => {
return await tables
.objectPreview(deps.db)
.select('*')
.where({ streamId, objectId })
.first()
}
export const createObjectPreviewFactory =
(deps: { db: Knex }): CreateObjectPreview =>
async ({
streamId,
objectId,
priority
}: Pick<ObjectPreviewRecord, 'streamId' | 'objectId' | 'priority'>) => {
const insertionObject: SetOptional<ObjectPreviewRecord, 'lastUpdate' | 'preview'> =
{
streamId,
objectId,
priority,
previewStatus: 0
}
const sqlQuery =
tables.objectPreview(deps.db).insert(insertionObject).toString() +
' on conflict do nothing'
await knex.raw(sqlQuery)
}
export const getPreviewImageFactory =
(deps: { db: Knex }): GetPreviewImage =>
async ({ previewId }: { previewId: string }) => {
const previewRow = await tables
.previews(deps.db)
.where({ id: previewId })
.first()
.select('*')
if (!previewRow) {
return null
}
return previewRow.data
}

Просмотреть файл

@ -0,0 +1,173 @@
import { logger } from '@/logging/logging'
import { getStream } from '@/modules/core/repositories/streams'
import { getObject } from '@/modules/core/services/objects'
import {
CheckStreamPermissions,
CreateObjectPreview,
GetObjectPreviewBufferOrFilepath,
GetObjectPreviewInfo,
GetPreviewImage,
SendObjectPreview
} from '@/modules/previews/domain/operations'
import { makeOgImage } from '@/modules/previews/ogImage'
import { authorizeResolver, validateScopes } from '@/modules/shared'
import { Roles, Scopes } from '@speckle/shared'
const noPreviewImage = require.resolve('#/assets/previews/images/no_preview.png')
const previewErrorImage = require.resolve('#/assets/previews/images/preview_error.png')
const defaultAngle = '0'
export const getObjectPreviewBufferOrFilepathFactory =
(deps: {
getObject: typeof getObject
getObjectPreviewInfo: GetObjectPreviewInfo
createObjectPreview: CreateObjectPreview
getPreviewImage: GetPreviewImage
}): GetObjectPreviewBufferOrFilepath =>
async ({ streamId, objectId, angle }) => {
angle = angle || defaultAngle
if (process.env.DISABLE_PREVIEWS) {
return {
type: 'file',
file: noPreviewImage
}
}
// Check if objectId is valid
const dbObj = await deps.getObject({ streamId, objectId })
if (!dbObj) {
return {
type: 'file',
file: require.resolve('#/assets/previews/images/preview_404.png'),
error: true,
errorCode: 'OBJECT_NOT_FOUND'
}
}
// Get existing preview metadata
const previewInfo = await deps.getObjectPreviewInfo({ streamId, objectId })
if (!previewInfo) {
await deps.createObjectPreview({ streamId, objectId, priority: 0 })
}
if (!previewInfo || previewInfo.previewStatus !== 2 || !previewInfo.preview) {
return { type: 'file', file: noPreviewImage }
}
const previewImgId = previewInfo.preview[angle]
if (!previewImgId) {
logger.warn(
`Preview angle '${angle}' not found for object ${streamId}:${objectId}`
)
return {
type: 'file',
error: true,
errorCode: 'ANGLE_NOT_FOUND',
file: previewErrorImage
}
}
const previewImg = await deps.getPreviewImage({ previewId: previewImgId })
if (!previewImg) {
logger.warn(`Preview image not found: ${previewImgId}`)
return {
type: 'file',
file: previewErrorImage,
error: true,
errorCode: 'PREVIEW_NOT_FOUND'
}
}
return { type: 'buffer', buffer: previewImg }
}
export const sendObjectPreviewFactory =
(deps: {
getObjectPreviewBufferOrFilepath: GetObjectPreviewBufferOrFilepath
getStream: typeof getStream
makeOgImage: typeof makeOgImage
}): SendObjectPreview =>
async (req, res, streamId, objectId, angle) => {
let previewBufferOrFile = await deps.getObjectPreviewBufferOrFilepath({
streamId,
objectId,
angle
})
if (req.query.postprocess === 'og') {
const stream = await deps.getStream({ streamId: req.params.streamId })
const streamName = stream!.name
if (previewBufferOrFile.type === 'file') {
previewBufferOrFile = {
type: 'buffer',
buffer: await deps.makeOgImage(previewBufferOrFile.file, streamName)
}
} else {
previewBufferOrFile = {
type: 'buffer',
buffer: await deps.makeOgImage(previewBufferOrFile.buffer, streamName)
}
}
}
if (previewBufferOrFile.error) {
res.set('X-Preview-Error', 'true')
}
if (previewBufferOrFile.errorCode) {
res.set('X-Preview-Error-Code', previewBufferOrFile.errorCode)
}
if (previewBufferOrFile.type === 'file') {
// we can't cache these cause they may switch to proper buffer previews in a sec
// at least if they're not in the error state which they will not get out of (and thus can be cached in that scenario)
if (previewBufferOrFile.error) {
res.set('Cache-Control', 'private, max-age=604800')
} else {
res.set('Cache-Control', 'no-cache, no-store')
}
res.sendFile(previewBufferOrFile.file)
} else {
res.contentType('image/png')
// If the preview is a buffer, it comes from the DB and can be cached on clients
res.set('Cache-Control', 'private, max-age=604800')
res.send(previewBufferOrFile.buffer)
}
}
export const checkStreamPermissionsFactory =
(deps: {
validateScopes: typeof validateScopes
authorizeResolver: typeof authorizeResolver
}): CheckStreamPermissions =>
async (req) => {
const stream = await getStream({
streamId: req.params.streamId,
userId: req.context.userId
})
if (!stream) {
return { hasPermissions: false, httpErrorCode: 404 }
}
if (!stream.isPublic && req.context.auth === false) {
return { hasPermissions: false, httpErrorCode: 401 }
}
if (!stream.isPublic) {
try {
await deps.validateScopes(req.context.scopes, Scopes.Streams.Read)
} catch {
return { hasPermissions: false, httpErrorCode: 401 }
}
try {
await deps.authorizeResolver(
req.context.userId,
req.params.streamId,
Roles.Stream.Reviewer,
req.context.resourceAccessRules
)
} catch {
return { hasPermissions: false, httpErrorCode: 401 }
}
}
return { hasPermissions: true, httpErrorCode: 200 }
}

Просмотреть файл

@ -1,33 +0,0 @@
/* istanbul ignore file */
'use strict'
const knex = require('@/db/knex')
const ObjectPreview = () => knex('object_preview')
const Previews = () => knex('previews')
module.exports = {
async getObjectPreviewInfo({ streamId, objectId }) {
return await ObjectPreview().select('*').where({ streamId, objectId }).first()
},
async createObjectPreview({ streamId, objectId, priority }) {
const insertionObject = {
streamId,
objectId,
priority,
previewStatus: 0
}
const sqlQuery =
ObjectPreview().insert(insertionObject).toString() + ' on conflict do nothing'
await knex.raw(sqlQuery)
},
async getPreviewImage({ previewId }) {
const previewRow = await Previews().where({ id: previewId }).first().select('*')
if (!previewRow) {
return null
}
return previewRow.data
}
}

Просмотреть файл

@ -1,39 +1,50 @@
import { getObjectCommitsWithStreamIds } from '@/modules/core/repositories/commits'
import { ProjectSubscriptions, publish } from '@/modules/shared/utils/subscriptions'
import {
ProjectSubscriptions,
PublishSubscription
} from '@/modules/shared/utils/subscriptions'
import { listenFor, MessageType } from '@/modules/core/utils/dbNotificationListener'
const payloadRegexp = /^([\w\d]+):([\w\d]+):([\w\d]+)$/i
async function messageProcessor(msg: MessageType) {
if (msg.channel !== 'preview_generation_update') return
const [, status, streamId, objectId] = payloadRegexp.exec(msg.payload) || [
null,
null,
null,
null
]
type MessageProcessorDeps = {
getObjectCommitsWithStreamIds: typeof getObjectCommitsWithStreamIds
publish: PublishSubscription
}
if (status !== 'finished' || !objectId || !streamId) return
const messageProcessorFactory =
(deps: MessageProcessorDeps) => async (msg: MessageType) => {
if (msg.channel !== 'preview_generation_update') return
const [, status, streamId, objectId] = payloadRegexp.exec(msg.payload) || [
null,
null,
null,
null
]
// Get all commits with that objectId
const commits = await getObjectCommitsWithStreamIds([objectId], {
streamIds: [streamId]
})
if (!commits.length) return
if (status !== 'finished' || !objectId || !streamId) return
await Promise.all(
commits.map((c) =>
publish(ProjectSubscriptions.ProjectVersionsPreviewGenerated, {
projectVersionsPreviewGenerated: {
versionId: c.id,
projectId: c.streamId,
objectId
}
})
// Get all commits with that objectId
const commits = await deps.getObjectCommitsWithStreamIds([objectId], {
streamIds: [streamId]
})
if (!commits.length) return
await Promise.all(
commits.map((c) =>
deps.publish(ProjectSubscriptions.ProjectVersionsPreviewGenerated, {
projectVersionsPreviewGenerated: {
versionId: c.id,
projectId: c.streamId,
objectId
}
})
)
)
)
}
}
export function listenForPreviewGenerationUpdates() {
listenFor('preview_generation_update', messageProcessor)
}
export const listenForPreviewGenerationUpdatesFactory =
(deps: MessageProcessorDeps) => () => {
const messageProcessor = messageProcessorFactory(deps)
listenFor('preview_generation_update', messageProcessor)
}

Просмотреть файл

@ -4,3 +4,4 @@ export * as SpeckleViewer from './viewer/index.js'
// export * as Environment from './environment/index.js' // Import from @speckle/shared/dist/...
export * as Automate from './automate/index.js'
export * from './core/index.js'
export * from './workspaces/index.js'

Просмотреть файл

@ -0,0 +1,37 @@
export const VALID_SLUG_CHARACTERS_REGEX = /^[a-z0-9-]+$/
export const VALID_SLUG_BOUNDARY_REGEX = /^[a-z0-9].*[a-z0-9]$/
const MIN_SLUG_LENGTH = 3
const MAX_SLUG_LENGTH = 30
export class InvalidWorkspaceSlugError extends Error {
constructor(message: string) {
super(message)
this.name = 'InvalidWorkspaceSlugError'
}
}
export function validateWorkspaceSlug(slug: string): void {
if (slug.length < MIN_SLUG_LENGTH) {
throw new InvalidWorkspaceSlugError(
`Workspace slug must be at least ${MIN_SLUG_LENGTH} characters long.`
)
}
if (slug.length > MAX_SLUG_LENGTH) {
throw new InvalidWorkspaceSlugError(
`Workspace slug must not exceed ${MAX_SLUG_LENGTH} characters.`
)
}
if (!VALID_SLUG_CHARACTERS_REGEX.test(slug)) {
throw new InvalidWorkspaceSlugError(
'Workspace slug must contain only lowercase letters, numbers, and hyphens.'
)
}
if (!VALID_SLUG_BOUNDARY_REGEX.test(slug)) {
throw new InvalidWorkspaceSlugError(
'Workspace slug cannot start or end with a hyphen.'
)
}
}

Просмотреть файл

@ -0,0 +1 @@
export * from './errors/index.js'