feat: proper sign up tracking (#1489)

* feat: register flag passed to fe

* feat: mixpanel tracking for all sign ups

* feat: utm first touch & last touch tracking

* feat(helm): Allows Environment Variable for MP to be configured
- default is enabled
- renames environment variable to ENABLE_MP

* feat(helm network policy): allowlist analytics.speckle.systems

---------

Co-authored-by: Iain Sproat <68657+iainsproat@users.noreply.github.com>

packages/frontend/src/helpers/md5.ts

@@ -0,0 +1,3 @@
+import { md5 } from '@speckle/shared'
+export default md5
+export { md5 }

packages/frontend/src/main/pages/auth/TheRegistration.vue

@@ -338,10 +338,6 @@ export default {
         )
 
         if (res.redirected) {
-          this.$mixpanel.track('Sign Up', {
-            isInvite: this.token !== null,
-            type: 'action'
-          })
           processSuccessfulAuth(res)
           this.loading = false
           return

packages/frontend/src/mixpanelManager.ts

@@ -4,9 +4,35 @@ import { Optional } from '@/helpers/typeHelpers'
 import { AppLocalStorage } from '@/utils/localStorage'
 import md5 from '@/helpers/md5'
 import * as ThemeStateManager from '@/main/utils/themeStateManager'
+import { intersection, mapKeys } from 'lodash'
 
 let mixpanelInitialized = false
 
+const campaignKeywords = [
+  'utm_source',
+  'utm_medium',
+  'utm_campaign',
+  'utm_content',
+  'utm_term'
+]
+
+function collectUtmTags() {
+  const currentUrl = new URL(window.location.href)
+  const foundParams = intersection(
+    [...currentUrl.searchParams.keys()],
+    campaignKeywords
+  )
+
+  const result: Record<string, string> = {}
+  for (const campaignParam of foundParams) {
+    const value = currentUrl.searchParams.get(campaignParam)
+    if (!value) continue
+    result[campaignParam] = value
+  }
+
+  return result
+}
+
 /**
  * Get mixpanel user ID, if user is authenticated and can be identified, or undefined otherwise
  */
@@ -56,6 +82,17 @@ export function initialize(params: {
     mp.people.set('Theme Web', ThemeStateManager.isDarkTheme() ? 'dark' : 'light')
   }
 
+  // Track UTM
+  const utmParams = collectUtmTags()
+  if (Object.values(utmParams).length) {
+    const firstTouch = mapKeys(utmParams, (_val, key) => `${key} [first touch]`)
+    const lastTouch = mapKeys(utmParams, (_val, key) => `${key} [last touch]`)
+
+    mp.people.set(lastTouch)
+    mp.people.set_once(firstTouch)
+    mp.register(lastTouch)
+  }
+
   // Track app visit
   mp.track(`Visit ${hostAppDisplayName}`)
 

packages/frontend/src/plugins/authHelpers.js

@@ -1,11 +1,11 @@
 import { mainUserDataQuery } from '@/graphql/user'
 import { LocalStorageKeys } from '@/helpers/mainConstants'
-import md5 from '@/helpers/md5'
 import { InvalidAuthTokenError } from '@/main/lib/auth/errors'
 import { VALID_EMAIL_REGEX } from '@/main/lib/common/vuetify/validators'
 import { AppLocalStorage } from '@/utils/localStorage'
 import { has } from 'lodash'
 import { deletePostAuthRedirect } from '@/main/lib/auth/utils/postAuthRedirectManager'
+import { resolveMixpanelUserId } from '@speckle/shared'
 
 const appId = 'spklwebapp'
 const appSecret = 'spklwebapp'
@@ -46,7 +46,7 @@ export async function prefetchUserAndSetID(apolloClient) {
 
   const user = data.activeUser
   if (user) {
-    const distinctId = '@' + md5(user.email.toLowerCase()).toUpperCase()
+    const distinctId = resolveMixpanelUserId(user.email)
     AppLocalStorage.set('distinct_id', distinctId)
     AppLocalStorage.set('uuid', user.id)
     AppLocalStorage.set('stcount', user.streams.totalCount)

packages/server/app.ts

@@ -41,7 +41,11 @@ import { Optional } from '@/modules/shared/helpers/typeHelper'
 import { createRateLimiterMiddleware } from '@/modules/core/services/ratelimiter'
 
 import { get, has, isString, toNumber } from 'lodash'
-import { authContextMiddleware, buildContext } from '@/modules/shared/middleware'
+import {
+  authContextMiddleware,
+  buildContext,
+  mixpanelTrackerHelperMiddleware
+} from '@/modules/shared/middleware'
 
 let graphqlServer: ApolloServer
 
@@ -203,6 +207,7 @@ export async function init() {
   app.use(errorLoggingMiddleware)
   app.use(authContextMiddleware)
   app.use(createRateLimiterMiddleware())
+  app.use(mixpanelTrackerHelperMiddleware)
 
   app.use(Sentry.Handlers.errorHandler())
 

packages/server/modules/auth/strategies.js

@@ -9,6 +9,7 @@ const { createAuthorizationCode } = require('./services/apps')
 const { isSSLServer, getRedisUrl } = require('@/modules/shared/helpers/envHelper')
 const { authLogger } = require('@/logging/logging')
 const { createRedisClient } = require('@/modules/shared/redis/redis')
+const { mixpanel, resolveMixpanelUserId } = require('@/modules/shared/utils/mixpanel')
 
 /**
  * TODO: Get rid of session entirely, we don't use it for the app and it's not really necessary for the auth flow, so it only complicates things
@@ -50,8 +51,9 @@ module.exports = async (app) => {
     next()
   }
 
-  /*
+  /**
   Finalizes authentication for the main frontend application.
+  @param {import('express').Request} req
    */
   const finalizeAuth = async (req, res) => {
     try {
@@ -66,6 +68,20 @@ module.exports = async (app) => {
       // Resolve redirect URL
       const urlObj = new URL(req.authRedirectPath || '/', process.env.CANONICAL_URL)
       urlObj.searchParams.set('access_code', ac)
+
+      if (req.user.isNewUser) {
+        urlObj.searchParams.set('register', 'true')
+
+        // Send event to MP
+        const userId = req.user.email ? resolveMixpanelUserId(req.user.email) : null
+        const isInvite = !!req.user.isInvite
+        if (userId) {
+          mixpanel({ mixpanelUserId: userId }).track('Sign Up', {
+            isInvite
+          })
+        }
+      }
+
       const redirectUrl = urlObj.toString()
 
       return res.redirect(redirectUrl)

packages/server/modules/auth/strategies/azure-ad.js

@@ -86,6 +86,7 @@ module.exports = async (app, session, sessionStorage, finalizeAuth) => {
           })
           // ID is used later for verifying access token
           req.user.id = myUser.id
+          req.user.isNewUser = myUser.isNewUser
 
           // process invites
           if (myUser.isNewUser) {
@@ -110,6 +111,7 @@ module.exports = async (app, session, sessionStorage, finalizeAuth) => {
 
         // ID is used later for verifying access token
         req.user.id = myUser.id
+        req.user.isInvite = !!validInvite
         req.log = req.log.child({ userId: myUser.id })
 
         // use the invite

packages/server/modules/auth/strategies/github.js

@@ -90,7 +90,10 @@ module.exports = async (app, session, sessionStorage, finalizeAuth) => {
         req.authRedirectPath = resolveAuthRedirectPath(validInvite)
 
         // return to the auth flow
-        return done(null, myUser)
+        return done(null, {
+          ...myUser,
+          isInvite: !!validInvite
+        })
       } catch (err) {
         switch (err.constructor) {
           case UserInputError:

packages/server/modules/auth/strategies/google.js

@@ -87,7 +87,10 @@ module.exports = async (app, session, sessionStorage, finalizeAuth) => {
         req.authRedirectPath = resolveAuthRedirectPath(validInvite)
 
         // return to the auth flow
-        return done(null, myUser)
+        return done(null, {
+          ...myUser,
+          isInvite: !!validInvite
+        })
       } catch (err) {
         switch (err.constructor) {
           case UserInputError:

packages/server/modules/auth/strategies/local.js

@@ -98,7 +98,12 @@ module.exports = async (app, session, sessionAppId, finalizeAuth) => {
         //    * the server public and the user doesn't have an invite
         // so we go ahead and register the user
         const userId = await createUser(user)
-        req.user = { id: userId, email: user.email }
+        req.user = {
+          id: userId,
+          email: user.email,
+          isNewUser: true,
+          isInvite: !!invite
+        }
         req.log = req.log.child({ userId })
 
         // 4. use up all server-only invites the email had attached to it

packages/server/modules/auth/strategies/oidc.js

@@ -105,7 +105,10 @@ module.exports = async (app, session, sessionStorage, finalizeAuth) => {
           req.authRedirectPath = resolveAuthRedirectPath(validInvite)
 
           // return to the auth flow
-          return done(null, myUser)
+          return done(null, {
+            ...myUser,
+            isInvite: !!validInvite
+          })
         } catch (err) {
           logger.error(err)
           return done(null, false, { message: err.message })

packages/server/modules/core/index.js

@@ -1,6 +1,6 @@
-'use strict'
 const { registerOrUpdateScope, registerOrUpdateRole } = require('@/modules/shared')
 const { moduleLogger } = require('@/logging/logging')
+const mp = require('@/modules/shared/utils/mixpanel')
 
 exports.init = async (app) => {
   moduleLogger.info('💥 Init core module')
@@ -26,6 +26,9 @@ exports.init = async (app) => {
   for (const role of roles) {
     await registerOrUpdateRole(role)
   }
+
+  // Init mp
+  mp.initialize()
 }
 
 exports.finalize = () => {}

packages/server/modules/core/services/ratelimiter.ts

@@ -1,4 +1,4 @@
-import express, { RequestWithAuthContext } from 'express'
+import express from 'express'
 import {
   getRedisUrl,
   getIntFromEnv,
@@ -282,9 +282,7 @@ export const getActionForPath = (path: string, verb: string): RateLimitAction =>
 }
 
 export const getSourceFromRequest = (req: express.Request): string => {
-  let source: string | null =
-    ((req as RequestWithAuthContext)?.context?.userId as string) ||
-    getIpFromRequest(req)
+  let source: string | null = req?.context?.userId || getIpFromRequest(req)
 
   if (!source) source = 'unknown'
   return source

packages/server/modules/core/services/users.js

@@ -89,8 +89,17 @@ module.exports = {
     return newUser.id
   },
 
+  /**
+   * @returns {Promise<{
+   *  id: string,
+   *  email: string,
+   *  isNewUser?: boolean
+   * }>}
+   */
   async findOrCreateUser({ user }) {
-    const existingUser = await userByEmailQuery(user.email).select('id').first()
+    const existingUser = await userByEmailQuery(user.email)
+      .select(['id', 'email'])
+      .first()
     if (existingUser) return existingUser
 
     user.password = crs({ length: 20 })

packages/server/modules/shared/helpers/envHelper.ts

@@ -103,3 +103,8 @@ export function isSSLServer() {
 export function adminOverrideEnabled() {
   return process.env.ADMIN_OVERRIDE_ENABLED === 'true'
 }
+
+export function enableMixpanel() {
+  // if not explicitly set to '0' or 'false', it is enabled by default
+  return !['0', 'false'].includes(process.env.ENABLE_MP || 'true')
+}

packages/server/modules/shared/middleware/index.ts

@@ -5,24 +5,27 @@ import {
   AuthParams,
   authHasFailed
 } from '@/modules/shared/authz'
-import { Request, Response, NextFunction, RequestWithAuthContext } from 'express'
+import { Request, Response, NextFunction } from 'express'
 import { ForbiddenError, UnauthorizedError } from '@/modules/shared/errors'
 import { ensureError } from '@/modules/shared/helpers/errorHelper'
 import { validateToken } from '@/modules/core/services/tokens'
 import { TokenValidationResult } from '@/modules/core/helpers/types'
 import { buildRequestLoaders } from '@/modules/core/loaders'
-import { GraphQLContext } from '@/modules/shared/helpers/typeHelper'
+import {
+  GraphQLContext,
+  MaybeNullOrUndefined,
+  Nullable
+} from '@/modules/shared/helpers/typeHelper'
+import { getUser } from '@/modules/core/repositories/users'
+import { resolveMixpanelUserId } from '@speckle/shared'
+import { mixpanel } from '@/modules/shared/utils/mixpanel'
 
 export const authMiddlewareCreator = (steps: AuthPipelineFunction[]) => {
   const pipeline = authPipelineCreator(steps)
 
-  const middleware = async (
-    req: RequestWithAuthContext,
-    res: Response,
-    next: NextFunction
-  ) => {
+  const middleware = async (req: Request, res: Response, next: NextFunction) => {
     const { authResult } = await pipeline({
-      context: req.context as AuthContext,
+      context: req.context,
       params: req.params as AuthParams,
       authResult: { authorized: false }
     })
@@ -92,7 +95,7 @@ export async function authContextMiddleware(
     if (authContext.err instanceof ForbiddenError) status = 403
     return res.status(status).json({ error: message })
   }
-  ;(req as RequestWithAuthContext).context = authContext
+  req.context = authContext
   next()
 }
 
@@ -100,11 +103,7 @@ export function addLoadersToCtx(ctx: AuthContext): GraphQLContext {
   const loaders = buildRequestLoaders(ctx)
   return { ...ctx, loaders }
 }
-type MaybeAuthenticatedRequest = Request | RequestWithAuthContext | null | undefined
-const isRequestWithAuthContext = (
-  req: MaybeAuthenticatedRequest
-): req is RequestWithAuthContext =>
-  req !== null && req !== undefined && 'context' in req
+
 /**
  * Build context for GQL operations
  */
@@ -112,13 +111,30 @@ export async function buildContext({
   req,
   token
 }: {
-  req: MaybeAuthenticatedRequest
-  token: string | null
+  req: MaybeNullOrUndefined<Request>
+  token: Nullable<string>
 }): Promise<GraphQLContext> {
-  const ctx = isRequestWithAuthContext(req)
-    ? req.context
-    : await createAuthContextFromToken(token ?? getTokenFromRequest(req))
+  const ctx =
+    req?.context ||
+    (await createAuthContextFromToken(token ?? getTokenFromRequest(req)))
 
   // Adding request data loaders
   return addLoadersToCtx(ctx)
 }
+
+/**
+ * Adds a .mixpanel helper onto the req object that is already pre-identified with the active user's identity
+ */
+export async function mixpanelTrackerHelperMiddleware(
+  req: Request,
+  _res: Response,
+  next: NextFunction
+) {
+  const ctx = req.context
+  const user = ctx.userId ? await getUser(ctx.userId) : null
+  const mixpanelUserId = user?.email ? resolveMixpanelUserId(user.email) : undefined
+  const mp = mixpanel({ mixpanelUserId })
+
+  req.mixpanel = mp
+  next()
+}

packages/server/modules/shared/utils/mixpanel.ts

@@ -0,0 +1,46 @@
+/* eslint-disable camelcase */
+import { Optional, resolveMixpanelUserId } from '@speckle/shared'
+import { enableMixpanel } from '@/modules/shared/helpers/envHelper'
+import Mixpanel from 'mixpanel'
+
+let client: Optional<Mixpanel.Mixpanel> = undefined
+
+export function initialize() {
+  if (client || !enableMixpanel()) return
+
+  client = Mixpanel.init('acd87c5a50b56df91a795e999812a3a4', {
+    host: 'analytics.speckle.systems'
+  })
+}
+
+/**
+ * Mixpanel client. Can be undefined if not initialized or disabled.
+ */
+export function getClient() {
+  return client
+}
+
+/**
+ * Mixpanel tracking helper. An abstraction layer over the client that makes it a bit nicer to work with.
+ */
+export function mixpanel(params: { mixpanelUserId: Optional<string> }) {
+  const { mixpanelUserId } = params
+  const userIdentificationProperties = () => ({
+    ...(mixpanelUserId
+      ? {
+          distinct_id: mixpanelUserId
+        }
+      : {})
+  })
+
+  return {
+    track: (eventName: string, extraProperties?: Record<string, unknown>) => {
+      return getClient()?.track(eventName, {
+        ...userIdentificationProperties(),
+        ...(extraProperties || {})
+      })
+    }
+  }
+}
+
+export { resolveMixpanelUserId }

packages/server/package.json

@@ -63,6 +63,7 @@
     "ioredis": "^5.2.2",
     "knex": "^2.4.1",
     "lodash": "^4.17.21",
+    "mixpanel": "^0.17.0",
     "mjml": "^4.13.0",
     "module-alias": "^2.2.2",
     "node-cron": "^3.0.2",

packages/server/type-augmentations/express.d.ts

@@ -1,9 +1,17 @@
-import { Request } from 'express'
 import { AuthContext } from '@/modules/shared/authz'
+import { mixpanel } from '@/modules/shared/utils/mixpanel'
 
 declare module 'express' {
-  interface RequestWithAuthContext extends Request {
+  interface Request {
     context: AuthContext
+    mixpanel: ReturnType<typeof mixpanel>
+  }
+}
+
+declare module 'express-serve-static-core' {
+  interface Request {
+    context: AuthContext
+    mixpanel: ReturnType<typeof mixpanel>
   }
 }
 

packages/shared/.eslintrc.js

@@ -32,6 +32,12 @@ const config = {
       rules: {
         '@typescript-eslint/no-explicit-any': 'off'
       }
+    },
+    {
+      files: '*.mjs',
+      parserOptions: {
+        sourceType: 'module'
+      }
     }
   ]
 }

packages/shared/src/core/helpers/tracking.ts

@@ -0,0 +1,5 @@
+import { md5 } from '../utils/md5'
+
+export function resolveMixpanelUserId(email: string): string {
+  return '@' + md5(email.toLowerCase()).toUpperCase()
+}

packages/shared/src/core/index.ts

@@ -3,4 +3,6 @@ export * from './helpers/batch'
 export * from './helpers/timeConstants'
 export * from './helpers/utility'
 export * from './helpers/utilityTypes'
+export * from './helpers/tracking'
 export * from './utils/localStorage'
+export * from './utils/md5'

packages/shared/src/core/utils/md5.ts

@@ -1,9 +1,11 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+
 /**
  * Lightweight MD5 implementation.
  * @see http://www.myersdaily.org/joseph/javascript/md5-text.html
  */
 
-function md5cycle(x, k) {
+function md5cycle(x: any, k: any) {
   let a = x[0],
     b = x[1],
     c = x[2],
@@ -83,28 +85,28 @@ function md5cycle(x, k) {
   x[3] = add32(d, x[3])
 }
 
-function cmn(q, a, b, x, s, t) {
+function cmn(q: any, a: any, b: any, x: any, s: any, t: any) {
   a = add32(add32(a, q), add32(x, t))
   return add32((a << s) | (a >>> (32 - s)), b)
 }
 
-function ff(a, b, c, d, x, s, t) {
+function ff(a: any, b: any, c: any, d: any, x: any, s: any, t: any) {
   return cmn((b & c) | (~b & d), a, b, x, s, t)
 }
 
-function gg(a, b, c, d, x, s, t) {
+function gg(a: any, b: any, c: any, d: any, x: any, s: any, t: any) {
   return cmn((b & d) | (c & ~d), a, b, x, s, t)
 }
 
-function hh(a, b, c, d, x, s, t) {
+function hh(a: any, b: any, c: any, d: any, x: any, s: any, t: any) {
   return cmn(b ^ c ^ d, a, b, x, s, t)
 }
 
-function ii(a, b, c, d, x, s, t) {
+function ii(a: any, b: any, c: any, d: any, x: any, s: any, t: any) {
   return cmn(c ^ (b | ~d), a, b, x, s, t)
 }
 
-function md51(s) {
+function md51(s: any) {
   const n = s.length,
     state = [1732584193, -271733879, -1732584194, 271733878]
 
@@ -125,7 +127,7 @@ function md51(s) {
   return state
 }
 
-function md5blk(s) {
+function md5blk(s: any) {
   /* I figured global was faster.   */
   const md5blks = []
   let i
@@ -141,7 +143,7 @@ function md5blk(s) {
 
 const HEX_CHR = '0123456789abcdef'.split('')
 
-function rhex(n) {
+function rhex(n: any) {
   let s = '',
     j = 0
   for (; j < 4; j++)
@@ -149,12 +151,12 @@ function rhex(n) {
   return s
 }
 
-function hex(x) {
+function hex(x: any) {
   for (let i = 0; i < x.length; i++) x[i] = rhex(x[i])
   return x.join('')
 }
 
-let add32 = (a, b) => {
+let add32 = (a: any, b: any) => {
   return (a + b) & 0xffffffff
 }
 
@@ -163,7 +165,7 @@ let add32 = (a, b) => {
  * @param {string} s input string
  * @returns {string} md5 hash
  */
-function md5(s) {
+function md5(s: string): string {
   return hex(md51(s))
 }
 

utils/helm/speckle-server/templates/server/deployment.yml

@@ -138,7 +138,11 @@ spec:
           - name: ADMIN_OVERRIDE_ENABLED
             value: "true"
           {{- end }}
-            
+
+          {{- if (and .Values.server.mp .Values.server.mp.enabled) }}
+          - name: ENABLE_MP
+            value: {{ default "true" ( .Values.server.mp.enabled | quote) }}
+          {{- end }}
 
           # *** S3 Object Storage ***
           {{- if (or .Values.s3.configMap.enabled .Values.s3.endpoint) }}

utils/helm/speckle-server/templates/server/networkpolicy.cilium.yml

@@ -55,6 +55,9 @@ spec:
               # DNS lookup for sentry
               - matchPattern: "*.ingest.sentry.io"
 {{- end }}
+{{- if (default true .Values.server.mp.enabled ) -}}
+              - matchName: 'analytics.speckle.systems'
+{{- end }}
 {{- if .Values.server.email.enabled }}
               # email server
 {{ include "speckle.networkpolicy.dns.email.cilium" $ | indent 14 }}

utils/helm/speckle-server/templates/server/networkpolicy.kubernetes.yml

@@ -30,7 +30,7 @@ spec:
       ports:
         - port: 443
 {{- end }}
-{{- if ( or .Values.server.auth.google.enabled .Values.server.auth.github.enabled .Values.server.auth.azure_ad.enabled .Values.server.auth.oidc.enabled ) }}
+{{- if ( or .Values.server.auth.google.enabled .Values.server.auth.github.enabled .Values.server.auth.azure_ad.enabled .Values.server.auth.oidc.enabled (default true .Values.server.mp.enabled) ) }}
     - to:
       - ipBlock:
           cidr: 0.0.0.0/0

yarn.lock

@@ -5296,6 +5296,7 @@ __metadata:
     ioredis: ^5.2.2
     knex: ^2.4.1
     lodash: ^4.17.21
+    mixpanel: ^0.17.0
     mjml: ^4.13.0
     mocha: ^10.1.0
     mocha-junit-reporter: ^2.0.2
@@ -12857,6 +12858,16 @@ __metadata:
   languageName: node
   linkType: hard
 
+"https-proxy-agent@npm:5.0.0":
+  version: 5.0.0
+  resolution: "https-proxy-agent@npm:5.0.0"
+  dependencies:
+    agent-base: 6
+    debug: 4
+  checksum: 165bfb090bd26d47693597661298006841ab733d0c7383a8cb2f17373387a94c903a3ac687090aa739de05e379ab6f868bae84ab4eac288ad85c328cd1ec9e53
+  languageName: node
+  linkType: hard
+
 "https-proxy-agent@npm:5.0.1, https-proxy-agent@npm:^5.0.0":
   version: 5.0.1
   resolution: "https-proxy-agent@npm:5.0.1"
@@ -14697,6 +14708,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"mixpanel@npm:^0.17.0":
+  version: 0.17.0
+  resolution: "mixpanel@npm:0.17.0"
+  dependencies:
+    https-proxy-agent: 5.0.0
+  checksum: 43afa7dfd5ad199318f9b5555bc8b4ed1e0536ad87545c7b9290d3c76da665f86fb7c336eb9346a026f71959039658e8fcea9a9eafdf2a743c9ec11789e6b143
+  languageName: node
+  linkType: hard
+
 "mjml-accordion@npm:4.13.0":
   version: 4.13.0
   resolution: "mjml-accordion@npm:4.13.0"