Immutability: store new repo payloads to immutable blob

api/client/organizations.ts

@@ -16,6 +16,8 @@ import {
   OrganizationManagementType,
 } from '../../middleware/business/organization';
 
+import memoryCache from 'memory-cache';
+
 const router: Router = Router();
 
 router.get(
@@ -34,6 +36,45 @@ router.get(
   })
 );
 
+router.get(
+  '/annotations',
+  asyncHandler(async (req: ReposAppRequest, res, next) => {
+    const { operations, organizationAnnotationsProvider } = getProviders(req);
+    const cacheTimeMs = 1000 * 60 * 60 * 24;
+    try {
+      const highlights = [];
+      const annotations = await organizationAnnotationsProvider.getAllAnnotations();
+      for (const annotation of annotations) {
+        try {
+          const key = `org:profile:${annotation.organizationId}`;
+          let profile = memoryCache.get(key);
+          if (!profile) {
+            const details = await operations.getOrganizationProfileById(Number(annotation.organizationId));
+            details.cost && delete details.cost;
+            details.headers && delete details.headers;
+            profile = details;
+            memoryCache.put(key, details, cacheTimeMs);
+          }
+          const scrubbedAnnotations = { ...annotation };
+          delete scrubbedAnnotations.administratorNotes;
+          delete scrubbedAnnotations.history;
+          highlights.push({
+            profile,
+            annotations: scrubbedAnnotations,
+          });
+        } catch (error) {
+          // we ignore any individual resolution error
+        }
+      }
+      return res.json({
+        highlights,
+      });
+    } catch (error) {
+      throw jsonError(error, 400);
+    }
+  })
+);
+
 router.get(
   '/list.txt',
   asyncHandler(async (req: ReposAppRequest, res, next) => {

api/webhook.ts

@@ -31,7 +31,8 @@ router.use(
       );
     }
 
-    const { operations } = getProviders(req);
+    const providers = getProviders(req);
+    const { operations } = providers;
     const body = req.body;
     const orgName = body && body.organization && body.organization.login ? body.organization.login : null;
     if (!orgName) {
@@ -63,7 +64,7 @@ router.use(
       rawBody: req._raw,
     };
     const options = {
-      operations,
+      providers,
       organization: req.organization,
       event,
     };

config/index.types.ts

@@ -46,6 +46,7 @@ import type { ConfigRootUserAgent } from './userAgent.types';
 import type { ConfigRootWebHealthProbes } from './webHealthProbes.types';
 import type { ConfigRootWeb } from './web.types';
 import type { ConfigRootWebServer } from './webServer.types';
+import { ConfigRootImmutable } from './immutable.types';
 
 type ObfuscatedConfig = any;
 
@@ -75,6 +76,7 @@ export type SiteConfiguration =
   ConfigRootFeatures &
   ConfigRootGitHub &
   ConfigRootGraph &
+  ConfigRootImmutable &
   ConfigRootImpersonation &
   ConfigRootJit &
   ConfigRootJobs &

interfaces/providers.ts

@@ -43,6 +43,7 @@ import { IEntityMetadataProvider } from '../lib/entityMetadataProvider';
 import { IRepositoryProvider } from '../entities/repository';
 import { IKeyVaultSecretResolver } from '../lib/keyVaultResolver';
 import { IOrganizationAnnotationMetadataProvider } from '../entities/organizationAnnotation';
+import type { IImmutableStorageProvider } from '../lib/immutable';
 
 type ProviderGenerator = (value: string) => IEntityMetadataProvider;
 
@@ -67,6 +68,7 @@ export interface IProviders {
   getEntityProviderByType?: ProviderGenerator;
   github?: RestLibrary;
   graphProvider?: IGraphProvider;
+  immutable?: IImmutableStorageProvider;
   insights?: TelemetryClient;
   linkProvider?: ILinkProvider;
   localExtensionKeyProvider?: ILocalExtensionKeyProvider;

jobs/firehose/task.ts

@@ -290,7 +290,7 @@ export default async function firehose({ providers, started }: IReposJob): Promi
       return;
     }
     const options = {
-      operations,
+      providers,
       organization,
       event: {
         properties: message.customProperties as unknown as IGitHubWebhookProperties,

middleware/initialize.ts

@@ -83,6 +83,7 @@ import {
   SiteConfiguration,
 } from '../interfaces';
 import initializeRepositoryProvider from '../entities/repository';
+import { tryGetImmutableStorageProvider } from '../lib/immutable';
 
 const DefaultApplicationProfile: IApplicationProfile = {
   applicationName: 'GitHub Management Portal',
@@ -125,6 +126,12 @@ async function initializeAsync(
     throw new Error('No cache provider available');
   }
 
+  const immutable = tryGetImmutableStorageProvider(config);
+  if (immutable) {
+    await immutable.initialize();
+    providers.immutable = immutable;
+  }
+
   providers.graphProvider = await createGraphProvider(providers, config);
   app.set('graphProvider', providers.graphProvider);
 

webhooks/organizationProcessor.ts

@@ -11,6 +11,7 @@ import { Organization } from '../business';
 
 import Tasks from './tasks';
 import { sleep } from '../utils';
+import { type IProviders } from '../interfaces';
 
 interface IValidationError extends Error {
   statusCode?: number;
@@ -19,7 +20,7 @@ interface IValidationError extends Error {
 
 export abstract class WebhookProcessor {
   abstract filter(data: any): boolean;
-  abstract run(operations: Operations, organization: Organization, data: any): Promise<boolean>;
+  abstract run(providers: IProviders, organization: Organization, data: any): Promise<boolean>;
 }
 
 export interface IOrganizationWebhookEvent {
@@ -36,7 +37,7 @@ export interface IGitHubWebhookProperties {
 }
 
 export interface IProcessOrganizationWebhookOptions {
-  operations: Operations;
+  providers: IProviders;
   organization: Organization;
   event: IOrganizationWebhookEvent;
   acknowledgeValidEvent?: any;
@@ -45,9 +46,9 @@ export interface IProcessOrganizationWebhookOptions {
 export default async function ProcessOrganizationWebhook(
   options: IProcessOrganizationWebhookOptions
 ): Promise<any> {
-  const operations = options.operations;
-  if (!operations) {
-    throw new Error('No operations instance provided');
+  const providers = options.providers;
+  if (!providers) {
+    throw new Error('No providers provided');
   }
   const organization = options.organization;
   const event = options.event;
@@ -119,7 +120,7 @@ export default async function ProcessOrganizationWebhook(
 
   for (const processor of work) {
     try {
-      await processor.run(operations, organization, event);
+      await processor.run(providers, organization, event);
     } catch (processInitializationError) {
       if (processInitializationError.status === 403) {
         console.log(`403: ${processInitializationError}`);

webhooks/tasks/auditLog.ts

@@ -9,10 +9,10 @@
 // for organizations, and also to import JSON-based audit export files.
 
 import { WebhookProcessor } from '../organizationProcessor';
-import { Operations } from '../../business';
 import { Organization } from '../../business';
 import { AuditLogRecord } from '../../entities/auditLogRecord/auditLogRecord';
 import { MapWebhookEventsToAuditEvents, AuditLogSource } from '../../entities/auditLogRecord';
+import type { IProviders } from '../../interfaces';
 
 // prettier-ignore
 const eventTypes = new Set([
@@ -30,8 +30,8 @@ const knownEventTypesToIgnore = new Set([
   'star',
 ]);
 
-async function runAsync(operations: Operations, organization: Organization, data: any) {
-  const { auditLogRecordProvider } = operations.providers;
+async function runAsync(providers: IProviders, organization: Organization, data: any) {
+  const { auditLogRecordProvider } = providers;
   if (!auditLogRecordProvider) {
     return;
   }
@@ -122,8 +122,8 @@ export default class AuditLogRecorderWebhookProcessor implements WebhookProcesso
     return has;
   }
 
-  async run(operations: Operations, organization: Organization, data: any): Promise<boolean> {
-    const result = await runAsync(operations, organization, data);
+  async run(providers: IProviders, organization: Organization, data: any): Promise<boolean> {
+    const result = await runAsync(providers, organization, data);
     return true;
   }
 }

webhooks/tasks/automaticTeams.ts

@@ -14,6 +14,7 @@ import RenderHtmlMail from '../../lib/emailRender';
 import { IMailProvider } from '../../lib/mailProvider';
 import getCompanySpecificDeployment from '../../middleware/companySpecificDeployment';
 import { GitHubRepositoryPermission } from '../../entities/repositoryMetadata/repositoryMetadata';
+import type { IProviders } from '../../interfaces';
 
 interface IAutomaticTeamsMail {
   to: string;
@@ -70,7 +71,8 @@ export default class AutomaticTeamsWebhookProcessor implements WebhookProcessor
     return false;
   }
 
-  async run(operations: Operations, organization: Organization, data: any): Promise<boolean> {
+  async run(providers: IProviders, organization: Organization, data: any): Promise<boolean> {
+    const operations = providers.operations as Operations;
     const eventType = data.properties.event;
     const eventAction = data.body.action;
     const { specialTeamIds, specialTeamLevels } = this.processOrgSpecialTeams(organization);

webhooks/tasks/member.ts

@@ -15,8 +15,8 @@ export default class MemberWebhookProcessor implements WebhookProcessor {
     return eventType === 'member';
   }
 
-  async run(operations: Operations, organization: Organization, data: any): Promise<any> {
-    const providers = operations.providers as IProviders;
+  async run(providers: IProviders, organization: Organization, data: any): Promise<any> {
+    const operations = providers.operations as Operations;
     const queryCache = providers.queryCache;
     const event = data.body;
     const organizationIdAsString = event.organization.id.toString();

webhooks/tasks/membership.ts

@@ -13,8 +13,8 @@ export default class MembershipWebhookProcessor implements WebhookProcessor {
     return eventType === 'membership';
   }
 
-  async run(operations: Operations, organization: Organization, data: any): Promise<any> {
-    const providers = operations.providers as IProviders;
+  async run(providers: IProviders, organization: Organization, data: any): Promise<any> {
+    const operations = providers.operations as Operations;
     const queryCache = providers.queryCache;
     const event = data.body;
     const organizationId = event.organization.id as number;

webhooks/tasks/organization.ts

@@ -5,11 +5,10 @@
 
 // ORGANIZATION membership and ownership
 
-import { Operations } from '../../business';
 import { Organization } from '../../business';
 import {
   OrganizationMembershipRole,
-  IProviders,
+  type IProviders,
   NoCacheNoBackground,
   OrganizationMembershipState,
 } from '../../interfaces';
@@ -34,8 +33,7 @@ export default class OrganizationWebhookProcessor implements WebhookProcessor {
     return eventType === 'organization';
   }
 
-  async run(operations: Operations, organization: Organization, data: any): Promise<boolean> {
-    const providers = operations.providers as IProviders;
+  async run(providers: IProviders, organization: Organization, data: any): Promise<boolean> {
     const queryCache = providers.queryCache;
 
     const event = data.body;

webhooks/tasks/repository.ts

@@ -6,10 +6,9 @@
 // REPOSITORY created or updated
 
 import { WebhookProcessor } from '../organizationProcessor';
-import { Operations } from '../../business';
 import { Organization } from '../../business';
 import NewRepositoryLockdownSystem from '../../features/newRepositoryLockdown';
-import { getRepositoryMetadataProvider } from '../../interfaces';
+import { getRepositoryMetadataProvider, type IProviders } from '../../interfaces';
 
 export default class RepositoryWebhookProcessor implements WebhookProcessor {
   filter(data: any) {
@@ -17,7 +16,8 @@ export default class RepositoryWebhookProcessor implements WebhookProcessor {
     return eventType === 'repository';
   }
 
-  async run(operations: Operations, organization: Organization, data: any): Promise<boolean> {
+  async run(providers: IProviders, organization: Organization, data: any): Promise<boolean> {
+    const { immutable, operations } = providers;
     const event = data.body;
     const queryCache = operations.providers.queryCache;
     let update = false;
@@ -32,6 +32,11 @@ export default class RepositoryWebhookProcessor implements WebhookProcessor {
       );
       return true;
     }
+    immutable?.saveObjectInBackground(
+      `orgs/${event?.organization?.login}/repos/${event?.repository?.name}/webhooks`,
+      action || 'unknown',
+      data
+    );
     if (action === 'created' || action === 'transferred') {
       console.log(
         `repo ${action}: ${event.repository.full_name} ${

webhooks/tasks/team.ts

@@ -11,6 +11,7 @@ import { WebhookProcessor } from '../organizationProcessor';
 import { Operations } from '../../business';
 import { Organization } from '../../business';
 import { permissionsObjectToValue } from '../../transitional';
+import type { IProviders } from '../../interfaces';
 
 // When teams are added or removed on GitHub, refresh the organization's list of
 // teams as well as the cross-organization view of the teams.
@@ -24,8 +25,9 @@ export default class TeamWebhookProcessor implements WebhookProcessor {
     return eventType === 'team';
   }
 
-  async run(operations: Operations, organization: Organization, data: any): Promise<boolean> {
-    const queryCache = operations.providers.queryCache;
+  async run(providers: IProviders, organization: Organization, data: any): Promise<boolean> {
+    const operations = providers.operations as Operations;
+    const queryCache = providers.queryCache;
     const event = data.body;
     let refresh = false;
     let expectedAfterRefresh = false;