Merge pull request #45930 from nextcloud/repair-mimetype-expensive

fix: move repair mimetype repair step to the expensive steps

apps/settings/composer/composer/autoload_classmap.php

@@ -102,6 +102,7 @@ return array(
     'OCA\\Settings\\SetupChecks\\LegacySSEKeyFormat' => $baseDir . '/../lib/SetupChecks/LegacySSEKeyFormat.php',
     'OCA\\Settings\\SetupChecks\\MaintenanceWindowStart' => $baseDir . '/../lib/SetupChecks/MaintenanceWindowStart.php',
     'OCA\\Settings\\SetupChecks\\MemcacheConfigured' => $baseDir . '/../lib/SetupChecks/MemcacheConfigured.php',
+    'OCA\\Settings\\SetupChecks\\MimeTypeMigrationAvailable' => $baseDir . '/../lib/SetupChecks/MimeTypeMigrationAvailable.php',
     'OCA\\Settings\\SetupChecks\\MysqlUnicodeSupport' => $baseDir . '/../lib/SetupChecks/MysqlUnicodeSupport.php',
     'OCA\\Settings\\SetupChecks\\OcxProviders' => $baseDir . '/../lib/SetupChecks/OcxProviders.php',
     'OCA\\Settings\\SetupChecks\\OverwriteCliUrl' => $baseDir . '/../lib/SetupChecks/OverwriteCliUrl.php',

apps/settings/composer/composer/autoload_static.php

@@ -117,6 +117,7 @@ class ComposerStaticInitSettings
         'OCA\\Settings\\SetupChecks\\LegacySSEKeyFormat' => __DIR__ . '/..' . '/../lib/SetupChecks/LegacySSEKeyFormat.php',
         'OCA\\Settings\\SetupChecks\\MaintenanceWindowStart' => __DIR__ . '/..' . '/../lib/SetupChecks/MaintenanceWindowStart.php',
         'OCA\\Settings\\SetupChecks\\MemcacheConfigured' => __DIR__ . '/..' . '/../lib/SetupChecks/MemcacheConfigured.php',
+        'OCA\\Settings\\SetupChecks\\MimeTypeMigrationAvailable' => __DIR__ . '/..' . '/../lib/SetupChecks/MimeTypeMigrationAvailable.php',
         'OCA\\Settings\\SetupChecks\\MysqlUnicodeSupport' => __DIR__ . '/..' . '/../lib/SetupChecks/MysqlUnicodeSupport.php',
         'OCA\\Settings\\SetupChecks\\OcxProviders' => __DIR__ . '/..' . '/../lib/SetupChecks/OcxProviders.php',
         'OCA\\Settings\\SetupChecks\\OverwriteCliUrl' => __DIR__ . '/..' . '/../lib/SetupChecks/OverwriteCliUrl.php',

apps/settings/lib/AppInfo/Application.php

@@ -45,6 +45,7 @@ use OCA\Settings\SetupChecks\JavaScriptSourceMaps;
 use OCA\Settings\SetupChecks\LegacySSEKeyFormat;
 use OCA\Settings\SetupChecks\MaintenanceWindowStart;
 use OCA\Settings\SetupChecks\MemcacheConfigured;
+use OCA\Settings\SetupChecks\MimeTypeMigrationAvailable;
 use OCA\Settings\SetupChecks\MysqlUnicodeSupport;
 use OCA\Settings\SetupChecks\OcxProviders;
 use OCA\Settings\SetupChecks\OverwriteCliUrl;
@@ -176,6 +177,7 @@ class Application extends App implements IBootstrap {
 		$context->registerSetupCheck(LegacySSEKeyFormat::class);
 		$context->registerSetupCheck(MaintenanceWindowStart::class);
 		$context->registerSetupCheck(MemcacheConfigured::class);
+		$context->registerSetupCheck(MimeTypeMigrationAvailable::class);
 		$context->registerSetupCheck(MysqlUnicodeSupport::class);
 		$context->registerSetupCheck(OcxProviders::class);
 		$context->registerSetupCheck(OverwriteCliUrl::class);

apps/settings/lib/SetupChecks/MimeTypeMigrationAvailable.php

@@ -0,0 +1,44 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+namespace OCA\Settings\SetupChecks;
+
+use OC\Repair\RepairMimeTypes;
+use OCP\IL10N;
+use OCP\L10N\IFactory;
+use OCP\SetupCheck\ISetupCheck;
+use OCP\SetupCheck\SetupResult;
+
+class MimeTypeMigrationAvailable implements ISetupCheck {
+	private IL10N $l10n;
+
+	public function __construct(
+		IFactory $l10nFactory,
+		private RepairMimeTypes $repairMimeTypes,
+	) {
+		$this->l10n = $l10nFactory->get('core');
+	}
+
+	public function getCategory(): string {
+		return 'system';
+	}
+
+	public function getName(): string {
+		return $this->l10n->t('Mimetype migrations available');
+	}
+
+	public function run(): SetupResult {
+		if ($this->repairMimeTypes->migrationsAvailable()) {
+			return SetupResult::warning(
+				$this->l10n->t('One or more mimetype migrations are available. Occasionally new mimetypes are added to better handle certain file types. Migrating the mimetypes take a long time on larger instances so this is not done automatically during upgrades. Use the command `occ maintenance:repair --include-expensive` to perform the migrations.'),
+			);
+		} else {
+			return SetupResult::success('None');
+		}
+	}
+}

lib/composer/composer/autoload_classmap.php

@@ -1607,6 +1607,7 @@ return array(
     'OC\\MemoryInfo' => $baseDir . '/lib/private/MemoryInfo.php',
     'OC\\Migration\\BackgroundRepair' => $baseDir . '/lib/private/Migration/BackgroundRepair.php',
     'OC\\Migration\\ConsoleOutput' => $baseDir . '/lib/private/Migration/ConsoleOutput.php',
+    'OC\\Migration\\NullOutput' => $baseDir . '/lib/private/Migration/NullOutput.php',
     'OC\\Migration\\SimpleOutput' => $baseDir . '/lib/private/Migration/SimpleOutput.php',
     'OC\\NaturalSort' => $baseDir . '/lib/private/NaturalSort.php',
     'OC\\NaturalSort_DefaultCollator' => $baseDir . '/lib/private/NaturalSort_DefaultCollator.php',

lib/composer/composer/autoload_static.php

@@ -1640,6 +1640,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
         'OC\\MemoryInfo' => __DIR__ . '/../../..' . '/lib/private/MemoryInfo.php',
         'OC\\Migration\\BackgroundRepair' => __DIR__ . '/../../..' . '/lib/private/Migration/BackgroundRepair.php',
         'OC\\Migration\\ConsoleOutput' => __DIR__ . '/../../..' . '/lib/private/Migration/ConsoleOutput.php',
+        'OC\\Migration\\NullOutput' => __DIR__ . '/../../..' . '/lib/private/Migration/NullOutput.php',
         'OC\\Migration\\SimpleOutput' => __DIR__ . '/../../..' . '/lib/private/Migration/SimpleOutput.php',
         'OC\\NaturalSort' => __DIR__ . '/../../..' . '/lib/private/NaturalSort.php',
         'OC\\NaturalSort_DefaultCollator' => __DIR__ . '/../../..' . '/lib/private/NaturalSort_DefaultCollator.php',

lib/private/Migration/NullOutput.php

@@ -0,0 +1,35 @@
+<?php
+/**
+ * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+namespace OC\Migration;
+
+use OCP\Migration\IOutput;
+
+/**
+ * Class NullOutput
+ *
+ * A simple IOutput that discards all output
+ *
+ * @package OC\Migration
+ */
+class NullOutput implements IOutput {
+	public function debug(string $message): void {
+	}
+
+	public function info($message): void {
+	}
+
+	public function warning($message): void {
+	}
+
+	public function startProgress($max = 0): void {
+	}
+
+	public function advance($step = 1, $description = ''): void {
+	}
+
+	public function finishProgress(): void {
+	}
+}

lib/private/Repair.php

@@ -149,7 +149,6 @@ class Repair implements IOutput {
 	public static function getRepairSteps(): array {
 		return [
 			new Collation(\OC::$server->getConfig(), \OC::$server->get(LoggerInterface::class), \OC::$server->getDatabaseConnection(), false),
-			new RepairMimeTypes(\OC::$server->getConfig(), \OC::$server->getDatabaseConnection()),
 			new CleanTags(\OC::$server->getDatabaseConnection(), \OC::$server->getUserManager()),
 			new RepairInvalidShares(\OC::$server->getConfig(), \OC::$server->getDatabaseConnection()),
 			new MoveUpdaterStepFile(\OC::$server->getConfig()),
@@ -200,6 +199,7 @@ class Repair implements IOutput {
 	public static function getExpensiveRepairSteps() {
 		return [
 			new OldGroupMembershipShares(\OC::$server->getDatabaseConnection(), \OC::$server->getGroupManager()),
+			new RepairMimeTypes(\OC::$server->getConfig(), \OC::$server->getDatabaseConnection()),
 			\OC::$server->get(ValidatePhoneNumber::class),
 			\OC::$server->get(DeleteSchedulingObjects::class),
 		];

lib/private/Repair/RepairMimeTypes.php

@@ -1,12 +1,12 @@
 <?php
 
 /**
- * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors
- * SPDX-FileCopyrightText: 2016 ownCloud, Inc.
+ * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors
  * SPDX-License-Identifier: AGPL-3.0-only
  */
 namespace OC\Repair;
 
+use OC\Migration\NullOutput;
 use OCP\DB\QueryBuilder\IQueryBuilder;
 use OCP\IConfig;
 use OCP\IDBConnection;
@@ -14,18 +14,16 @@ use OCP\Migration\IOutput;
 use OCP\Migration\IRepairStep;
 
 class RepairMimeTypes implements IRepairStep {
-	/** @var IConfig */
-	protected $config;
-	/** @var IDBConnection */
-	protected $connection;
+	private bool $dryRun = false;
+	private int $changeCount = 0;
 
 	/** @var int */
 	protected $folderMimeTypeId;
 
-	public function __construct(IConfig $config,
-		IDBConnection $connection) {
-		$this->config = $config;
-		$this->connection = $connection;
+	public function __construct(
+		protected IConfig $config,
+		protected IDBConnection $connection
+	) {
 	}
 
 	public function getName() {
@@ -33,6 +31,10 @@ class RepairMimeTypes implements IRepairStep {
 	}
 
 	private function updateMimetypes($updatedMimetypes) {
+		if ($this->dryRun) {
+			$this->changeCount += count($updatedMimetypes);
+			return;
+		}
 		$query = $this->connection->getQueryBuilder();
 		$query->select('id')
 			->from('mimetypes')
@@ -236,77 +238,99 @@ class RepairMimeTypes implements IRepairStep {
 		return $this->updateMimetypes($updatedMimetypes);
 	}
 
+	public function migrationsAvailable(): bool {
+		$this->dryRun = true;
+		$this->run(new NullOutput());
+		$this->dryRun = false;
+		return $this->changeCount > 0;
+	}
+
+	private function getMimeTypeVersion(): string {
+		$serverVersion = $this->config->getSystemValueString('version', '0.0.0');
+		// 29.0.0.10 is the last version with a mimetype migration before it was moved to a separate version number
+		if (version_compare($serverVersion, '29.0.0.10', '>')) {
+			return $this->config->getAppValue('files', 'mimetype_version', '29.0.0.10');
+		} else {
+			return $serverVersion;
+		}
+	}
+
 	/**
 	 * Fix mime types
 	 */
 	public function run(IOutput $out) {
-		$ocVersionFromBeforeUpdate = $this->config->getSystemValueString('version', '0.0.0');
+		$serverVersion = $this->config->getSystemValueString('version', '0.0.0');
+		$mimeTypeVersion = $this->getMimeTypeVersion();
 
 		// NOTE TO DEVELOPERS: when adding new mime types, please make sure to
 		// add a version comparison to avoid doing it every time
 
-		if (version_compare($ocVersionFromBeforeUpdate, '12.0.0.14', '<') && $this->introduceImageTypes()) {
+		if (version_compare($mimeTypeVersion, '12.0.0.14', '<') && $this->introduceImageTypes()) {
 			$out->info('Fixed image mime types');
 		}
 
-		if (version_compare($ocVersionFromBeforeUpdate, '12.0.0.13', '<') && $this->introduceWindowsProgramTypes()) {
+		if (version_compare($mimeTypeVersion, '12.0.0.13', '<') && $this->introduceWindowsProgramTypes()) {
 			$out->info('Fixed windows program mime types');
 		}
 
-		if (version_compare($ocVersionFromBeforeUpdate, '13.0.0.0', '<') && $this->introduceLocationTypes()) {
+		if (version_compare($mimeTypeVersion, '13.0.0.0', '<') && $this->introduceLocationTypes()) {
 			$out->info('Fixed geospatial mime types');
 		}
 
-		if (version_compare($ocVersionFromBeforeUpdate, '13.0.0.3', '<') && $this->introduceInternetShortcutTypes()) {
+		if (version_compare($mimeTypeVersion, '13.0.0.3', '<') && $this->introduceInternetShortcutTypes()) {
 			$out->info('Fixed internet-shortcut mime types');
 		}
 
-		if (version_compare($ocVersionFromBeforeUpdate, '13.0.0.6', '<') && $this->introduceStreamingTypes()) {
+		if (version_compare($mimeTypeVersion, '13.0.0.6', '<') && $this->introduceStreamingTypes()) {
 			$out->info('Fixed streaming mime types');
 		}
 
-		if (version_compare($ocVersionFromBeforeUpdate, '14.0.0.8', '<') && $this->introduceVisioTypes()) {
+		if (version_compare($mimeTypeVersion, '14.0.0.8', '<') && $this->introduceVisioTypes()) {
 			$out->info('Fixed visio mime types');
 		}
 
-		if (version_compare($ocVersionFromBeforeUpdate, '14.0.0.10', '<') && $this->introduceComicbookTypes()) {
+		if (version_compare($mimeTypeVersion, '14.0.0.10', '<') && $this->introduceComicbookTypes()) {
 			$out->info('Fixed comicbook mime types');
 		}
 
-		if (version_compare($ocVersionFromBeforeUpdate, '20.0.0.5', '<') && $this->introduceOpenDocumentTemplates()) {
+		if (version_compare($mimeTypeVersion, '20.0.0.5', '<') && $this->introduceOpenDocumentTemplates()) {
 			$out->info('Fixed OpenDocument template mime types');
 		}
 
-		if (version_compare($ocVersionFromBeforeUpdate, '21.0.0.7', '<') && $this->introduceOrgModeType()) {
+		if (version_compare($mimeTypeVersion, '21.0.0.7', '<') && $this->introduceOrgModeType()) {
 			$out->info('Fixed orgmode mime types');
 		}
 
-		if (version_compare($ocVersionFromBeforeUpdate, '23.0.0.2', '<') && $this->introduceFlatOpenDocumentType()) {
+		if (version_compare($mimeTypeVersion, '23.0.0.2', '<') && $this->introduceFlatOpenDocumentType()) {
 			$out->info('Fixed Flat OpenDocument mime types');
 		}
 
-		if (version_compare($ocVersionFromBeforeUpdate, '25.0.0.2', '<') && $this->introduceOnlyofficeFormType()) {
+		if (version_compare($mimeTypeVersion, '25.0.0.2', '<') && $this->introduceOnlyofficeFormType()) {
 			$out->info('Fixed ONLYOFFICE Forms OpenXML mime types');
 		}
 
-		if (version_compare($ocVersionFromBeforeUpdate, '26.0.0.1', '<') && $this->introduceAsciidocType()) {
+		if (version_compare($mimeTypeVersion, '26.0.0.1', '<') && $this->introduceAsciidocType()) {
 			$out->info('Fixed AsciiDoc mime types');
 		}
 
-		if (version_compare($ocVersionFromBeforeUpdate, '28.0.0.5', '<') && $this->introduceEnhancedMetafileFormatType()) {
+		if (version_compare($mimeTypeVersion, '28.0.0.5', '<') && $this->introduceEnhancedMetafileFormatType()) {
 			$out->info('Fixed Enhanced Metafile Format mime types');
 		}
 
-		if (version_compare($ocVersionFromBeforeUpdate, '29.0.0.2', '<') && $this->introduceEmlAndMsgFormatType()) {
+		if (version_compare($mimeTypeVersion, '29.0.0.2', '<') && $this->introduceEmlAndMsgFormatType()) {
 			$out->info('Fixed eml and msg mime type');
 		}
 
-		if (version_compare($ocVersionFromBeforeUpdate, '29.0.0.6', '<') && $this->introduceAacAudioType()) {
+		if (version_compare($mimeTypeVersion, '29.0.0.6', '<') && $this->introduceAacAudioType()) {
 			$out->info('Fixed aac mime type');
 		}
 
-		if (version_compare($ocVersionFromBeforeUpdate, '29.0.0.10', '<') && $this->introduceReStructuredTextFormatType()) {
+		if (version_compare($mimeTypeVersion, '29.0.0.10', '<') && $this->introduceReStructuredTextFormatType()) {
 			$out->info('Fixed ReStructured Text mime type');
 		}
+
+		if (!$this->dryRun) {
+			$this->config->setAppValue('files', 'mimetype_version', $serverVersion);
+		}
 	}
 }

tests/lib/Repair/RepairMimeTypesTest.php

@@ -39,10 +39,12 @@ class RepairMimeTypesTest extends \Test\TestCase {
 		$config = $this->getMockBuilder(IConfig::class)
 			->disableOriginalConstructor()
 			->getMock();
-		$config->expects($this->any())
-			->method('getSystemValueString')
+		$config->method('getSystemValueString')
 			->with('version')
 			->willReturn('11.0.0.0');
+		$config->method('getAppValue')
+			->with('files', 'mimetype_version')
+			->willReturn('11.0.0.0');
 
 		$this->storage = new \OC\Files\Storage\Temporary([]);