Export forms for User Migration

Signed-off-by: Jonas Rittershofer <jotoeri@users.noreply.github.com>
2022-05-26 18:00:59 +02:00 · 2022-05-26 18:00:59 +02:00 · 9713694973
--- a/lib/AppInfo/Application.php
+++ b/lib/AppInfo/Application.php
@ -29,6 +29,7 @@ declare(strict_types=1);
 namespace OCA\Forms\AppInfo;

 use OCA\Forms\Capabilities;
+use OCA\Forms\FormsMigrator;
 use OCA\Forms\Listener\UserDeletedListener;
 use OCP\AppFramework\App;
 use OCP\AppFramework\Bootstrap\IBootstrap;
@ -57,6 +58,11 @@ class Application extends App implements IBootstrap {

 		$context->registerCapability(Capabilities::class);
 		$context->registerEventListener(UserDeletedEvent::class, UserDeletedListener::class);
+
+		// TODO: drop conditional registration once server-minversion is 24
+		if (method_exists($context, 'registerUserMigrator')) {
+			$context->registerUserMigrator(FormsMigrator::class);
+		}
 	}

 	/**
--- a/lib/FormsMigrator.php
+++ b/lib/FormsMigrator.php
@ -0,0 +1,271 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * @copyright Copyright (c) 2022 Jonas Rittershofer <jotoeri@users.noreply.github.com>
+ *
+ * @author Jonas Rittershofer <jotoeri@users.noreply.github.com>
+ *
+ * @license AGPL-3.0-or-later
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OCA\Forms;
+
+use OCA\Forms\AppInfo\Application;
+use OCA\Forms\Db\Answer;
+use OCA\Forms\Db\AnswerMapper;
+use OCA\Forms\Db\Form;
+use OCA\Forms\Db\FormMapper;
+use OCA\Forms\Db\Option;
+use OCA\Forms\Db\OptionMapper;
+use OCA\Forms\Db\Question;
+use OCA\Forms\Db\QuestionMapper;
+use OCA\Forms\Db\Submission;
+use OCA\Forms\Db\SubmissionMapper;
+use OCA\Forms\Service\FormsService;
+use OCA\Forms\Service\SubmissionService;
+use OCP\IL10N;
+use OCP\IUser;
+use OCP\IUserManager;
+use OCP\UserMigration\IExportDestination;
+use OCP\UserMigration\IImportSource;
+use OCP\UserMigration\IMigrator;
+use OCP\UserMigration\TMigratorBasicVersionHandling;
+use OCP\UserMigration\UserMigrationException;
+use Symfony\Component\Console\Output\OutputInterface;
+use Throwable;
+
+class FormsMigrator implements IMigrator {
+	use TMigratorBasicVersionHandling;
+
+	/** @var AnswerMapper */
+	private $answerMapper;
+
+	/** @var FormMapper */
+	private $formMapper;
+
+	/** @var OptionMapper */
+	private $optionMapper;
+
+	/** @var QuestionMapper */
+	private $questionMapper;
+
+	/** @var SubmissionMapper */
+	private $submissionMapper;
+
+	/** @var FormsService */
+	private $formsService;
+
+	/** @var SubmissionService */
+	private $submissionService;
+
+	/** @var IL10N */
+	private $l10n;
+
+	/** @var IUserManager */
+	private $userManager;
+
+	private const PATH_ROOT = Application::APP_ID . '/';
+	private const PATH_MYAPP_FILE = FormsMigrator::PATH_ROOT . 'forms.json';
+
+	public function __construct(AnswerMapper $answerMapper,
+								FormMapper $formMapper,
+								OptionMapper $optionMapper,
+								QuestionMapper $questionMapper,
+								SubmissionMapper $submissionMapper,
+								FormsService $formsService,
+								SubmissionService $submissionService,
+								IL10N $l10n,
+								IUserManager $userManager) {
+		$this->answerMapper = $answerMapper;
+		$this->formMapper = $formMapper;
+		$this->optionMapper = $optionMapper;
+		$this->questionMapper = $questionMapper;
+		$this->submissionMapper = $submissionMapper;
+		$this->formsService = $formsService;
+		$this->submissionService = $submissionService;
+		$this->l10n = $l10n;
+		$this->userManager = $userManager;
+	}
+
+	/**
+	 * Export user data
+	 *
+	 * @throws UserMigrationException
+	 */
+	public function export(IUser $user, IExportDestination $exportDestination, OutputInterface $output): void {
+		$output->writeln('Exporting forms information in ' . FormsMigrator::PATH_MYAPP_FILE . '…');
+
+		try {
+			$data = [];
+
+			$forms = $this->formMapper->findAllByOwnerId($user->getUID());
+			foreach ($forms as $form) {
+				$formData = $form->read();
+				$formData['questions'] = $this->formsService->getQuestions($formData['id']);
+				$formData['submissions'] = $this->submissionService->getSubmissions($formData['id']);
+
+				// Unset ids and hash as we will anyways create new ones on import. UID is known through export.
+				unset($formData['id']);
+				unset($formData['hash']);
+				unset($formData['ownerId']);
+				foreach ($formData['questions'] as $qKey => $question) {
+					// Do NOT unset ID of question here, as it is necessary for answers.
+					unset($formData['questions'][$qKey]['formId']);
+					foreach ($question['options'] as $oKey => $option) {
+						unset($formData['questions'][$qKey]['options'][$oKey]['id']);
+						unset($formData['questions'][$qKey]['options'][$oKey]['questionId']);
+					}
+				}
+				foreach ($formData['submissions'] as $sKey => $submission) {
+					unset($formData['submissions'][$sKey]['id']);
+					unset($formData['submissions'][$sKey]['formId']);
+					foreach ($submission['answers'] as $aKey => $answer) {
+						// Do NOT unset questionId here, it is necessary to identify question/answers.
+						unset($formData['submissions'][$sKey]['answers'][$aKey]['id']);
+						unset($formData['submissions'][$sKey]['answers'][$aKey]['submissionId']);
+					}
+				}
+
+				// Mark userIds with instance.
+				foreach ($formData['submissions'] as $sKey => $submission) {
+					// Anonymous submission or already migrated, just keep it.
+					if (substr($submission['userId'], 0, 10) === 'anon-user-' ||
+						substr($submission['userId'], 0, 8) === 'unknown~') {
+						continue;
+					}
+
+					// Try loading federated UserId, otherwise just mark userId as unknown.
+					$exportId = '';
+					$userEntity = $this->userManager->get($submission['userId']);
+					if ($userEntity instanceof IUser) {
+						$exportId = $userEntity->getCloudId();
+					} else {
+						// Fallback, should not occur regularly.
+						$exportId = 'unknown~' . $submission['userId'];
+					}
+					$formData['submissions'][$sKey]['userId'] = $exportId;
+				}
+
+				// Add to catalog
+				$data[] = $formData;
+			}
+
+			$exportDestination->addFileContents(FormsMigrator::PATH_MYAPP_FILE, json_encode($data));
+		} catch (Throwable $e) {
+			throw new UserMigrationException('Could not export forms', 0, $e);
+		}
+	}
+
+	/**
+	 * Import user data
+	 *
+	 * @throws UserMigrationException
+	 */
+	public function import(IUser $user, IImportSource $importSource, OutputInterface $output): void {
+		if ($importSource->getMigratorVersion($this->getId()) === null) {
+			$output->writeln('No version for ' . static::class . ', skipping import…');
+			return;
+		}
+
+		$output->writeln('Importing forms information from ' . FormsMigrator::PATH_MYAPP_FILE . '…');
+
+		$data = json_decode($importSource->getFileContents(FormsMigrator::PATH_MYAPP_FILE), true, 512, JSON_THROW_ON_ERROR);
+		try {
+			foreach ($data as $formData) {
+				$form = new Form();
+				$form->setHash($this->formsService->generateFormHash());
+				$form->setTitle($formData['title']);
+				$form->setDescription($formData['description']);
+				$form->setOwnerId($user->getUID());
+				$form->setCreated($formData['created']);
+				$form->setAccess($formData['access']);
+				$form->setExpires($formData['expires']);
+				$form->setIsAnonymous($formData['isAnonymous']);
+				$form->setSubmitMultiple($formData['submitMultiple']);
+		
+				$this->formMapper->insert($form);
+
+				$questionIdMap = [];
+				foreach ($formData['questions'] as $questionData) {
+					$question = new Question();
+					$question->setFormId($form->getId());
+					$question->setOrder($questionData['order']);
+					$question->setType($questionData['type']);
+					$question->setIsRequired($questionData['isRequired']);
+					$question->setText($questionData['text']);
+					$question->setDescription($questionData['description']);
+
+					$this->questionMapper->insert($question);
+
+					// Store QuestionId to map Answers.
+					$questionIdMap[$questionData['id']] = $question->getId();
+
+					foreach ($questionData['options'] as $optionData) {
+						$option = new Option();
+						$option->setQuestionId($question->getId());
+						$option->setText($optionData['text']);
+
+						$this->optionMapper->insert($option);
+					}
+				}
+
+				foreach ($formData['submissions'] as $submissionData) {
+					$submission = new Submission();
+					$submission->setFormId($form->getId());
+					$submission->setUserId($submissionData['userId']);
+					$submission->setTimestamp($submissionData['timestamp']);
+
+					$this->submissionMapper->insert($submission);
+
+					foreach ($submissionData['answers'] as $answerData) {
+						$answer = new Answer();
+						$answer->setSubmissionId($submission->getId());
+						$answer->setQuestionId($questionIdMap[$answerData['questionId']]);
+						$answer->setText($answerData['text']);
+
+						$this->answerMapper->insert($answer);
+					}
+				}
+			}
+		} catch (Throwable $e) {
+			throw new UserMigrationException('Could not properly import forms information', 0, $e);
+		}
+	}
+
+	/**
+	 * Unique AppID
+	 */
+	public function getId(): string {
+		return 'forms';
+	}
+
+	/**
+	 * App display name
+	 */
+	public function getDisplayName(): string {
+		return $this->l10n->t('Forms');
+	}
+
+	/**
+	 * Description for Data-Export
+	 */
+	public function getDescription(): string {
+		return $this->l10n->t('Forms including questions and submissions');
+	}
+}
--- a/tests/Unit/FormsMigratorTest.php
+++ b/tests/Unit/FormsMigratorTest.php
@ -0,0 +1,329 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * @copyright Copyright (c) 2022 Jonas Rittershofer <jotoeri@users.noreply.github.com>
+ *
+ * @author Jonas Rittershofer <jotoeri@users.noreply.github.com>
+ *
+ * @license AGPL-3.0-or-later
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+namespace OCA\Forms\Tests\Unit;
+
+use OCA\Forms\FormsMigrator;
+
+// use OCA\Forms\Db\Answer;
+use OCA\Forms\Db\AnswerMapper;
+use OCA\Forms\Db\Form;
+use OCA\Forms\Db\FormMapper;
+// use OCA\Forms\Db\Option;
+use OCA\Forms\Db\OptionMapper;
+// use OCA\Forms\Db\Question;
+use OCA\Forms\Db\QuestionMapper;
+// use OCA\Forms\Db\Submission;
+use OCA\Forms\Db\SubmissionMapper;
+use OCA\Forms\Service\FormsService;
+use OCA\Forms\Service\SubmissionService;
+use OCP\IL10N;
+use OCP\IUser;
+use OCP\IUserManager;
+use OCP\UserMigration\IExportDestination;
+use OCP\UserMigration\IImportSource;
+use Symfony\Component\Console\Output\OutputInterface;
+
+use PHPUnit\Framework\MockObject\MockObject;
+use Test\TestCase;
+
+class FormsMigratorTest extends TestCase {
+
+	/** @var FormsMigrator */
+	private $formsMigrator;
+
+	/** @var AnswerMapper|MockObject */
+	private $answerMapper;
+
+	/** @var FormMapper|MockObject */
+	private $formMapper;
+
+	/** @var OptionMapper|MockObject */
+	private $optionMapper;
+
+	/** @var QuestionMapper|MockObject */
+	private $questionMapper;
+
+	/** @var SubmissionMapper|MockObject */
+	private $submissionMapper;
+
+	/** @var FormsService|MockObject */
+	private $formsService;
+
+	/** @var SubmissionService|MockObject */
+	private $submissionService;
+
+	/** @var IL10N|MockObject */
+	private $l10n;
+
+	/** @var IUserManager|MockObject */
+	private $userManager;
+
+	public function setUp(): void {
+		parent::setUp();
+
+		// UserMigration is not available below NC24, skip all tests here.
+		if (\OC_Util::getVersion()[0] < 24) {
+			$this->markTestSkipped('UserMigration not available below NC24');
+		}
+
+		$this->answerMapper = $this->createMock(AnswerMapper::class);
+		$this->formMapper = $this->createMock(FormMapper::class);
+		$this->optionMapper = $this->createMock(OptionMapper::class);
+		$this->questionMapper = $this->createMock(QuestionMapper::class);
+		$this->submissionMapper = $this->createMock(SubmissionMapper::class);
+		$this->formsService = $this->createMock(FormsService::class);
+		$this->submissionService = $this->createMock(SubmissionService::class);
+		$this->l10n = $this->createMock(IL10N::class);
+		$this->userManager = $this->createMock(IUserManager::class);
+
+		$this->formsMigrator = new FormsMigrator(
+			$this->answerMapper,
+			$this->formMapper,
+			$this->optionMapper,
+			$this->questionMapper,
+			$this->submissionMapper,
+			$this->formsService,
+			$this->submissionService,
+			$this->l10n,
+			$this->userManager
+		);
+	}
+
+	public function dataExport() {
+		return [
+			'exactlyOneOfEach' => [
+				'expectedJson' => '[{"title":"Link","description":"","created":1646251830,"access":{"permitAllUsers":false,"showToAllUsers":false},"expires":0,"isAnonymous":false,"submitMultiple":false,"questions":[{"id":14,"order":2,"type":"multiple","isRequired":false,"text":"checkbox","description":"huhu","options":[{"text":"ans1"}]}],"submissions":[{"userId":"anyUser@localhost","timestamp":1651354059,"answers":[{"questionId":14,"text":"ans1"}]}]}]'
+			]
+		];
+	}
+	/**
+	 * @dataProvider dataExport
+	 *
+	 * @param string $expectedJson
+	 */
+	public function testExport(string $expectedJson) {
+		$user = $this->createMock(IUser::class);
+		$exportDestination = $this->createMock(IExportDestination::class);
+		$output = $this->createMock(OutputInterface::class);
+
+		$output->expects($this->once())
+			->method('writeln');
+
+		$user->expects($this->once())
+			->method('getUID')
+			->willReturn('someUser');
+
+		$form = new Form();
+		$form->setId(42);
+		$form->setHash('abcdefg');
+		$form->setTitle('Link');
+		$form->setDescription('');
+		$form->setOwnerId('someUser');
+		$form->setCreated(1646251830);
+		$form->setAccess([
+			'permitAllUsers' => false,
+			'showToAllUsers' => false
+		]);
+		$form->setExpires(0);
+		$form->setIsAnonymous(false);
+		$form->setSubmitMultiple(false);
+
+		$this->formsService->expects($this->once())
+			->method('getQuestions')
+			->with(42)
+			->willReturn([
+				[
+					"id" => 14,
+					'formId' => 42,
+					"order" => 2,
+					"type" => "multiple",
+					"isRequired" => false,
+					"text" => "checkbox",
+					"description" => "huhu",
+					"options" => [
+						[
+							'id' => 35,
+							'questionId' => 14,
+							"text" => "ans1"
+						]
+					]
+				]
+			]);
+		$this->submissionService->expects($this->once())
+			->method('getSubmissions')
+			->with(42)
+			->willReturn([
+				[
+					'id' => 28,
+					'formId' => 42,
+					'userId' => "anyUser",
+					"timestamp" => 1651354059,
+					'answers' => [
+						[
+							'id' => 35,
+							'submissionId' => 28,
+							"questionId" => 14,
+							"text" => "ans1"
+						]
+					]
+				]
+			]);
+
+		$this->formMapper->expects($this->once())
+			->method('findAllByOwnerId')
+			->with('someUser')
+			->willReturn([$form]);
+
+		$any_user = $this->createMock(IUser::class);
+		$any_user->expects($this->once())
+			->method('getCloudId')
+			->willReturn('anyUser@localhost');
+
+		$this->userManager->expects($this->once())
+			->method('get')
+			->with('anyUser')
+			->willReturn($any_user);
+
+		$exportDestination->expects($this->once())
+			->method('addFileContents')
+			->will($this->returnCallback(function ($path, $jsonData) use ($expectedJson) {
+				$this->assertEquals($expectedJson, $jsonData);
+				return;
+			}));
+		$this->formsMigrator->export($user, $exportDestination, $output);
+	}
+
+	public function dataImport() {
+		return [
+			'exactlyOneOfEach' => [
+				'$inputJson' => '[{"title":"Link","description":"","created":1646251830,"access":{"permitAllUsers":false,"showToAllUsers":false},"expires":0,"isAnonymous":false,"submitMultiple":false,"questions":[{"id":14,"order":2,"type":"multiple","isRequired":false,"text":"checkbox","description":"huhu","options":[{"text":"ans1"}]}],"submissions":[{"userId":"anyUser@localhost","timestamp":1651354059,"answers":[{"questionId":14,"text":"ans1"}]}]}]'
+			]
+		];
+	}
+
+	/**
+	 * @dataProvider dataImport
+	 *
+	 * @param string $inputJson JsonString to input
+	 */
+	public function testImport(string $inputJson) {
+		$user = $this->createMock(IUser::class);
+		$importSource = $this->createMock(IImportSource::class);
+		$output = $this->createMock(OutputInterface::class);
+
+		$importSource->expects($this->once())
+			->method('getMigratorVersion')
+			->with('forms')
+			->willReturn(1);
+		$importSource->expects($this->once())
+			->method('getFileContents')
+			->willReturn($inputJson);
+
+		$user->expects($this->once())
+			->method('getUID')
+			->willReturn('someUser');
+
+		$this->formsService->expects($this->once())
+			->method('generateFormHash')
+			->willReturn('abcdefg');
+
+		$this->formMapper->expects($this->once())->method('insert');
+		$this->questionMapper->expects($this->once())->method('insert');
+		$this->optionMapper->expects($this->once())->method('insert');
+		$this->submissionMapper->expects($this->once())->method('insert');
+		$this->answerMapper->expects($this->once())->method('insert');
+
+		$this->formsMigrator->import($user, $importSource, $output);
+	}
+
+	public function testImport_NoVersion() {
+		$user = $this->createMock(IUser::class);
+		$importSource = $this->createMock(IImportSource::class);
+		$output = $this->createMock(OutputInterface::class);
+
+		$importSource->expects($this->once())
+			->method('getMigratorVersion')
+			->with('forms')
+			->willReturn(null);
+		$output->expects($this->once())
+			->method('writeln');
+		$importSource->expects($this->never())
+			->method('getFileContents');
+
+		$this->formsMigrator->import($user, $importSource, $output);
+	}
+
+	public function testGetId() {
+		$this->assertEquals('forms', $this->formsMigrator->getId());
+	}
+
+	public function testGetDisplayName() {
+		$this->l10n->expects($this->once())
+			->method('t')
+			->with('Forms')
+			->willReturn('Translated Forms');
+		$this->assertEquals('Translated Forms', $this->formsMigrator->getDisplayName());
+	}
+
+	public function testGetDescription() {
+		$this->l10n->expects($this->once())
+			->method('t')
+			->with('Forms including questions and submissions')
+			->willReturn('Translated Description');
+		$this->assertEquals('Translated Description', $this->formsMigrator->getDescription());
+	}
+
+	public function testGetVersion() {
+		$this->assertEquals(1, $this->formsMigrator->getVersion());
+	}
+
+	public function dataCanImport() {
+		return [
+			'goodVersion' => [
+				'version' => 1,
+				'expected' => true
+			],
+			'badVersion' => [
+				'version' => 2,
+				'expected' => false
+			],
+		];
+	}
+
+	/**
+	 * @dataProvider dataCanImport
+	 *
+	 * @param int $version Version to import
+	 * @param bool $expected Expected boolean result
+	 */
+	public function testCanImport(int $version, bool $expected) {
+		$importSource = $this->createMock(IImportSource::class);
+		$importSource->expects($this->once())
+			->method('getMigratorVersion')
+			->willReturn($version);
+
+		$this->assertEquals($expected, $this->formsMigrator->canImport($importSource));
+	}
+}
--- a/tests/Unit/Service/SubmissionServiceTest.php
+++ b/tests/Unit/Service/SubmissionServiceTest.php
@ -129,6 +129,74 @@ class SubmissionServiceTest extends TestCase {
 		);
 	}

+	public function testGetSubmissions() {
+		$submission_1 = new Submission();
+		$submission_1->setId(42);
+		$submission_1->setFormId(5);
+		$submission_1->setUserId('someUser');
+		$submission_1->setTimestamp(123456);
+		$answer_1 = new Answer();
+		$answer_1->setId(35);
+		$answer_1->setSubmissionId(42);
+		$answer_1->setQuestionId(422);
+		$answer_1->setText('Just some Text');
+		$answer_2 = new Answer();
+		$answer_2->setId(36);
+		$answer_2->setSubmissionId(42);
+		$answer_2->setQuestionId(423);
+		$answer_2->setText('Just some more Text');
+
+		$submission_2 = new Submission();
+		$submission_2->setId(43);
+		$submission_2->setFormId(5);
+		$submission_2->setUserId('someOtherUser');
+		$submission_2->setTimestamp(1234);
+
+		$this->submissionMapper->expects($this->once())
+			->method('findByForm')
+			->with(5)
+			->willReturn([$submission_1, $submission_2]);
+
+		$this->answerMapper->expects($this->any())
+			->method('findBySubmission')
+			->will($this->returnValueMap([
+				[42, [$answer_1, $answer_2]],
+				[43, []]
+			]));
+
+		$expected = [
+			[
+				'id' => 42,
+				'formId' => 5,
+				'userId' => 'someUser',
+				'timestamp' => 123456,
+				'answers' => [
+					[
+						'id' => 35,
+						'submissionId' => 42,
+						'questionId' => 422,
+						'text' => 'Just some Text'
+					],
+					[
+						'id' => 36,
+						'submissionId' => 42,
+						'questionId' => 423,
+						'text' => 'Just some more Text'
+					]
+				]
+			],
+			[
+				'id' => 43,
+				'formId' => 5,
+				'userId' => 'someOtherUser',
+				'timestamp' => 1234,
+				'answers' => []
+			]
+		];
+
+		$this->assertEquals($expected, $this->submissionService->getSubmissions(5));
+	}
+
 	public function dataWriteCsvToCloud() {
 		return [
 			'rootFolder' => ['', Folder::class, '', 'Some nice Form Title (responses).csv', false],