Merge pull request #3096 from remcohaszing/mdx

Add mdx language
2023-07-07 16:55:57 +02:00 · 2023-07-07 16:55:57 +02:00 · e6c4acec69
--- a/src/basic-languages/mdx/mdx.contribution.ts
+++ b/src/basic-languages/mdx/mdx.contribution.ts
@ -0,0 +1,24 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { registerLanguage } from '../_.contribution';
+
+declare var AMD: any;
+declare var require: any;
+
+registerLanguage({
+	id: 'mdx',
+	extensions: ['.mdx'],
+	aliases: ['MDX', 'mdx'],
+	loader: () => {
+		if (AMD) {
+			return new Promise((resolve, reject) => {
+				require(['vs/basic-languages/mdx/mdx'], resolve, reject);
+			});
+		} else {
+			return import('./mdx');
+		}
+	}
+});
--- a/src/basic-languages/mdx/mdx.test.ts
+++ b/src/basic-languages/mdx/mdx.test.ts
@ -0,0 +1,171 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { testTokenization } from '../test/testRunner';
+
+testTokenization(
+	['mdx', 'yaml'],
+	[
+		// headers
+		[
+			{
+				line: '# header 1',
+				tokens: [{ startIndex: 0, type: 'keyword.mdx' }]
+			},
+			{
+				line: '## header 2',
+				tokens: [{ startIndex: 0, type: 'keyword.mdx' }]
+			},
+			{
+				line: '### header 3',
+				tokens: [{ startIndex: 0, type: 'keyword.mdx' }]
+			},
+			{
+				line: '#### header 4',
+				tokens: [{ startIndex: 0, type: 'keyword.mdx' }]
+			},
+			{
+				line: '##### header 5',
+				tokens: [{ startIndex: 0, type: 'keyword.mdx' }]
+			},
+			{
+				line: '###### header 6',
+				tokens: [{ startIndex: 0, type: 'keyword.mdx' }]
+			}
+		],
+
+		// Lists
+		[
+			{
+				line: '- apple',
+				tokens: [
+					{ startIndex: 0, type: 'keyword.mdx' },
+					{ startIndex: 1, type: 'white.mdx' },
+					{ startIndex: 2, type: '' }
+				]
+			},
+			{
+				line: '* pear',
+				tokens: [
+					{ startIndex: 0, type: 'keyword.mdx' },
+					{ startIndex: 1, type: 'white.mdx' },
+					{ startIndex: 2, type: '' }
+				]
+			},
+			{
+				line: '+ pineapple',
+				tokens: [
+					{ startIndex: 0, type: 'keyword.mdx' },
+					{ startIndex: 1, type: 'white.mdx' },
+					{ startIndex: 2, type: '' }
+				]
+			},
+			{
+				line: '1. orange',
+				tokens: [
+					{ startIndex: 0, type: 'number.mdx' },
+					{ startIndex: 2, type: 'white.mdx' },
+					{ startIndex: 3, type: '' }
+				]
+			}
+		],
+
+		// Frontmatter
+		[
+			{
+				line: '---',
+				tokens: [{ startIndex: 0, type: 'meta.content.mdx' }]
+			},
+			{
+				line: 'frontmatter: yaml',
+				tokens: [
+					{ startIndex: 0, type: 'type.yaml' },
+					{ startIndex: 11, type: 'operators.yaml' },
+					{ startIndex: 12, type: 'white.yaml' },
+					{ startIndex: 13, type: 'string.yaml' }
+				]
+			},
+			{
+				line: '---',
+				tokens: [{ startIndex: 0, type: 'meta.content.mdx' }]
+			}
+		],
+
+		// links
+		[
+			{
+				line: '[MDX](https://mdxjs.com)',
+				tokens: [
+					{ startIndex: 0, type: '' },
+					{ startIndex: 1, type: 'type.identifier.mdx' },
+					{ startIndex: 4, type: '' },
+					{ startIndex: 6, type: 'string.link.mdx' },
+					{ startIndex: 23, type: '' }
+				]
+			},
+			{
+				line: '[monaco][monaco]',
+				tokens: [
+					{ startIndex: 0, type: '' },
+					{ startIndex: 1, type: 'type.identifier.mdx' },
+					{ startIndex: 7, type: '' },
+					{ startIndex: 9, type: 'type.identifier.mdx' },
+					{ startIndex: 15, type: '' }
+				]
+			},
+			{
+				line: '[monaco][]',
+				tokens: [
+					{ startIndex: 0, type: '' },
+					{ startIndex: 1, type: 'type.identifier.mdx' },
+					{ startIndex: 9, type: '' }
+				]
+			},
+			{
+				line: '[monaco]',
+				tokens: [
+					{ startIndex: 0, type: '' },
+					{ startIndex: 1, type: 'type.identifier.mdx' },
+					{ startIndex: 7, type: '' }
+				]
+			},
+			{
+				line: '[monaco]: https://github.com/microsoft/monaco-editor',
+				tokens: [
+					{ startIndex: 0, type: '' },
+					{ startIndex: 1, type: 'type.identifier.mdx' },
+					{ startIndex: 7, type: '' },
+					{ startIndex: 10, type: 'string.link.mdx' }
+				]
+			}
+		],
+
+		// JSX
+		[
+			{
+				line: '<div>**child**</div>',
+				tokens: [
+					{ startIndex: 0, type: 'type.identifier.mdx' },
+					// This is incorrect. MDX children that start on the same line are JSX, not markdown
+					{ startIndex: 5, type: 'strong.mdx' },
+					{ startIndex: 14, type: 'type.identifier.mdx' }
+				]
+			},
+			{
+				line: '{console.log("This is JavaScript")}',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.bracket.mdx' },
+					{ startIndex: 1, type: 'identifier.js' },
+					{ startIndex: 8, type: 'delimiter.js' },
+					{ startIndex: 9, type: 'identifier.js' },
+					{ startIndex: 12, type: 'delimiter.parenthesis.js' },
+					{ startIndex: 13, type: 'string.js' },
+					{ startIndex: 33, type: 'delimiter.parenthesis.js' },
+					{ startIndex: 34, type: 'delimiter.bracket.mdx' }
+				]
+			}
+		]
+	]
+);
--- a/src/basic-languages/mdx/mdx.ts
+++ b/src/basic-languages/mdx/mdx.ts
@ -0,0 +1,163 @@
+/*---------------------------------------------------------------------------------------------
+ *	Copyright (c) Microsoft Corporation. All rights reserved.
+ *	Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { languages } from '../../fillers/monaco-editor-core';
+
+export const conf: languages.LanguageConfiguration = {
+	comments: {
+		blockComment: ['{/*', '*/}']
+	},
+	brackets: [['{', '}']],
+	autoClosingPairs: [
+		{ open: '"', close: '"' },
+		{ open: "'", close: "'" },
+		{ open: '“', close: '”' },
+		{ open: '‘', close: '’' },
+		{ open: '`', close: '`' },
+		{ open: '{', close: '}' },
+		{ open: '(', close: ')' },
+		{ open: '_', close: '_' },
+		{ open: '**', close: '**' },
+		{ open: '<', close: '>' }
+	],
+	onEnterRules: [
+		{
+			beforeText: /^\s*- .+/,
+			action: { indentAction: languages.IndentAction.None, appendText: '- ' }
+		},
+		{
+			beforeText: /^\s*\+ .+/,
+			action: { indentAction: languages.IndentAction.None, appendText: '+ ' }
+		},
+		{
+			beforeText: /^\s*\* .+/,
+			action: { indentAction: languages.IndentAction.None, appendText: '* ' }
+		},
+		{
+			beforeText: /^> /,
+			action: { indentAction: languages.IndentAction.None, appendText: '> ' }
+		},
+		{
+			beforeText: /<\w+/,
+			action: { indentAction: languages.IndentAction.Indent }
+		},
+		{
+			beforeText: /\s+>\s*$/,
+			action: { indentAction: languages.IndentAction.Indent }
+		},
+		{
+			beforeText: /<\/\w+>/,
+			action: { indentAction: languages.IndentAction.Outdent }
+		},
+		...Array.from({ length: 100 }, (_, index) => ({
+			beforeText: new RegExp(`^${index}\\. .+`),
+			action: { indentAction: languages.IndentAction.None, appendText: `${index + 1}. ` }
+		}))
+	]
+};
+
+export const language = <languages.IMonarchLanguage>{
+	defaultToken: '',
+	tokenPostfix: '.mdx',
+	control: /[!#()*+.[\\\]_`{}\-]/,
+	escapes: /\\@control/,
+
+	tokenizer: {
+		root: [
+			[/^---$/, { token: 'meta.content', next: '@frontmatter', nextEmbedded: 'yaml' }],
+			[/^\s*import/, { token: 'keyword', next: '@import', nextEmbedded: 'js' }],
+			[/^\s*export/, { token: 'keyword', next: '@export', nextEmbedded: 'js' }],
+			[/<\w+/, { token: 'type.identifier', next: '@jsx' }],
+			[/<\/?\w+>/, 'type.identifier'],
+			[
+				/^(\s*)(>*\s*)(#{1,6}\s)/,
+				[{ token: 'white' }, { token: 'comment' }, { token: 'keyword', next: '@header' }]
+			],
+			[/^(\s*)(>*\s*)([*+-])(\s+)/, ['white', 'comment', 'keyword', 'white']],
+			[/^(\s*)(>*\s*)(\d{1,9}\.)(\s+)/, ['white', 'comment', 'number', 'white']],
+			[/^(\s*)(>*\s*)(\d{1,9}\.)(\s+)/, ['white', 'comment', 'number', 'white']],
+			[/^(\s*)(>*\s*)(-{3,}|\*{3,}|_{3,})$/, ['white', 'comment', 'keyword']],
+			[/`{3,}(\s.*)?$/, { token: 'string', next: '@codeblock_backtick' }],
+			[/~{3,}(\s.*)?$/, { token: 'string', next: '@codeblock_tilde' }],
+			[
+				/`{3,}(\S+).*$/,
+				{ token: 'string', next: '@codeblock_highlight_backtick', nextEmbedded: '$1' }
+			],
+			[
+				/~{3,}(\S+).*$/,
+				{ token: 'string', next: '@codeblock_highlight_tilde', nextEmbedded: '$1' }
+			],
+			[/^(\s*)(-{4,})$/, ['white', 'comment']],
+			[/^(\s*)(>+)/, ['white', 'comment']],
+			{ include: 'content' }
+		],
+		content: [
+			[
+				/(\[)(.+)(]\()(.+)(\s+".*")(\))/,
+				['', 'string.link', '', 'type.identifier', 'string.link', '']
+			],
+			[/(\[)(.+)(]\()(.+)(\))/, ['', 'type.identifier', '', 'string.link', '']],
+			[/(\[)(.+)(]\[)(.+)(])/, ['', 'type.identifier', '', 'type.identifier', '']],
+			[/(\[)(.+)(]:\s+)(\S*)/, ['', 'type.identifier', '', 'string.link']],
+			[/(\[)(.+)(])/, ['', 'type.identifier', '']],
+			[/`.*`/, 'variable.source'],
+			[/_/, { token: 'emphasis', next: '@emphasis_underscore' }],
+			[/\*(?!\*)/, { token: 'emphasis', next: '@emphasis_asterisk' }],
+			[/\*\*/, { token: 'strong', next: '@strong' }],
+			[/{/, { token: 'delimiter.bracket', next: '@expression', nextEmbedded: 'js' }]
+		],
+		import: [[/'\s*(;|$)/, { token: 'string', next: '@pop', nextEmbedded: '@pop' }]],
+		expression: [
+			[/{/, { token: 'delimiter.bracket', next: '@expression' }],
+			[/}/, { token: 'delimiter.bracket', next: '@pop', nextEmbedded: '@pop' }]
+		],
+		export: [[/^\s*$/, { token: 'delimiter.bracket', next: '@pop', nextEmbedded: '@pop' }]],
+		jsx: [
+			[/\s+/, ''],
+			[/(\w+)(=)("(?:[^"\\]|\\.)*")/, ['attribute.name', 'operator', 'string']],
+			[/(\w+)(=)('(?:[^'\\]|\\.)*')/, ['attribute.name', 'operator', 'string']],
+			[/(\w+(?=\s|>|={|$))/, ['attribute.name']],
+			[/={/, { token: 'delimiter.bracket', next: '@expression', nextEmbedded: 'js' }],
+			[/>/, { token: 'type.identifier', next: '@pop' }]
+		],
+		header: [
+			[/.$/, { token: 'keyword', next: '@pop' }],
+			{ include: 'content' },
+			[/./, { token: 'keyword' }]
+		],
+		strong: [
+			[/\*\*/, { token: 'strong', next: '@pop' }],
+			{ include: 'content' },
+			[/./, { token: 'strong' }]
+		],
+		emphasis_underscore: [
+			[/_/, { token: 'emphasis', next: '@pop' }],
+			{ include: 'content' },
+			[/./, { token: 'emphasis' }]
+		],
+		emphasis_asterisk: [
+			[/\*(?!\*)/, { token: 'emphasis', next: '@pop' }],
+			{ include: 'content' },
+			[/./, { token: 'emphasis' }]
+		],
+		frontmatter: [[/^---$/, { token: 'meta.content', nextEmbedded: '@pop', next: '@pop' }]],
+		codeblock_highlight_backtick: [
+			[/\s*`{3,}\s*$/, { token: 'string', next: '@pop', nextEmbedded: '@pop' }],
+			[/.*$/, 'variable.source']
+		],
+		codeblock_highlight_tilde: [
+			[/\s*~{3,}\s*$/, { token: 'string', next: '@pop', nextEmbedded: '@pop' }],
+			[/.*$/, 'variable.source']
+		],
+		codeblock_backtick: [
+			[/\s*`{3,}\s*$/, { token: 'string', next: '@pop' }],
+			[/.*$/, 'variable.source']
+		],
+		codeblock_tilde: [
+			[/\s*~{3,}\s*$/, { token: 'string', next: '@pop' }],
+			[/.*$/, 'variable.source']
+		]
+	}
+};
--- a/src/basic-languages/monaco.contribution.ts
+++ b/src/basic-languages/monaco.contribution.ts
@ -39,6 +39,7 @@ import './lua/lua.contribution';
 import './liquid/liquid.contribution';
 import './m3/m3.contribution';
 import './markdown/markdown.contribution';
+import './mdx/mdx.contribution';
 import './mips/mips.contribution';
 import './msdax/msdax.contribution';
 import './mysql/mysql.contribution';
--- a/website/index/samples/sample.mdx.txt
+++ b/website/index/samples/sample.mdx.txt
@ -0,0 +1,92 @@
+---
+frontmatter: data
+yaml: true
+---
+
+[link](https://example.com)
+
+~~~
+aasd
+asd
+asd
+~~~
+
+# Hello MDX {1+2}
+
+import { MyComponent } from './MyComponent'
+
+This is **bold {'foo' + 1} text**
+
+This is _emphasis {'foo' + 1} text_
+
+This is *emphasis {'foo' + 1} text too*
+
+    This is an indented *code* block
+
+export function foo() {
+  console.log('asd', 1)
+  if(true) {
+    return 'yep'
+  }
+  return 'nope'
+}
+
+
+This is regular content
+
+- this is a list
+
+* this is also a list
+
+ me too!
+
+1. pizza
+2. fries
+3. ice cream
+
+----
+
+_________
+
+***\
+~~~css
+body {
+  color: red;
+}
+~~~
+
+> - this is a list
+>
+> * this is also a list
+>
+> + me too!
+>
+> 1. pizza
+> 2. fries
+> 3. ice cream
+>
+> ---
+>
+> _________
+>
+> ***
+>
+> ```css
+> body {
+>   color: red;
+> }
+> ```
+
+> This is a blockquote
+>
+>> This is a nested {'blockquote'}
+
+{'foo' + 1 + 2 + {} + 12}
+
+{/* this is a comment */}
+
+<MyComponent contenteditable className="text-green-700" id='foo' width={100 + 100}>
+
+  This is **also** markdown.
+
+</MyComponent>
--- a/website/src/website/data/home-samples/sample.mdx.txt
+++ b/website/src/website/data/home-samples/sample.mdx.txt
@ -0,0 +1,91 @@
+---
+title: Hello!
+---
+
+import {Chart} from './chart.js'
+import population from './population.js'
+import {External} from './some/place.js'
+
+export const year = 2018
+export const pi = 3.14
+
+export function SomeComponent(props) {
+  const name = (props || {}).name || 'world'
+
+  return <div>
+    <p>Hi, {name}!</p>
+
+    <p>and some more things</p>
+  </div>
+}
+
+export function Local(props) {
+  return <span style={{color: 'red'}} {...props} />
+}
+
+# Last year’s snowfall
+
+In {year}, the snowfall was above average.
+It was followed by a warm spring which caused
+flood conditions in many of the nearby rivers.
+
+<Chart year={year} color="#fcb32c" />
+
+<div className="note">
+  > Some notable things in a block quote!
+</div>
+
+# Heading (rank 1)
+## Heading 2
+### 3
+#### 4
+##### 5
+###### 6
+
+> Block quote
+
+* Unordered
+* List
+
+1. Ordered
+2. List
+
+A paragraph, introducing a thematic break:
+
+---
+
+```js
+// Get an element.
+const element = document.querySelectorAll('#hi')
+
+// Add a class.
+element.classList.add('asd')
+```
+
+a [link](https://example.com), an ![image](./image.png), some *emphasis*,
+something **strong**, and finally a little `code()`.
+
+<Component
+  open
+  x={1}
+  label={'this is a string, *not* markdown!'}
+  icon={<Icon />}
+/>
+
+Two 🍰 is: {Math.PI * 2}
+
+{(function () {
+  const guess = Math.random()
+
+  if (guess > 0.66) {
+    return <span style={{color: 'tomato'}}>Look at us.</span>
+  }
+
+  if (guess > 0.33) {
+    return <span style={{color: 'violet'}}>Who would have guessed?!</span>
+  }
+
+  return <span style={{color: 'goldenrod'}}>Not me.</span>
+})()}
+
+{/* A comment! */}