Unity-Technologies
/
quicktype
зеркало из https://github.com/Unity-Technologies/quicktype.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Merge pull request #962 from quicktype/fix-schema-id 

Fix $id and $ref.  Fixes #921
This commit is contained in:
Mark Probst 2018-07-17 12:09:57 +02:00 коммит произвёл GitHub
Родитель a50022b64a 6a6801496f
Коммит 5f686d6358
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
18 изменённых файлов: 217 добавлений и 79 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

6
src/cli/index.ts
Просмотреть файл

@ -428,7 +428,7 @@ function makeOptionDefinitions(targetLanguages: TargetLanguage[]): OptionDefinit
type: String,
typeLabel: "OPTIONS or all",
description:
"Comma separated debug options: print-graph, print-reconstitution, print-gather-names, print-transformations, print-times, provenance"
"Comma separated debug options: print-graph, print-reconstitution, print-gather-names, print-transformations, print-schema-resolving, print-times, provenance"
},
{
name: "telemetry",
@ -765,6 +765,7 @@ export async function makeQuicktypeOptions(
let debugPrintReconstitution = debugAll;
let debugPrintGatherNames = debugAll;
let debugPrintTransformations = debugAll;
let debugPrintSchemaResolving = debugAll;
let debugPrintTimes = debugAll;
if (components !== undefined) {
for (let component of components) {
@ -779,6 +780,8 @@ export async function makeQuicktypeOptions(
debugPrintTransformations = true;
} else if (component === "print-times") {
debugPrintTimes = true;
} else if (component === "print-schema-resolving") {
debugPrintSchemaResolving = true;
} else if (component === "provenance") {
checkProvenance = true;
} else if (component !== "all") {
@ -814,6 +817,7 @@ export async function makeQuicktypeOptions(
debugPrintReconstitution,
debugPrintGatherNames,
debugPrintTransformations,
debugPrintSchemaResolving,
debugPrintTimes
};
for (const flagName of inferenceFlagNames) {

8
src/quicktype-core/Messages.ts
Просмотреть файл

@ -32,14 +32,13 @@ export type ErrorProperties =
kind: "SchemaSetOperationCasesIsNotArray";
properties: { operation: string; cases: any; ref: Ref };
}
| { kind: "SchemaCannotFetch"; properties: { address: string } }
| { kind: "SchemaMoreThanOneUnionMemberName"; properties: { names: string[] } }
| { kind: "SchemaCannotGetTypesFromBoolean"; properties: { ref: string } }
| { kind: "SchemaCannotIndexArrayWithNonNumber"; properties: { actual: string; ref: Ref } }
| { kind: "SchemaIndexNotInArray"; properties: { index: number; ref: Ref } }
| { kind: "SchemaKeyNotInObject"; properties: { key: string; ref: Ref } }
| { kind: "SchemaFetchError"; properties: { address: string; ref: Ref; error: any } }
| { kind: "SchemaFetchErrorTopLevel"; properties: { address: string; error: any } }
| { kind: "SchemaFetchError"; properties: { address: string; base: Ref } }
| { kind: "SchemaFetchErrorTopLevel"; properties: { address: string } }
// GraphQL input
| { kind: "GraphQLNoQueriesDefined"; properties: {} }
@ -104,7 +103,6 @@ const errorMessages: ErrorMessages = {
SchemaWrongAccessorEntryArrayLength:
"Accessor entry array must have the same number of entries as the ${operation} at ${ref}",
SchemaSetOperationCasesIsNotArray: "${operation} cases must be an array, but is ${cases}, at ${ref}",
SchemaCannotFetch: "Cannot fetch schema at address ${address}",
SchemaMoreThanOneUnionMemberName: "More than one name given for union member: ${names}",
SchemaCannotGetTypesFromBoolean:
"Schema value to get top-level types from must be an object, but is boolean, at ${ref}",
@ -112,7 +110,7 @@ const errorMessages: ErrorMessages = {
"Trying to index array in schema with key that is not a number, but is ${actual} at ${ref}",
SchemaIndexNotInArray: "Index ${index} out of range of schema array at ${ref}",
SchemaKeyNotInObject: "Key ${key} not in schema object at ${ref}",
SchemaFetchError: "Could not fetch schema ${address}, referred to from ${ref}: ${error}",
SchemaFetchError: "Could not fetch schema ${address}, referred to from ${base}: ${error}",
SchemaFetchErrorTopLevel: "Could not fetch top-level schema ${address}: ${error}",
// GraphQL input

11
src/quicktype-core/Run.ts
Просмотреть файл

@ -130,6 +130,8 @@ export type NonInferenceOptions = {
debugPrintTransformations: boolean;
/** Print the time it took for each pass to run */
debugPrintTimes: boolean;
/** Print schema resolving steps */
debugPrintSchemaResolving: boolean;
};
export type Options = NonInferenceOptions & InferenceFlags;
@ -150,13 +152,15 @@ const defaultOptions: NonInferenceOptions = {
debugPrintReconstitution: false,
debugPrintGatherNames: false,
debugPrintTransformations: false,
debugPrintTimes: false
debugPrintTimes: false,
debugPrintSchemaResolving: false
};
export interface RunContext {
stringTypeMapping: StringTypeMapping;
debugPrintReconstitution: boolean;
debugPrintTransformations: boolean;
debugPrintSchemaResolving: boolean;
timeSync<T>(name: string, f: () => Promise<T>): Promise<T>;
time<T>(name: string, f: () => T): T;
@ -202,6 +206,10 @@ class Run implements RunContext {
return this._options.debugPrintTransformations;
}
get debugPrintSchemaResolving(): boolean {
return this._options.debugPrintSchemaResolving;
}
async timeSync<T>(name: string, f: () => Promise<T>): Promise<T> {
const start = Date.now();
const result = await f();
@ -239,6 +247,7 @@ class Run implements RunContext {
"read input",
async () =>
await allInputs.addTypes(
this,
typeBuilder,
this._options.inferMaps,
this._options.inferEnums,

3
src/quicktype-core/index.ts
Просмотреть файл

@ -5,7 +5,8 @@ export {
quicktypeMultiFile,
quicktype,
inferenceFlags,
inferenceFlagNames
inferenceFlagNames,
RunContext
} from "./Run";
export { CompressedJSON } from "./input/CompressedJSON";
export {

17
src/quicktype-core/input/Inputs.ts
Просмотреть файл

@ -18,6 +18,7 @@ import { descriptionTypeAttributeKind, descriptionAttributeProducer } from "../D
import { TypeInference } from "./Inference";
import { TargetLanguage } from "../TargetLanguage";
import { accessorNamesAttributeProducer } from "../AccessorNames";
import { RunContext } from "../Run";
class InputJSONSchemaStore extends JSONSchemaStore {
constructor(private readonly _inputs: Map<string, StringInput>, private readonly _delegate?: JSONSchemaStore) {
@ -49,7 +50,13 @@ export interface Input<T> {
singleStringSchemaSource(): string | undefined;
addTypes(typeBuilder: TypeBuilder, inferMaps: boolean, inferEnums: boolean, fixedTopLevels: boolean): Promise<void>;
addTypes(
ctx: RunContext,
typeBuilder: TypeBuilder,
inferMaps: boolean,
inferEnums: boolean,
fixedTopLevels: boolean
): Promise<void>;
}
type JSONTopLevel = { samples: Value[]; description: string | undefined };
@ -116,6 +123,7 @@ export class JSONInput implements Input<JSONSourceData> {
}
async addTypes(
_ctx: RunContext,
typeBuilder: TypeBuilder,
inferMaps: boolean,
inferEnums: boolean,
@ -181,8 +189,8 @@ export class JSONSchemaInput implements Input<JSONSchemaSourceData> {
this._topLevels.set(name, ref);
}
async addTypes(typeBuilder: TypeBuilder): Promise<void> {
await addTypesInSchema(typeBuilder, defined(this._schemaStore), this._topLevels, this._attributeProducers);
async addTypes(ctx: RunContext, typeBuilder: TypeBuilder): Promise<void> {
await addTypesInSchema(ctx, typeBuilder, defined(this._schemaStore), this._topLevels, this._attributeProducers);
}
async addSource(schemaSource: JSONSchemaSourceData): Promise<void> {
@ -299,13 +307,14 @@ export class InputData {
}
async addTypes(
ctx: RunContext,
typeBuilder: TypeBuilder,
inferMaps: boolean,
inferEnums: boolean,
fixedTopLevels: boolean
): Promise<void> {
for (const input of this._inputs) {
await input.addTypes(typeBuilder, inferMaps, inferEnums, fixedTopLevels);
await input.addTypes(ctx, typeBuilder, inferMaps, inferEnums, fixedTopLevels);
}
}

185
src/quicktype-core/input/JSONSchemaInput.ts
Просмотреть файл

@ -15,7 +15,6 @@ import {
arrayMapSync,
arrayLast,
arrayGetFromEnd,
arrayPop,
hashCodeOf,
hasOwnProperty,
definedMap,
@ -38,6 +37,7 @@ import { messageAssert, messageError } from "../Messages";
import { StringTypes } from "../StringTypes";
import { TypeRef } from "../TypeGraph";
import { RunContext } from "../Run";
export enum PathElementKind {
Root,
@ -98,8 +98,8 @@ export function checkJSONSchema(x: any, refOrLoc: Ref | (() => Ref)): JSONSchema
const numberRegexp = new RegExp("^[0-9]+$");
export class Ref {
static root(address: string): Ref {
const uri = new URI(address);
static root(address: string | undefined): Ref {
const uri = definedMap(address, a => new URI(a));
return new Ref(uri, []);
}
@ -327,28 +327,31 @@ class Location {
public readonly canonicalRef: Ref;
public readonly virtualRef: Ref;
constructor(canonicalRef: Ref, virtualRef?: Ref) {
constructor(canonicalRef: Ref, virtualRef?: Ref, readonly haveID: boolean = false) {
this.canonicalRef = canonicalRef;
this.virtualRef = virtualRef !== undefined ? virtualRef : canonicalRef;
}
updateWithID(id: any) {
if (typeof id !== "string") return this;
// FIXME: This is incorrect. If the parsed ref doesn't have an address, the
// current virtual one's must be used. The canonizer must do this, too.
return new Location(this.canonicalRef, Ref.parse(id).resolveAgainst(this.virtualRef));
const parsed = Ref.parse(id);
const virtual = this.haveID ? parsed.resolveAgainst(this.virtualRef) : parsed;
if (!this.haveID) {
messageAssert(virtual.hasAddress, "SchemaIDMustHaveAddress", withRef(this, { id }));
}
return new Location(this.canonicalRef, virtual, true);
}
push(...keys: string[]): Location {
return new Location(this.canonicalRef.push(...keys), this.virtualRef.push(...keys));
return new Location(this.canonicalRef.push(...keys), this.virtualRef.push(...keys), this.haveID);
}
pushObject(): Location {
return new Location(this.canonicalRef.pushObject(), this.virtualRef.pushObject());
return new Location(this.canonicalRef.pushObject(), this.virtualRef.pushObject(), this.haveID);
}
pushType(index: number): Location {
return new Location(this.canonicalRef.pushType(index), this.virtualRef.pushType(index));
return new Location(this.canonicalRef.pushType(index), this.virtualRef.pushType(index), this.haveID);
}
toString(): string {
@ -357,14 +360,10 @@ class Location {
}
class Canonizer {
private readonly _map = new EqualityMap<Ref, Ref>();
private readonly _map = new EqualityMap<Ref, Location>();
private readonly _schemaAddressesAdded = new Set<string>();
private addID(mapped: string, loc: Location): void {
const ref = Ref.parse(mapped).resolveAgainst(loc.virtualRef);
messageAssert(ref.hasAddress, "SchemaIDMustHaveAddress", withRef(loc, { id: mapped }));
this._map.set(ref, loc.canonicalRef);
}
constructor(private readonly _ctx: RunContext) {}
private addIDs(schema: any, loc: Location) {
if (schema === null) return;
@ -377,43 +376,40 @@ class Canonizer {
if (typeof schema !== "object") {
return;
}
const locWithoutID = loc;
const maybeID = schema["$id"];
if (typeof maybeID === "string") {
this.addID(maybeID, loc);
loc = loc.updateWithID(maybeID);
}
if (loc.haveID) {
if (this._ctx.debugPrintSchemaResolving) {
console.log(`adding mapping ${loc.toString()}`);
}
this._map.set(loc.virtualRef, locWithoutID);
}
for (const property of Object.getOwnPropertyNames(schema)) {
this.addIDs(schema[property], loc.push(property));
}
}
addSchema(schema: any, address: string) {
if (this._schemaAddressesAdded.has(address)) return;
addSchema(schema: any, address: string): boolean {
if (this._schemaAddressesAdded.has(address)) return false;
this.addIDs(schema, new Location(Ref.root(address)));
this.addIDs(schema, new Location(Ref.root(address), Ref.root(undefined)));
this._schemaAddressesAdded.add(address);
return true;
}
// Returns: Canonical ref, full virtual ref
canonize(virtualBase: Ref | undefined, ref: Ref): [Ref, Ref] {
const fullVirtual = ref.resolveAgainst(virtualBase);
let virtual = fullVirtual;
const relative: PathElement[] = [];
for (;;) {
const maybeCanonical = this._map.get(virtual);
if (maybeCanonical !== undefined) {
return [new Ref(maybeCanonical.addressURI, maybeCanonical.path.concat(relative)), fullVirtual];
}
const last = arrayLast(virtual.path);
if (last === undefined) {
// We've exhausted our options - it's not a mapped ref.
return [fullVirtual, fullVirtual];
}
if (last.kind !== PathElementKind.Root) {
relative.unshift(last);
}
virtual = new Ref(virtual.addressURI, arrayPop(virtual.path));
// Returns: Canonical ref
canonize(base: Location, ref: Ref): Location {
const virtual = ref.resolveAgainst(base.virtualRef);
const loc = this._map.get(virtual);
if (loc !== undefined) {
return loc;
}
const canonicalRef =
virtual.addressURI === undefined ? new Ref(base.canonicalRef.addressURI, virtual.path) : virtual;
return new Location(canonicalRef, new Ref(undefined, virtual.path));
}
}
@ -454,18 +450,6 @@ function checkRequiredArray(arr: any, loc: Location): string[] {
return arr;
}
async function getFromStore(store: JSONSchemaStore, address: string, ref: Ref | undefined): Promise<JSONSchema> {
try {
return await store.get(address);
} catch (error) {
if (ref === undefined) {
return messageError("SchemaFetchErrorTopLevel", { address, error });
} else {
return messageError("SchemaFetchError", { address, ref, error });
}
}
}
export const schemaTypeDict = {
null: true,
boolean: true,
@ -496,20 +480,95 @@ function typeKindForJSONSchemaFormat(format: string): TransformedStringTypeKind
return target[0] as TransformedStringTypeKind;
}
function schemaFetchError(base: Location | undefined, address: string): never {
if (base === undefined) {
return messageError("SchemaFetchErrorTopLevel", { address });
} else {
return messageError("SchemaFetchError", { address, base: base.canonicalRef });
}
}
export async function addTypesInSchema(
ctx: RunContext,
typeBuilder: TypeBuilder,
store: JSONSchemaStore,
references: ReadonlyMap<string, Ref>,
attributeProducers: JSONSchemaAttributeProducer[]
): Promise<void> {
const canonizer = new Canonizer();
const canonizer = new Canonizer(ctx);
async function resolveVirtualRef(base: Location | undefined, virtualRef: Ref): Promise<[JSONSchema, Location]> {
const [canonical, fullVirtual] = canonizer.canonize(definedMap(base, b => b.virtualRef), virtualRef);
assert(canonical.hasAddress, "Canonical ref can't be resolved without an address");
const schema = await getFromStore(store, canonical.address, definedMap(base, l => l.canonicalRef));
canonizer.addSchema(schema, canonical.address);
return [canonical.lookupRef(schema), new Location(canonical, fullVirtual)];
async function tryResolveVirtualRef(
fetchBase: Location,
lookupBase: Location,
virtualRef: Ref
): Promise<[JSONSchema | undefined, Location]> {
let didAdd = false;
// If we are resolving into a schema file that we haven't seen yet then
// we don't know its $id mapping yet, which means we don't know where we
// will end up. What we do if we encounter a new schema is add all its
// IDs first, and then try to canonize again.
for (;;) {
const loc = canonizer.canonize(fetchBase, virtualRef);
const canonical = loc.canonicalRef;
assert(canonical.hasAddress, "Canonical ref can't be resolved without an address");
const address = canonical.address;
let schema =
canonical.addressURI === undefined
? undefined
: await store.get(address, ctx.debugPrintSchemaResolving);
if (schema === undefined) {
return [undefined, loc];
}
if (canonizer.addSchema(schema, address)) {
assert(!didAdd, "We can't add a schema twice");
didAdd = true;
} else {
let lookupLoc = canonizer.canonize(lookupBase, virtualRef);
if (fetchBase !== undefined) {
lookupLoc = new Location(
new Ref(loc.canonicalRef.addressURI, lookupLoc.canonicalRef.path),
lookupLoc.virtualRef,
lookupLoc.haveID
);
}
return [lookupLoc.canonicalRef.lookupRef(schema), lookupLoc];
}
}
}
async function resolveVirtualRef(base: Location, virtualRef: Ref): Promise<[JSONSchema, Location]> {
if (ctx.debugPrintSchemaResolving) {
console.log(`resolving ${virtualRef.toString()} relative to ${base.toString()}`);
}
// Try with the virtual base first. If that doesn't work, use the
// canonical ref's address with the virtual base's path.
let result = await tryResolveVirtualRef(base, base, virtualRef);
let schema = result[0];
if (schema !== undefined) {
if (ctx.debugPrintSchemaResolving) {
console.log(`resolved to ${result[1].toString()}`);
}
return [schema, result[1]];
}
const altBase = new Location(
base.canonicalRef,
new Ref(base.canonicalRef.addressURI, base.virtualRef.path),
base.haveID
);
result = await tryResolveVirtualRef(altBase, base, virtualRef);
schema = result[0];
if (schema !== undefined) {
if (ctx.debugPrintSchemaResolving) {
console.log(`resolved to ${result[1].toString()}`);
}
return [schema, result[1]];
}
return schemaFetchError(base, virtualRef.address);
}
let typeForCanonicalRef = new EqualityMap<Ref, TypeRef>();
@ -836,7 +895,10 @@ export async function addTypesInSchema(
}
for (const [topLevelName, topLevelRef] of references) {
const [target, loc] = await resolveVirtualRef(undefined, topLevelRef);
const [target, loc] = await resolveVirtualRef(
new Location(new Ref(topLevelRef.addressURI, [])),
new Ref(undefined, topLevelRef.path)
);
const t = await toType(target, loc, makeNamesTypeAttributes(topLevelName, false));
typeBuilder.addTopLevel(topLevelName, t);
}
@ -877,7 +939,10 @@ export async function refsInSchemaForURI(
propertiesAreTypes = false;
}
const rootSchema = await getFromStore(store, ref.address, undefined);
let rootSchema = await store.get(ref.address, false);
if (rootSchema === undefined) {
return schemaFetchError(undefined, ref.address);
}
const schema = ref.lookupRef(rootSchema);
if (propertiesAreTypes) {

18
src/quicktype-core/input/JSONSchemaStore.ts
Просмотреть файл

@ -1,5 +1,4 @@
import { StringMap, assert } from "../support/Support";
import { messageError } from "../Messages";
export type JSONSchema = StringMap | boolean;
@ -14,14 +13,25 @@ export abstract class JSONSchemaStore {
// FIXME: Remove the undefined option
abstract async fetch(_address: string): Promise<JSONSchema | undefined>;
async get(address: string): Promise<JSONSchema> {
async get(address: string, debugPrint: boolean): Promise<JSONSchema | undefined> {
let schema = this._schemas.get(address);
if (schema !== undefined) {
return schema;
}
schema = await this.fetch(address);
if (debugPrint) {
console.log(`trying to fetch ${address}`);
}
try {
schema = await this.fetch(address);
} catch {}
if (schema === undefined) {
return messageError("SchemaCannotFetch", { address });
if (debugPrint) {
console.log(`couldn't fetch ${address}`);
}
return undefined;
}
if (debugPrint) {
console.log(`successully fetched ${address}`);
}
this.add(address, schema);
return schema;

5
src/quicktype-graphql-input/index.ts
Просмотреть файл

@ -30,7 +30,8 @@ import {
emptyTypeAttributes,
StringTypes,
Input,
derefTypeRef
derefTypeRef,
RunContext
} from "../quicktype-core";
import { TypeKind, GraphQLSchema } from "./GraphQLSchema";
@ -499,7 +500,7 @@ export class GraphQLInput implements Input<GraphQLSourceData> {
return undefined;
}
async addTypes(typeBuilder: TypeBuilder): Promise<void> {
async addTypes(_ctx: RunContext, typeBuilder: TypeBuilder): Promise<void> {
for (const [name, { schema, query }] of this._topLevels) {
const newTopLevels = makeGraphQLQueryTypes(name, typeBuilder, schema, query);
for (const [actualName, t] of newTopLevels) {

5
test/inputs/schema/a/test2.json Normal file
Просмотреть файл

@ -0,0 +1,5 @@
{
"$schema": "http://json-schema.org/draft-07/schema",
"$id": "a/test2.json",
"$ref": "../b/test3.json#/foo"
}

14
test/inputs/schema/b/test3.json Normal file
Просмотреть файл

@ -0,0 +1,14 @@
{
"$schema": "http://json-schema.org/draft-07/schema",
"$id": "b/test3.json",
"definitions": {
"foo": {
"$id": "#/foo",
"type": "object",
"properties": {
"foo": { "type": "integer" }
},
"required": ["foo"]
}
}
}

1
test/inputs/schema/ref-id-files.1.json Normal file
Просмотреть файл

@ -0,0 +1 @@
{ "foo": 123 }

5
test/inputs/schema/ref-id-files.schema Normal file
Просмотреть файл

@ -0,0 +1,5 @@
{
"$schema": "http://json-schema.org/draft-07/schema",
"$id": "test.json",
"$ref": "a/test2.json#/"
}

1
test/inputs/schema/ref-remote.1.json Normal file
Просмотреть файл

@ -0,0 +1 @@
{}

1
test/inputs/schema/ref-remote.2.json Normal file
Просмотреть файл

@ -0,0 +1 @@
{"next": {}}

1
test/inputs/schema/ref-remote.3.json Normal file
Просмотреть файл

@ -0,0 +1 @@
{"next": {"next": {}}}

5
test/inputs/schema/ref-remote.schema Normal file
Просмотреть файл

@ -0,0 +1,5 @@
{
"$schema": "http://json-schema.org/draft-07/schema",
"$id": "ref-remote.schema",
"$ref": "https://raw.githubusercontent.com/quicktype/quicktype/master/test/inputs/schema/list.schema"
}

1
test/languages.ts
Просмотреть файл

@ -363,6 +363,7 @@ export const ElmLanguage: Language = {
skipSchema: [
"union-list.schema", // recursion
"list.schema", // recursion
"ref-remote.schema", // recursion
"mutually-recursive.schema", // recursion
"postman-collection.schema", // recursion
"vega-lite.schema", // recursion

9
test/utils.ts
Просмотреть файл

@ -108,7 +108,14 @@ export async function quicktypeForLanguage(
debug: graphqlSchema === undefined ? "provenance" : undefined
});
} catch (e) {
failWith("quicktype threw an exception", { error: e });
failWith("quicktype threw an exception", {
error: e,
languageName: language.name,
sourceFile,
sourceLanguage,
graphqlSchema,
additionalRendererOptions
});
}
}