microsoft
/
ghcrawler
зеркало из https://github.com/microsoft/ghcrawler.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Add rootRequest in metadata (#172)

This commit is contained in:
Gene Hazan 2019-09-30 13:20:25 -07:00 коммит произвёл GitHub
Родитель 5a8de626b0
Коммит 0ac110780f
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
3 изменённых файлов: 98 добавлений и 33 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

6
lib/crawler.js
Просмотреть файл

@ -377,6 +377,9 @@ class Crawler {
throw new Error('Delete without metadata');
}
request.document._metadata = request.response._metadataTemplate;
if (request.context.history) {
request.document._metadata.rootRequest = request.context.history[0];
}
return Q(request);
}
@ -384,7 +387,8 @@ class Crawler {
type: request.type,
url: request.url,
fetchedAt: moment.utc().toISOString(),
links: {}
links: {},
rootRequest: request.context.history ? request.context.history[0] : undefined,
};
if (request.response) {
if (request.response.headers) {

123
package-lock.json сгенерированный
Просмотреть файл

@ -51,7 +51,8 @@
"extend": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/extend/-/extend-3.0.1.tgz",
"integrity": "sha1-p1Xqe8Gt/MWjHOfnYtuq3F5jZEQ="
"integrity": "sha1-p1Xqe8Gt/MWjHOfnYtuq3F5jZEQ=",
"optional": true
},
"form-data": {
"version": "2.3.2",
@ -311,6 +312,7 @@
"version": "5.5.2",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-5.5.2.tgz",
"integrity": "sha1-c7Xuyj+rZT49P5Qis0GtQiBdyWU=",
"optional": true,
"requires": {
"co": "^4.6.0",
"fast-deep-equal": "^1.0.0",
@ -431,7 +433,8 @@
"arrify": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz",
"integrity": "sha1-iYUI2iIm84DfkEcoRWhJwVAaSw0="
"integrity": "sha1-iYUI2iIm84DfkEcoRWhJwVAaSw0=",
"optional": true
},
"asn1": {
"version": "0.2.3",
@ -448,6 +451,7 @@
"version": "2.6.0",
"resolved": "https://registry.npmjs.org/async/-/async-2.6.0.tgz",
"integrity": "sha512-xAfGg1/NTLBBKlHFmnd7PlmUW9KhVQIUuSrYem9xzFUZy13ScvtyGGejaae9iAVRiRq9+Cx7DPFaAAhCpyxyPw==",
"optional": true,
"requires": {
"lodash": "^4.14.0"
}
@ -460,12 +464,14 @@
"aws4": {
"version": "1.6.0",
"resolved": "https://registry.npmjs.org/aws4/-/aws4-1.6.0.tgz",
"integrity": "sha1-g+9cqGCysy5KDe7e6MdxudtXRx4="
"integrity": "sha1-g+9cqGCysy5KDe7e6MdxudtXRx4=",
"optional": true
},
"axios": {
"version": "0.18.0",
"resolved": "https://registry.npmjs.org/axios/-/axios-0.18.0.tgz",
"integrity": "sha1-MtU+SFHv3AoRmTts0AB4nXDAUQI=",
"optional": true,
"requires": {
"follow-redirects": "^1.3.0",
"is-buffer": "^1.1.5"
@ -685,12 +691,14 @@
"buffer-equal-constant-time": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz",
"integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk="
"integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk=",
"optional": true
},
"buffer-from": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.0.tgz",
"integrity": "sha512-c5mRlguI/Pe2dSZmpER62rSCu0ryKmWddzRYsuXc50U2/g8jMOulc31VZMa4mYx31U5xsmSOpDCgH88Vl9cDGQ=="
"integrity": "sha512-c5mRlguI/Pe2dSZmpER62rSCu0ryKmWddzRYsuXc50U2/g8jMOulc31VZMa4mYx31U5xsmSOpDCgH88Vl9cDGQ==",
"optional": true
},
"buffer-more-ints": {
"version": "0.0.2",
@ -710,7 +718,8 @@
"capture-stack-trace": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/capture-stack-trace/-/capture-stack-trace-1.0.0.tgz",
"integrity": "sha1-Sm+gc5nCa7pH8LJJa00PtAjFVQ0="
"integrity": "sha1-Sm+gc5nCa7pH8LJJa00PtAjFVQ0=",
"optional": true
},
"chai": {
"version": "4.2.0",
@ -743,7 +752,8 @@
"co": {
"version": "4.6.0",
"resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz",
"integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ="
"integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ=",
"optional": true
},
"code-point-at": {
"version": "1.1.0",
@ -806,6 +816,7 @@
"version": "1.6.2",
"resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz",
"integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==",
"optional": true,
"requires": {
"buffer-from": "^1.0.0",
"inherits": "^2.0.3",
@ -865,6 +876,7 @@
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/create-error-class/-/create-error-class-3.0.2.tgz",
"integrity": "sha1-Br56vvlHo/FKMP1hBnHUAbyot7Y=",
"optional": true,
"requires": {
"capture-stack-trace": "^1.0.0"
}
@ -1002,6 +1014,7 @@
"version": "3.6.0",
"resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.6.0.tgz",
"integrity": "sha512-fO3Di4tBKJpYTFHAxTU00BcfWMY9w24r/x21a6rZRbsD/ToUgGxsMbiGRmB7uVAXeGKXD9MwiLZa5E97EVgIRQ==",
"optional": true,
"requires": {
"end-of-stream": "^1.0.0",
"inherits": "^2.0.1",
@ -1022,6 +1035,7 @@
"version": "1.0.10",
"resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.10.tgz",
"integrity": "sha1-HFlQAPBKiJffuFAAiSoPTDOvhsM=",
"optional": true,
"requires": {
"safe-buffer": "^5.0.1"
}
@ -1327,7 +1341,8 @@
"fast-deep-equal": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-1.1.0.tgz",
"integrity": "sha1-wFNHeBfIa1HaqFPIHgWbcz0CNhQ="
"integrity": "sha1-wFNHeBfIa1HaqFPIHgWbcz0CNhQ=",
"optional": true
},
"fast-json-patch": {
"version": "1.1.3",
@ -1399,6 +1414,7 @@
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.0.tgz",
"integrity": "sha512-fdrt472/9qQ6Kgjvb935ig6vJCuofpBUD14f9Vb+SLlm7xIe4Qva5gey8EKtv8lp7ahE1wilg3xL1znpVGtZIA==",
"optional": true,
"requires": {
"debug": "^3.1.0"
},
@ -1407,6 +1423,7 @@
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
"integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
"optional": true,
"requires": {
"ms": "2.0.0"
}
@ -1453,6 +1470,7 @@
"version": "0.6.3",
"resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.6.3.tgz",
"integrity": "sha512-MSmczZctbz91AxCvqp9GHBoZOSbJKAICV7Ow/AIWSJZRrRchUd5NL1b2P4OfP+4m490BEUPhhARfpHdqCxuCvg==",
"optional": true,
"requires": {
"axios": "^0.18.0",
"extend": "^3.0.1",
@ -1462,7 +1480,8 @@
"extend": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/extend/-/extend-3.0.1.tgz",
"integrity": "sha1-p1Xqe8Gt/MWjHOfnYtuq3F5jZEQ="
"integrity": "sha1-p1Xqe8Gt/MWjHOfnYtuq3F5jZEQ=",
"optional": true
}
}
},
@ -1662,6 +1681,7 @@
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-1.5.0.tgz",
"integrity": "sha512-xpibA/hkq4waBcpIkSJg4GiDAqcBWjJee3c47zj7xP3RQ0A9mc8MP3Vc9sc8SGRoDYA0OszZxTjW7SbcC4pJIA==",
"optional": true,
"requires": {
"axios": "^0.18.0",
"gcp-metadata": "^0.6.3",
@ -1676,6 +1696,7 @@
"version": "0.10.1",
"resolved": "https://registry.npmjs.org/google-auto-auth/-/google-auto-auth-0.10.1.tgz",
"integrity": "sha512-iIqSbY7Ypd32mnHGbYctp80vZzXoDlvI9gEfvtl3kmyy5HzOcrZCIGCBdSlIzRsg7nHpQiHE3Zl6Ycur6TSodQ==",
"optional": true,
"requires": {
"async": "^2.3.0",
"gcp-metadata": "^0.6.1",
@ -1686,27 +1707,32 @@
"assert-plus": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
"integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU="
"integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=",
"optional": true
},
"aws-sign2": {
"version": "0.7.0",
"resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz",
"integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg="
"integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=",
"optional": true
},
"caseless": {
"version": "0.12.0",
"resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz",
"integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw="
"integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=",
"optional": true
},
"extend": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/extend/-/extend-3.0.1.tgz",
"integrity": "sha1-p1Xqe8Gt/MWjHOfnYtuq3F5jZEQ="
"integrity": "sha1-p1Xqe8Gt/MWjHOfnYtuq3F5jZEQ=",
"optional": true
},
"form-data": {
"version": "2.3.2",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.2.tgz",
"integrity": "sha1-SXBJi+YEwgwAXU9cI67NIda0kJk=",
"optional": true,
"requires": {
"asynckit": "^0.4.0",
"combined-stream": "1.0.6",
@ -1717,6 +1743,7 @@
"version": "5.0.3",
"resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.0.3.tgz",
"integrity": "sha1-ukAsJmGU8VlW7xXg/PJCmT9qff0=",
"optional": true,
"requires": {
"ajv": "^5.1.0",
"har-schema": "^2.0.0"
@ -1726,6 +1753,7 @@
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz",
"integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=",
"optional": true,
"requires": {
"assert-plus": "^1.0.0",
"jsprim": "^1.2.2",
@ -1735,12 +1763,14 @@
"qs": {
"version": "6.5.2",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz",
"integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA=="
"integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==",
"optional": true
},
"request": {
"version": "2.87.0",
"resolved": "https://registry.npmjs.org/request/-/request-2.87.0.tgz",
"integrity": "sha512-fcogkm7Az5bsS6Sl0sibkbhcKsnyon/jV1kF3ajGmF0c8HrttdKTPRT9hieOaQHA5HEq6r8OyWOo/o781C1tNw==",
"optional": true,
"requires": {
"aws-sign2": "~0.7.0",
"aws4": "^1.6.0",
@ -1768,6 +1798,7 @@
"version": "0.6.0",
"resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
"integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=",
"optional": true,
"requires": {
"safe-buffer": "^5.0.1"
}
@ -1775,7 +1806,8 @@
"uuid": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.2.1.tgz",
"integrity": "sha512-jZnMwlb9Iku/O3smGWvZhauCf6cvvpKi4BKRiliS3cxnI+Gz9j5MEpTz2UFuXiKPJocb7gnsLHwiS05ige5BEA=="
"integrity": "sha512-jZnMwlb9Iku/O3smGWvZhauCf6cvvpKi4BKRiliS3cxnI+Gz9j5MEpTz2UFuXiKPJocb7gnsLHwiS05ige5BEA==",
"optional": true
}
}
},
@ -1783,6 +1815,7 @@
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-1.0.2.tgz",
"integrity": "sha512-+EuKr4CLlGsnXx4XIJIVkcKYrsa2xkAmCvxRhX2HsazJzUBAJ35wARGeApHUn4nNfPD03Vl057FskNr20VaCyg==",
"optional": true,
"requires": {
"node-forge": "^0.7.4",
"pify": "^3.0.0"
@ -1791,14 +1824,16 @@
"pify": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz",
"integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY="
"integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=",
"optional": true
}
}
},
"graceful-fs": {
"version": "4.1.11",
"resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.11.tgz",
"integrity": "sha1-Dovf5NHduIVNZOBOp8AOKgJuVlg="
"integrity": "sha1-Dovf5NHduIVNZOBOp8AOKgJuVlg=",
"optional": true
},
"growl": {
"version": "1.10.5",
@ -1810,6 +1845,7 @@
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/gtoken/-/gtoken-2.3.0.tgz",
"integrity": "sha512-Jc9/8mV630cZE9FC5tIlJCZNdUjwunvlwOtCz6IDlaiB4Sz68ki29a1+q97sWTnTYroiuF9B135rod9zrQdHLw==",
"optional": true,
"requires": {
"axios": "^0.18.0",
"google-p12-pem": "^1.0.0",
@ -1821,12 +1857,14 @@
"mime": {
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/mime/-/mime-2.3.1.tgz",
"integrity": "sha512-OEUllcVoydBHGN1z84yfQDimn58pZNNNXgZlHXSboxMlFvgI6MXSWpWKpFRra7H1HxpVhHTkrghfRW49k6yjeg=="
"integrity": "sha512-OEUllcVoydBHGN1z84yfQDimn58pZNNNXgZlHXSboxMlFvgI6MXSWpWKpFRra7H1HxpVhHTkrghfRW49k6yjeg==",
"optional": true
},
"pify": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz",
"integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY="
"integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=",
"optional": true
}
}
},
@ -1980,12 +2018,14 @@
"is": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/is/-/is-3.2.1.tgz",
"integrity": "sha1-0Kwq1V63sL7JJqUmb2xmKqqD3KU="
"integrity": "sha1-0Kwq1V63sL7JJqUmb2xmKqqD3KU=",
"optional": true
},
"is-buffer": {
"version": "1.1.6",
"resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
"integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w=="
"integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==",
"optional": true
},
"is-callable": {
"version": "1.1.4",
@ -2174,7 +2214,8 @@
"json-schema-traverse": {
"version": "0.3.1",
"resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz",
"integrity": "sha1-NJptRMU6Ud6JtAgFxdXlm0F9M0A="
"integrity": "sha1-NJptRMU6Ud6JtAgFxdXlm0F9M0A=",
"optional": true
},
"json-stringify-safe": {
"version": "5.0.1",
@ -2208,6 +2249,7 @@
"version": "1.1.6",
"resolved": "https://registry.npmjs.org/jwa/-/jwa-1.1.6.tgz",
"integrity": "sha512-tBO/cf++BUsJkYql/kBbJroKOgHWEigTKBAjjBEmrMGYd1QMBC74Hr4Wo2zCZw6ZrVhlJPvoMrkcOnlWR/DJfw==",
"optional": true,
"requires": {
"buffer-equal-constant-time": "1.0.1",
"ecdsa-sig-formatter": "1.0.10",
@ -2218,6 +2260,7 @@
"version": "3.1.5",
"resolved": "https://registry.npmjs.org/jws/-/jws-3.1.5.tgz",
"integrity": "sha512-GsCSexFADNQUr8T5HPJvayTjvPIfoyJPtLQBwn5a4WZQchcrPMPMAWcC1AzJVRDKyD6ZPROPAxgv6rfHViO4uQ==",
"optional": true,
"requires": {
"jwa": "^1.1.5",
"safe-buffer": "^5.0.1"
@ -2259,7 +2302,8 @@
"lodash.isstring": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz",
"integrity": "sha1-1SfftUVuynzJu5XV2ur4i6VKVFE="
"integrity": "sha1-1SfftUVuynzJu5XV2ur4i6VKVFE=",
"optional": true
},
"log-driver": {
"version": "1.2.7",
@ -2323,6 +2367,7 @@
"version": "4.1.3",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.3.tgz",
"integrity": "sha512-fFEhvcgzuIoJVUF8fYr5KR0YqxD238zgObTps31YdADwPPAp82a4M8TrckkWyx7ekNlf9aBcVn81cFwwXngrJA==",
"optional": true,
"requires": {
"pseudomap": "^1.0.2",
"yallist": "^2.1.2"
@ -2791,7 +2836,8 @@
"node-forge": {
"version": "0.7.5",
"resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.7.5.tgz",
"integrity": "sha512-MmbQJ2MTESTjt3Gi/3yG1wGpIMhUfcIypUCGtTizFR9IiccFwxSpfp0vtIZlkFclEqERemxfnSdZEMR9VqqEFQ=="
"integrity": "sha512-MmbQJ2MTESTjt3Gi/3yG1wGpIMhUfcIypUCGtTizFR9IiccFwxSpfp0vtIZlkFclEqERemxfnSdZEMR9VqqEFQ==",
"optional": true
},
"node-int64": {
"version": "0.4.0",
@ -2830,7 +2876,8 @@
"oauth-sign": {
"version": "0.8.2",
"resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.2.tgz",
"integrity": "sha1-Rqarfwrq2N6unsBWV4C31O/rnUM="
"integrity": "sha1-Rqarfwrq2N6unsBWV4C31O/rnUM=",
"optional": true
},
"object-keys": {
"version": "1.1.1",
@ -3044,7 +3091,8 @@
"pseudomap": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz",
"integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM="
"integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=",
"optional": true
},
"psl": {
"version": "1.1.29",
@ -3055,6 +3103,7 @@
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz",
"integrity": "sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==",
"optional": true,
"requires": {
"end-of-stream": "^1.1.0",
"once": "^1.3.1"
@ -3064,6 +3113,7 @@
"version": "1.5.1",
"resolved": "https://registry.npmjs.org/pumpify/-/pumpify-1.5.1.tgz",
"integrity": "sha512-oClZI37HvuUJJxSKKrC17bZ9Cu0ZYhEAGPsPUy9KlMUmv9dKX2o77RUmq7f3XjIxbwyGwYzbzQ1L2Ks8sIradQ==",
"optional": true,
"requires": {
"duplexify": "^3.6.0",
"inherits": "^2.0.3",
@ -3397,7 +3447,8 @@
"retry-axios": {
"version": "0.3.2",
"resolved": "https://registry.npmjs.org/retry-axios/-/retry-axios-0.3.2.tgz",
"integrity": "sha512-jp4YlI0qyDFfXiXGhkCOliBN1G7fRH03Nqy8YdShzGqbY5/9S2x/IR6C88ls2DFkbWuL3ASkP7QD3pVrNpPgwQ=="
"integrity": "sha512-jp4YlI0qyDFfXiXGhkCOliBN1G7fRH03Nqy8YdShzGqbY5/9S2x/IR6C88ls2DFkbWuL3ASkP7QD3pVrNpPgwQ==",
"optional": true
},
"retry-request": {
"version": "3.3.1",
@ -3704,6 +3755,7 @@
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/stream-events/-/stream-events-1.0.4.tgz",
"integrity": "sha512-D243NJaYs/xBN2QnoiMDY7IesJFIK7gEhnvAYqJa5JvDdnh2dC4qDBwlCf0ohPpX2QRlA/4gnbnPd3rs3KxVcA==",
"optional": true,
"requires": {
"stubs": "^3.0.0"
}
@ -3711,7 +3763,8 @@
"stream-shift": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.0.tgz",
"integrity": "sha1-1cdSgl5TZ+eG944Y5EXqIjoVWVI="
"integrity": "sha1-1cdSgl5TZ+eG944Y5EXqIjoVWVI=",
"optional": true
},
"string-format-obj": {
"version": "1.1.1",
@ -3772,7 +3825,8 @@
"stubs": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/stubs/-/stubs-3.0.0.tgz",
"integrity": "sha1-6NK6H6nJBXAwPAMLaQD31fiavls="
"integrity": "sha1-6NK6H6nJBXAwPAMLaQD31fiavls=",
"optional": true
},
"text-encoding": {
"version": "0.6.4",
@ -3789,6 +3843,7 @@
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/through2/-/through2-2.0.3.tgz",
"integrity": "sha1-AARWmzfHx0ujnEPzzteNGtlBQL4=",
"optional": true,
"requires": {
"readable-stream": "^2.1.5",
"xtend": "~4.0.1"
@ -3798,6 +3853,7 @@
"version": "2.3.4",
"resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.4.tgz",
"integrity": "sha512-TZ6TTfI5NtZnuyy/Kecv+CnoROnyXn2DN97LontgQpCwsX2XyLYCC0ENhYkehSOwAp8rTQKc/NUIF7BkQ5rKLA==",
"optional": true,
"requires": {
"punycode": "^1.4.1"
}
@ -3840,7 +3896,8 @@
"typedarray": {
"version": "0.0.6",
"resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz",
"integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c="
"integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=",
"optional": true
},
"underscore": {
"version": "1.8.3",
@ -4064,7 +4121,8 @@
"xdg-basedir": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/xdg-basedir/-/xdg-basedir-3.0.0.tgz",
"integrity": "sha1-SWsswQnsqNus/i3HK2A8F8WHCtQ="
"integrity": "sha1-SWsswQnsqNus/i3HK2A8F8WHCtQ=",
"optional": true
},
"xml2js": {
"version": "0.2.7",
@ -4093,7 +4151,8 @@
"yallist": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz",
"integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI="
"integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=",
"optional": true
},
"yargs-parser": {
"version": "13.0.0",

2
test/unit/crawlerTests.js
Просмотреть файл

@ -545,6 +545,7 @@ describe('Crawler convert to document', () => {
expect(metadata.etag).to.be.equal(42);
expect(metadata.links).to.be.not.null;
expect(metadata.fetchedAt).to.be.not.null;
expect(metadata.rootRequest).to.be.undefined;
});
});
@ -564,6 +565,7 @@ describe('Crawler convert to document', () => {
expect(metadata.etag).to.be.equal(42);
expect(metadata.links).to.be.not.null;
expect(metadata.fetchedAt).to.be.not.null;
expect(metadata.rootRequest).to.be.undefined;
expect(request.document.elements === array).to.be.true;
});
});