зеркало из https://github.com/nextcloud/news.git
Merge pull request #282 from SMillerDev/switch_feedIO
Switch to feed-io for parsing
This commit is contained in:
John Molakvoæ
GitHub
Коммит
249352c9d3
4
Makefile
4
Makefile
|
|
@ -41,10 +41,10 @@
|
|||
|
||||
app_name:=$(notdir $(CURDIR))
|
||||
build_tools_directory:=$(CURDIR)/build/tools
|
||||
source_build_directory:=$(CURDIR)/build/source/news
|
||||
source_build_directory:=$(CURDIR)/build/source/$(app_name)
|
||||
source_artifact_directory:=$(CURDIR)/build/artifacts/source
|
||||
source_package_name:=$(source_artifact_directory)/$(app_name)
|
||||
appstore_build_directory:=$(CURDIR)/build/appstore/news
|
||||
appstore_build_directory:=$(CURDIR)/build/appstore/$(app_name)
|
||||
appstore_artifact_directory:=$(CURDIR)/build/artifacts/appstore
|
||||
appstore_package_name:=$(appstore_artifact_directory)/$(app_name)
|
||||
npm:=$(shell which npm 2> /dev/null)
|
||||
|
|
|
|||
|
|
@ -10,6 +10,12 @@
|
|||
"homepage": "https://bernhard-posselt.com",
|
||||
"role": "Developer"
|
||||
},
|
||||
{
|
||||
"name": "Sean Molenaar",
|
||||
"email": "sean@seanmolenaar.eu",
|
||||
"homepage": "https://seanmolenaar.eu",
|
||||
"role": "Developer"
|
||||
},
|
||||
{
|
||||
"name": "Alessandro Cosentino",
|
||||
"homepage": "http://algorithmsforthekitchen.com/",
|
||||
|
|
@ -33,11 +39,16 @@
|
|||
"ezyang/htmlpurifier": "4.10.0",
|
||||
"pear/net_url2": "2.2.2",
|
||||
"riimu/kit-pathjoin": "1.2.0",
|
||||
"nicolus/picofeed": "0.1.35"
|
||||
"debril/feed-io": "^3.0",
|
||||
"arthurhoaro/favicon": "^1.2"
|
||||
},
|
||||
"require-dev": {
|
||||
"phpunit/phpunit": "^6.5",
|
||||
"squizlabs/php_codesniffer": "^3.3"
|
||||
"squizlabs/php_codesniffer": "^3.3",
|
||||
"guzzlehttp/guzzle": "~6.3"
|
||||
},
|
||||
"replace": {
|
||||
"guzzlehttp/guzzle": "*"
|
||||
},
|
||||
"autoload": {
|
||||
"psr-4": {
|
||||
|
|
|
|||
|
|
@ -4,8 +4,118 @@
|
|||
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
|
||||
"This file is @generated automatically"
|
||||
],
|
||||
"content-hash": "f4a0d96b7e83ec4d9d232412b9e61566",
|
||||
"content-hash": "1630b553e70e8245b11922394d4d9f59",
|
||||
"packages": [
|
||||
{
|
||||
"name": "arthurhoaro/favicon",
|
||||
"version": "v1.2.2",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/ArthurHoaro/favicon.git",
|
||||
"reference": "50fd2a0f984db13948a69ab120451e03e41979fa"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/ArthurHoaro/favicon/zipball/50fd2a0f984db13948a69ab120451e03e41979fa",
|
||||
"reference": "50fd2a0f984db13948a69ab120451e03e41979fa",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"php": ">=5.3.3"
|
||||
},
|
||||
"require-dev": {
|
||||
"phpunit/phpunit": "~4.8",
|
||||
"weew/helpers-filesystem": "~1.0"
|
||||
},
|
||||
"type": "library",
|
||||
"autoload": {
|
||||
"psr-4": {
|
||||
"Favicon\\": "src/Favicon/"
|
||||
}
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"license": [
|
||||
"Apache-2.0"
|
||||
],
|
||||
"authors": [
|
||||
{
|
||||
"name": "Chris Shiflett",
|
||||
"homepage": "http://shiflett.org/"
|
||||
},
|
||||
{
|
||||
"name": "Arthur Hoaro",
|
||||
"homepage": "http://hoa.ro"
|
||||
}
|
||||
],
|
||||
"description": "PHP Library used to discover favicon from given URL",
|
||||
"homepage": "https://github.com/ArthurHoaro/favicon",
|
||||
"keywords": [
|
||||
"favicon",
|
||||
"finder",
|
||||
"icon"
|
||||
],
|
||||
"time": "2018-09-08T09:37:54+00:00"
|
||||
},
|
||||
{
|
||||
"name": "debril/feed-io",
|
||||
"version": "v3.1.1",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/alexdebril/feed-io.git",
|
||||
"reference": "a79a09a915540b5475b12c82effb3dd43c2b2a0b"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/alexdebril/feed-io/zipball/a79a09a915540b5475b12c82effb3dd43c2b2a0b",
|
||||
"reference": "a79a09a915540b5475b12c82effb3dd43c2b2a0b",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"guzzlehttp/guzzle": "~6.2",
|
||||
"php": ">=5.6.0",
|
||||
"psr/log": "~1.0"
|
||||
},
|
||||
"require-dev": {
|
||||
"friendsofphp/php-cs-fixer": "^2.4",
|
||||
"monolog/monolog": "1.*",
|
||||
"phpunit/phpunit": "~5.6.0"
|
||||
},
|
||||
"suggest": {
|
||||
"monolog/monolog": "Allows to handle logs",
|
||||
"symfony/console": "Allows to use the command line interface"
|
||||
},
|
||||
"bin": [
|
||||
"bin/feedio"
|
||||
],
|
||||
"type": "library",
|
||||
"autoload": {
|
||||
"psr-4": {
|
||||
"FeedIo\\": "src/FeedIo"
|
||||
}
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"license": [
|
||||
"MIT"
|
||||
],
|
||||
"authors": [
|
||||
{
|
||||
"name": "Alexandre Debril",
|
||||
"email": "alex.debril@gmail.com"
|
||||
}
|
||||
],
|
||||
"description": "PHP library built to consume and serve JSONFeed / RSS / Atom feeds",
|
||||
"homepage": "https://feed-io.net",
|
||||
"keywords": [
|
||||
"atom",
|
||||
"cli",
|
||||
"client",
|
||||
"feed",
|
||||
"jsonfeed",
|
||||
"news",
|
||||
"rss"
|
||||
],
|
||||
"time": "2018-06-18T12:31:47+00:00"
|
||||
},
|
||||
{
|
||||
"name": "ezyang/htmlpurifier",
|
||||
"version": "v4.10.0",
|
||||
|
|
@ -53,59 +163,6 @@
|
|||
],
|
||||
"time": "2018-02-23T01:58:20+00:00"
|
||||
},
|
||||
{
|
||||
"name": "nicolus/picofeed",
|
||||
"version": "v0.1.35",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/nicolus/picoFeed.git",
|
||||
"reference": "3a27b47de31eedec075c719f961783c5db7a7b08"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/nicolus/picoFeed/zipball/3a27b47de31eedec075c719f961783c5db7a7b08",
|
||||
"reference": "3a27b47de31eedec075c719f961783c5db7a7b08",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"ext-dom": "*",
|
||||
"ext-iconv": "*",
|
||||
"ext-libxml": "*",
|
||||
"ext-simplexml": "*",
|
||||
"ext-xml": "*",
|
||||
"php": ">=5.3.0",
|
||||
"zendframework/zendxml": "^1.0"
|
||||
},
|
||||
"require-dev": {
|
||||
"phpdocumentor/reflection-docblock": "2.0.4",
|
||||
"phpunit/phpunit": "4.8.26",
|
||||
"symfony/yaml": "2.8.7"
|
||||
},
|
||||
"suggest": {
|
||||
"ext-curl": "PicoFeed will use cURL if present"
|
||||
},
|
||||
"bin": [
|
||||
"picofeed"
|
||||
],
|
||||
"type": "library",
|
||||
"autoload": {
|
||||
"psr-0": {
|
||||
"PicoFeed": "lib/"
|
||||
}
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"license": [
|
||||
"MIT"
|
||||
],
|
||||
"authors": [
|
||||
{
|
||||
"name": "Frédéric Guillot"
|
||||
}
|
||||
],
|
||||
"description": "Modern library to handle RSS/Atom feeds",
|
||||
"homepage": "https://github.com/miniflux/picoFeed",
|
||||
"time": "2017-06-20T22:54:47+00:00"
|
||||
},
|
||||
{
|
||||
"name": "pear/net_url2",
|
||||
"version": "v2.2.2",
|
||||
|
|
@ -170,6 +227,53 @@
|
|||
],
|
||||
"time": "2017-08-25T06:16:11+00:00"
|
||||
},
|
||||
{
|
||||
"name": "psr/log",
|
||||
"version": "1.1.0",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/php-fig/log.git",
|
||||
"reference": "6c001f1daafa3a3ac1d8ff69ee4db8e799a654dd"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/php-fig/log/zipball/6c001f1daafa3a3ac1d8ff69ee4db8e799a654dd",
|
||||
"reference": "6c001f1daafa3a3ac1d8ff69ee4db8e799a654dd",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"php": ">=5.3.0"
|
||||
},
|
||||
"type": "library",
|
||||
"extra": {
|
||||
"branch-alias": {
|
||||
"dev-master": "1.0.x-dev"
|
||||
}
|
||||
},
|
||||
"autoload": {
|
||||
"psr-4": {
|
||||
"Psr\\Log\\": "Psr/Log/"
|
||||
}
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"license": [
|
||||
"MIT"
|
||||
],
|
||||
"authors": [
|
||||
{
|
||||
"name": "PHP-FIG",
|
||||
"homepage": "http://www.php-fig.org/"
|
||||
}
|
||||
],
|
||||
"description": "Common interface for logging libraries",
|
||||
"homepage": "https://github.com/php-fig/log",
|
||||
"keywords": [
|
||||
"log",
|
||||
"psr",
|
||||
"psr-3"
|
||||
],
|
||||
"time": "2018-11-20T15:27:04+00:00"
|
||||
},
|
||||
{
|
||||
"name": "riimu/kit-pathjoin",
|
||||
"version": "v1.2.0",
|
||||
|
|
@ -219,52 +323,6 @@
|
|||
"system"
|
||||
],
|
||||
"time": "2017-07-09T14:41:04+00:00"
|
||||
},
|
||||
{
|
||||
"name": "zendframework/zendxml",
|
||||
"version": "1.1.0",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/zendframework/ZendXml.git",
|
||||
"reference": "267db6a2c431a08a8f8ff0f1f4c302a5ba6f5b99"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/zendframework/ZendXml/zipball/267db6a2c431a08a8f8ff0f1f4c302a5ba6f5b99",
|
||||
"reference": "267db6a2c431a08a8f8ff0f1f4c302a5ba6f5b99",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"php": "^5.6 || ^7.0"
|
||||
},
|
||||
"require-dev": {
|
||||
"phpunit/phpunit": "^5.7.27 || ^6.5.8 || ^7.1.4",
|
||||
"zendframework/zend-coding-standard": "~1.0.0"
|
||||
},
|
||||
"type": "library",
|
||||
"extra": {
|
||||
"branch-alias": {
|
||||
"dev-master": "1.1.x-dev",
|
||||
"dev-develop": "1.2.x-dev"
|
||||
}
|
||||
},
|
||||
"autoload": {
|
||||
"psr-4": {
|
||||
"ZendXml\\": "src/"
|
||||
}
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"license": [
|
||||
"BSD-3-Clause"
|
||||
],
|
||||
"description": "Utility library for XML usage, best practices, and security in PHP",
|
||||
"keywords": [
|
||||
"ZendFramework",
|
||||
"security",
|
||||
"xml",
|
||||
"zf"
|
||||
],
|
||||
"time": "2018-04-30T15:11:04+00:00"
|
||||
}
|
||||
],
|
||||
"packages-dev": [
|
||||
|
|
@ -938,16 +996,16 @@
|
|||
},
|
||||
{
|
||||
"name": "phpunit/phpunit",
|
||||
"version": "6.5.13",
|
||||
"version": "6.5.14",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/sebastianbergmann/phpunit.git",
|
||||
"reference": "0973426fb012359b2f18d3bd1e90ef1172839693"
|
||||
"reference": "bac23fe7ff13dbdb461481f706f0e9fe746334b7"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/0973426fb012359b2f18d3bd1e90ef1172839693",
|
||||
"reference": "0973426fb012359b2f18d3bd1e90ef1172839693",
|
||||
"url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/bac23fe7ff13dbdb461481f706f0e9fe746334b7",
|
||||
"reference": "bac23fe7ff13dbdb461481f706f0e9fe746334b7",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
|
|
@ -1018,7 +1076,7 @@
|
|||
"testing",
|
||||
"xunit"
|
||||
],
|
||||
"time": "2018-09-08T15:10:43+00:00"
|
||||
"time": "2019-02-01T05:22:47+00:00"
|
||||
},
|
||||
{
|
||||
"name": "phpunit/phpunit-mock-objects",
|
||||
|
|
@ -1077,6 +1135,7 @@
|
|||
"mock",
|
||||
"xunit"
|
||||
],
|
||||
"abandoned": true,
|
||||
"time": "2018-08-09T05:50:03+00:00"
|
||||
},
|
||||
{
|
||||
|
|
@ -1640,16 +1699,16 @@
|
|||
},
|
||||
{
|
||||
"name": "squizlabs/php_codesniffer",
|
||||
"version": "3.3.2",
|
||||
"version": "3.4.0",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/squizlabs/PHP_CodeSniffer.git",
|
||||
"reference": "6ad28354c04b364c3c71a34e4a18b629cc3b231e"
|
||||
"reference": "379deb987e26c7cd103a7b387aea178baec96e48"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/squizlabs/PHP_CodeSniffer/zipball/6ad28354c04b364c3c71a34e4a18b629cc3b231e",
|
||||
"reference": "6ad28354c04b364c3c71a34e4a18b629cc3b231e",
|
||||
"url": "https://api.github.com/repos/squizlabs/PHP_CodeSniffer/zipball/379deb987e26c7cd103a7b387aea178baec96e48",
|
||||
"reference": "379deb987e26c7cd103a7b387aea178baec96e48",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
|
|
@ -1687,7 +1746,65 @@
|
|||
"phpcs",
|
||||
"standards"
|
||||
],
|
||||
"time": "2018-09-23T23:08:17+00:00"
|
||||
"time": "2018-12-19T23:57:18+00:00"
|
||||
},
|
||||
{
|
||||
"name": "symfony/polyfill-ctype",
|
||||
"version": "v1.10.0",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/symfony/polyfill-ctype.git",
|
||||
"reference": "e3d826245268269cd66f8326bd8bc066687b4a19"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/e3d826245268269cd66f8326bd8bc066687b4a19",
|
||||
"reference": "e3d826245268269cd66f8326bd8bc066687b4a19",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"php": ">=5.3.3"
|
||||
},
|
||||
"suggest": {
|
||||
"ext-ctype": "For best performance"
|
||||
},
|
||||
"type": "library",
|
||||
"extra": {
|
||||
"branch-alias": {
|
||||
"dev-master": "1.9-dev"
|
||||
}
|
||||
},
|
||||
"autoload": {
|
||||
"psr-4": {
|
||||
"Symfony\\Polyfill\\Ctype\\": ""
|
||||
},
|
||||
"files": [
|
||||
"bootstrap.php"
|
||||
]
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"license": [
|
||||
"MIT"
|
||||
],
|
||||
"authors": [
|
||||
{
|
||||
"name": "Symfony Community",
|
||||
"homepage": "https://symfony.com/contributors"
|
||||
},
|
||||
{
|
||||
"name": "Gert de Pagter",
|
||||
"email": "BackEndTea@gmail.com"
|
||||
}
|
||||
],
|
||||
"description": "Symfony polyfill for ctype functions",
|
||||
"homepage": "https://symfony.com",
|
||||
"keywords": [
|
||||
"compatibility",
|
||||
"ctype",
|
||||
"polyfill",
|
||||
"portable"
|
||||
],
|
||||
"time": "2018-08-06T14:22:27+00:00"
|
||||
},
|
||||
{
|
||||
"name": "theseer/tokenizer",
|
||||
|
|
@ -1731,20 +1848,21 @@
|
|||
},
|
||||
{
|
||||
"name": "webmozart/assert",
|
||||
"version": "1.3.0",
|
||||
"version": "1.4.0",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/webmozart/assert.git",
|
||||
"reference": "0df1908962e7a3071564e857d86874dad1ef204a"
|
||||
"reference": "83e253c8e0be5b0257b881e1827274667c5c17a9"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/webmozart/assert/zipball/0df1908962e7a3071564e857d86874dad1ef204a",
|
||||
"reference": "0df1908962e7a3071564e857d86874dad1ef204a",
|
||||
"url": "https://api.github.com/repos/webmozart/assert/zipball/83e253c8e0be5b0257b881e1827274667c5c17a9",
|
||||
"reference": "83e253c8e0be5b0257b881e1827274667c5c17a9",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"php": "^5.3.3 || ^7.0"
|
||||
"php": "^5.3.3 || ^7.0",
|
||||
"symfony/polyfill-ctype": "^1.8"
|
||||
},
|
||||
"require-dev": {
|
||||
"phpunit/phpunit": "^4.6",
|
||||
|
|
@ -1777,7 +1895,7 @@
|
|||
"check",
|
||||
"validate"
|
||||
],
|
||||
"time": "2018-01-29T19:49:41+00:00"
|
||||
"time": "2018-12-25T11:19:39+00:00"
|
||||
}
|
||||
],
|
||||
"aliases": [],
|
||||
|
|
|
|||
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
|
|
@ -43,21 +43,23 @@
|
|||
"gulp-uglify": "^2.0.0",
|
||||
"jasmine-core": "^2.99.1",
|
||||
"jquery": "^2.2.4",
|
||||
"jshint": "^2.9.6",
|
||||
"jshint": "^2.10.1",
|
||||
"karma": "^1.3.0",
|
||||
"karma-chrome-launcher": "^2.0.0",
|
||||
"karma-coverage": "^1.1.2",
|
||||
"karma-firefox-launcher": "^1.0.0",
|
||||
"karma-jasmine": "^1.1.2"
|
||||
"karma-jasmine": "^1.1.2",
|
||||
"natives": "^1.1.6",
|
||||
"minimatch": "^3.0.4"
|
||||
},
|
||||
"dependencies": {
|
||||
"angular": "^1.7.5",
|
||||
"angular-animate": "^1.7.5",
|
||||
"angular-mocks": "^1.7.5",
|
||||
"angular-route": "^1.7.5",
|
||||
"angular-sanitize": "^1.7.5",
|
||||
"angular": "^1.7.7",
|
||||
"angular-animate": "^1.7.7",
|
||||
"angular-mocks": "^1.7.7",
|
||||
"angular-route": "^1.7.7",
|
||||
"angular-sanitize": "^1.7.7",
|
||||
"debug": "^2.6.8",
|
||||
"masonry-layout": "^4.2.2",
|
||||
"moment": "^2.22.2"
|
||||
"moment": "^2.24.0"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,27 +13,46 @@
|
|||
|
||||
namespace OCA\News\AppInfo;
|
||||
|
||||
use Closure;
|
||||
use FeedIo\FeedIo;
|
||||
use HTMLPurifier;
|
||||
use HTMLPurifier_Config;
|
||||
|
||||
use OCA\News\Config\FetcherConfig;
|
||||
use OCA\News\Utility\PsrLogger;
|
||||
use OCP\BackgroundJob\IJobList;
|
||||
|
||||
use OCP\IContainer;
|
||||
use OCP\INavigationManager;
|
||||
use OCP\IURLGenerator;
|
||||
use OCP\IConfig;
|
||||
use OCP\AppFramework\App;
|
||||
use OCP\Files\IRootFolder;
|
||||
use OCP\Files\Node;
|
||||
|
||||
|
||||
use OCA\News\Config\AppConfig;
|
||||
use OCA\News\Config\Config;
|
||||
use OCA\News\Db\ItemMapper;
|
||||
use OCA\News\Db\MapperFactory;
|
||||
use OCA\News\Db\ItemMapper;
|
||||
use OCA\News\Fetcher\FeedFetcher;
|
||||
use OCA\News\Fetcher\Fetcher;
|
||||
use OCA\News\Fetcher\YoutubeFetcher;
|
||||
use OCA\News\Utility\ProxyConfigParser;
|
||||
use OCP\AppFramework\App;
|
||||
use OCP\Files\IRootFolder;
|
||||
use OCP\Files\Node;
|
||||
use OCP\IConfig;
|
||||
use OCP\IContainer;
|
||||
use OCP\ILogger;
|
||||
use PicoFeed\Config\Config as PicoFeedConfig;
|
||||
use PicoFeed\Reader\Reader as PicoFeedReader;
|
||||
|
||||
/**
|
||||
* Class Application
|
||||
*
|
||||
* @package OCA\News\AppInfo
|
||||
*/
|
||||
class Application extends App
|
||||
{
|
||||
|
||||
/**
|
||||
* Application constructor.
|
||||
*
|
||||
* @param array $urlParams Parameters
|
||||
*/
|
||||
public function __construct(array $urlParams = [])
|
||||
{
|
||||
parent::__construct('news', $urlParams);
|
||||
|
|
@ -53,10 +72,25 @@ class Application extends App
|
|||
$container->registerParameter('configFile', 'config.ini');
|
||||
|
||||
// factories
|
||||
$container->registerService(ItemMapper::class, function (IContainer $c) {
|
||||
$container->registerService(ItemMapper::class, function (IContainer $c): ItemMapper {
|
||||
return $c->query(MapperFactory::class)->build();
|
||||
});
|
||||
|
||||
|
||||
/**
|
||||
* App config parser.
|
||||
*/
|
||||
$container->registerService(AppConfig::class, function (IContainer $c): AppConfig {
|
||||
$config = new AppConfig(
|
||||
$c->query(INavigationManager::class),
|
||||
$c->query(IURLGenerator::class),
|
||||
$c->query(IJobList::class)
|
||||
);
|
||||
|
||||
$config->loadConfig($c->query('info'));
|
||||
return $config;
|
||||
});
|
||||
|
||||
/**
|
||||
* Core
|
||||
*/
|
||||
|
|
@ -79,10 +113,21 @@ class Application extends App
|
|||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Logger base
|
||||
*/
|
||||
$container->registerService(PsrLogger::class, function (IContainer $c): PsrLogger {
|
||||
return new PsrLogger(
|
||||
$c->query('ServerContainer')->getLogger(),
|
||||
$c->query('AppName')
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
$container->registerService(Config::class, function (IContainer $c): Config {
|
||||
$config = new Config(
|
||||
$c->query('ConfigView'),
|
||||
$c->query(ILogger::class),
|
||||
$c->query(PsrLogger::class),
|
||||
$c->query('LoggerParameters')
|
||||
);
|
||||
$config->read($c->query('configFile'), true);
|
||||
|
|
@ -115,47 +160,26 @@ class Application extends App
|
|||
/**
|
||||
* Fetchers
|
||||
*/
|
||||
$container->registerService(PicoFeedConfig::class, function (IContainer $c): PicoFeedConfig {
|
||||
$container->registerService(FetcherConfig::class, function (IContainer $c): FetcherConfig {
|
||||
// FIXME: move this into a separate class for testing?
|
||||
$config = $c->query(Config::class);
|
||||
$proxy = $c->query(ProxyConfigParser::class);
|
||||
$proxy = $c->query(ProxyConfigParser::class);
|
||||
|
||||
$userAgent = 'NextCloud-News/1.0';
|
||||
$fConfig = new FetcherConfig();
|
||||
$fConfig->setClientTimeout($config->getFeedFetcherTimeout());
|
||||
$fConfig->setProxy($proxy);
|
||||
|
||||
$pico = new PicoFeedConfig();
|
||||
$pico->setClientUserAgent($userAgent)
|
||||
->setClientTimeout($config->getFeedFetcherTimeout())
|
||||
->setMaxRedirections($config->getMaxRedirects())
|
||||
->setMaxBodySize($config->getMaxSize())
|
||||
->setParserHashAlgo('md5');
|
||||
|
||||
// proxy settings
|
||||
$proxySettings = $proxy->parse();
|
||||
$host = $proxySettings['host'];
|
||||
$port = $proxySettings['port'];
|
||||
$user = $proxySettings['user'];
|
||||
$password = $proxySettings['password'];
|
||||
|
||||
if ($host) {
|
||||
$pico->setProxyHostname($host);
|
||||
|
||||
if ($port) {
|
||||
$pico->setProxyPort($port);
|
||||
}
|
||||
}
|
||||
|
||||
if ($user) {
|
||||
$pico->setProxyUsername($user)
|
||||
->setProxyPassword($password);
|
||||
}
|
||||
|
||||
return $pico;
|
||||
return $fConfig;
|
||||
});
|
||||
|
||||
$container->registerService(PicoFeedReader::class, function (IContainer $c): PicoFeedReader {
|
||||
return new PicoFeedReader($c->query(PicoFeedConfig::class));
|
||||
$container->registerService(FeedIo::class, function (IContainer $c): FeedIo {
|
||||
$config = $c->query(FetcherConfig::class);
|
||||
return new FeedIo($config->getClient(), $c->query(PsrLogger::class));
|
||||
});
|
||||
|
||||
/**
|
||||
* @noinspection PhpParamsInspection
|
||||
*/
|
||||
$container->registerService(Fetcher::class, function (IContainer $c): Fetcher {
|
||||
$fetcher = new Fetcher();
|
||||
|
||||
|
|
@ -163,7 +187,6 @@ class Application extends App
|
|||
// the last one
|
||||
$fetcher->registerFetcher($c->query(YoutubeFetcher::class));
|
||||
$fetcher->registerFetcher($c->query(FeedFetcher::class));
|
||||
|
||||
return $fetcher;
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
|
||||
namespace OCA\News\Config;
|
||||
|
||||
use OCP\ILogger;
|
||||
use OCA\News\Utility\PsrLogger;
|
||||
use OCP\Files\Folder;
|
||||
|
||||
class Config
|
||||
|
|
@ -35,7 +35,7 @@ class Config
|
|||
|
||||
public function __construct(
|
||||
Folder $fileSystem,
|
||||
ILogger $logger,
|
||||
PsrLogger $logger,
|
||||
$LoggerParameters
|
||||
) {
|
||||
$this->fileSystem = $fileSystem;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,118 @@
|
|||
<?php
|
||||
/**
|
||||
* Nextcloud - News
|
||||
*
|
||||
* This file is licensed under the Affero General Public License version 3 or
|
||||
* later. See the COPYING file.
|
||||
*
|
||||
* @author Alessandro Cosentino <cosenal@gmail.com>
|
||||
* @author Bernhard Posselt <dev@bernhard-posselt.com>
|
||||
* @copyright 2012 Alessandro Cosentino
|
||||
* @copyright 2012-2014 Bernhard Posselt
|
||||
*/
|
||||
|
||||
namespace OCA\News\Config;
|
||||
|
||||
use FeedIo\Adapter\ClientInterface;
|
||||
use \GuzzleHttp\Client;
|
||||
use \FeedIo\Adapter\Guzzle\Client as FeedIoClient;
|
||||
|
||||
/**
|
||||
* Class FetcherConfig
|
||||
*
|
||||
* @package OCA\News\Config
|
||||
*/
|
||||
class FetcherConfig
|
||||
{
|
||||
protected $client_timeout;
|
||||
protected $proxy;
|
||||
|
||||
/**
|
||||
* Configure a guzzle client
|
||||
*
|
||||
* @return ClientInterface Legacy client to guzzle.
|
||||
*/
|
||||
public function getClient()
|
||||
{
|
||||
if (!class_exists('GuzzleHttp\Collection')) {
|
||||
$config = [
|
||||
'timeout' => $this->getClientTimeout(),
|
||||
];
|
||||
|
||||
if (!empty($this->proxy)) {
|
||||
$config['proxy'] = $this->proxy;
|
||||
}
|
||||
|
||||
$guzzle = new Client();
|
||||
$client = new FeedIoClient($guzzle);
|
||||
|
||||
return $client;
|
||||
}
|
||||
|
||||
$config = [
|
||||
'request.options' => [
|
||||
'timeout' => $this->getClientTimeout(),
|
||||
],
|
||||
];
|
||||
|
||||
if (!empty($this->proxy)) {
|
||||
$config['request.options']['proxy'] = $this->proxy;
|
||||
}
|
||||
|
||||
$guzzle = new Client($config);
|
||||
return new LegacyGuzzleClient($guzzle);
|
||||
}
|
||||
|
||||
/**
|
||||
* Set a timeout for the client
|
||||
*
|
||||
* @param int $timeout The timeout
|
||||
*
|
||||
* @return self
|
||||
*/
|
||||
public function setClientTimeout($timeout)
|
||||
{
|
||||
$this->client_timeout = $timeout;
|
||||
|
||||
return $this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the client timeout.
|
||||
*
|
||||
* @return mixed
|
||||
*/
|
||||
public function getClientTimeout()
|
||||
{
|
||||
return $this->client_timeout;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the proxy
|
||||
*
|
||||
* @param \OCA\News\Utility\ProxyConfigParser $proxy The proxy to set.
|
||||
*
|
||||
* @return self
|
||||
*/
|
||||
public function setProxy($proxy)
|
||||
{
|
||||
// proxy settings
|
||||
$proxySettings = $proxy->parse();
|
||||
$host = $proxySettings['host'];
|
||||
$port = $proxySettings['port'];
|
||||
$user = $proxySettings['user'];
|
||||
$password = $proxySettings['password'];
|
||||
|
||||
$proxy_string = 'https://';
|
||||
if (!empty($user)) {
|
||||
$proxy_string .= $user . ':' . $password . '@';
|
||||
}
|
||||
$proxy_string .= $host;
|
||||
if (!empty($port)) {
|
||||
$proxy_string .= ':' . $port;
|
||||
}
|
||||
$this->proxy = $proxy_string;
|
||||
|
||||
return $this;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,65 @@
|
|||
<?php
|
||||
/**
|
||||
* Nextcloud - News
|
||||
*
|
||||
* This file is licensed under the Affero General Public License version 3 or
|
||||
* later. See the COPYING file.
|
||||
*
|
||||
* @author Sean Molenaar <smillernl@me.com>
|
||||
* @copyright 2018 Sean Molenaar
|
||||
*/
|
||||
|
||||
namespace OCA\News\Config;
|
||||
|
||||
use FeedIo\Adapter\ClientInterface as FeedIoClientInterface;
|
||||
use FeedIo\Adapter\NotFoundException;
|
||||
use FeedIo\Adapter\ServerErrorException;
|
||||
use Guzzle\Service\ClientInterface;
|
||||
use GuzzleHttp\Exception\BadResponseException;
|
||||
|
||||
/**
|
||||
* Guzzle dependent HTTP client
|
||||
*/
|
||||
class LegacyGuzzleClient implements FeedIoClientInterface
|
||||
{
|
||||
/**
|
||||
* @var ClientInterface
|
||||
*/
|
||||
protected $guzzleClient;
|
||||
|
||||
/**
|
||||
* @param ClientInterface $guzzleClient
|
||||
*/
|
||||
public function __construct(ClientInterface $guzzleClient)
|
||||
{
|
||||
$this->guzzleClient = $guzzleClient;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $url
|
||||
* @param \DateTime $modifiedSince
|
||||
* @throws \FeedIo\Adapter\NotFoundException
|
||||
* @throws \FeedIo\Adapter\ServerErrorException
|
||||
* @return \FeedIo\Adapter\ResponseInterface
|
||||
*/
|
||||
public function getResponse($url, \DateTime $modifiedSince)
|
||||
{
|
||||
try {
|
||||
$options = [
|
||||
'headers' => [
|
||||
'User-Agent' => 'NextCloud-News/1.0',
|
||||
'If-Modified-Since' => $modifiedSince->format(\DateTime::RFC2822)
|
||||
]
|
||||
];
|
||||
|
||||
return new LegacyGuzzleResponse($this->guzzleClient->get($url, $options));
|
||||
} catch (BadResponseException $e) {
|
||||
switch ((int) $e->getResponse()->getStatusCode()) {
|
||||
case 404:
|
||||
throw new NotFoundException($e->getMessage());
|
||||
default:
|
||||
throw new ServerErrorException($e->getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,86 @@
|
|||
<?php
|
||||
/**
|
||||
* Nextcloud - News
|
||||
*
|
||||
* This file is licensed under the Affero General Public License version 3 or
|
||||
* later. See the COPYING file.
|
||||
*
|
||||
* @author Sean Molenaar <smillernl@me.com>
|
||||
* @copyright 2018 Sean Molenaar
|
||||
*/
|
||||
|
||||
namespace OCA\News\Config;
|
||||
|
||||
use FeedIo\Adapter\ResponseInterface;
|
||||
use GuzzleHttp\Message\ResponseInterface as GuzzleResponseInterface;
|
||||
|
||||
/**
|
||||
* Guzzle dependent HTTP Response
|
||||
*/
|
||||
class LegacyGuzzleResponse implements ResponseInterface
|
||||
{
|
||||
const HTTP_LAST_MODIFIED = 'Last-Modified';
|
||||
|
||||
/**
|
||||
* @var \GuzzleHttp\Message\ResponseInterface
|
||||
*/
|
||||
protected $response;
|
||||
|
||||
/**
|
||||
* @param \GuzzleHttp\Message\ResponseInterface
|
||||
*/
|
||||
public function __construct(GuzzleResponseInterface $psrResponse)
|
||||
{
|
||||
$this->response = $psrResponse;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return boolean
|
||||
*/
|
||||
public function isModified()
|
||||
{
|
||||
return $this->response->getStatusCode() !== 304 && $this->response->getBody()->getSize() > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return \Psr\Http\Message\StreamInterface
|
||||
*/
|
||||
public function getBody()
|
||||
{
|
||||
return $this->response->getBody();
|
||||
}
|
||||
|
||||
/**
|
||||
* @return \DateTime|null
|
||||
*/
|
||||
public function getLastModified()
|
||||
{
|
||||
if ($this->response->hasHeader(static::HTTP_LAST_MODIFIED)) {
|
||||
$lastModified = \DateTime::createFromFormat(
|
||||
\DateTime::RFC2822,
|
||||
$this->getHeader(static::HTTP_LAST_MODIFIED)
|
||||
);
|
||||
|
||||
return false === $lastModified ? null : $lastModified;
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array
|
||||
*/
|
||||
public function getHeaders()
|
||||
{
|
||||
return $this->response->getHeaders();
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $name
|
||||
* @return string[]
|
||||
*/
|
||||
public function getHeader($name)
|
||||
{
|
||||
return $this->response->getHeader($name);
|
||||
}
|
||||
}
|
||||
|
|
@ -491,4 +491,18 @@ class Item extends Entity implements IAPI, \JsonSerializable
|
|||
$this->getEnclosureLink()
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a given mimetype is supported
|
||||
*
|
||||
* @param string $mime mimetype to check
|
||||
*
|
||||
* @return boolean
|
||||
*/
|
||||
public function isSupportedMime($mime)
|
||||
{
|
||||
return (
|
||||
stripos($mime, 'audio/') !== false ||
|
||||
stripos($mime, 'video/') !== false);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,29 +13,17 @@
|
|||
|
||||
namespace OCA\News\Fetcher;
|
||||
|
||||
use Exception;
|
||||
|
||||
use OCA\News\PostProcessor\LWNProcessor;
|
||||
use OCP\Http\Client\IClientService;
|
||||
use PicoFeed\Parser\MalFormedXmlException;
|
||||
use PicoFeed\Reader\Reader;
|
||||
use PicoFeed\Parser\Parser;
|
||||
use PicoFeed\Reader\SubscriptionNotFoundException;
|
||||
use PicoFeed\Reader\UnsupportedFeedFormatException;
|
||||
use PicoFeed\Client\InvalidCertificateException;
|
||||
use PicoFeed\Client\InvalidUrlException;
|
||||
use PicoFeed\Client\MaxRedirectException;
|
||||
use PicoFeed\Client\MaxSizeException;
|
||||
use PicoFeed\Client\TimeoutException;
|
||||
use PicoFeed\Client\ForbiddenException;
|
||||
use PicoFeed\Client\UnauthorizedException;
|
||||
use DateTime;
|
||||
use Favicon\Favicon;
|
||||
use FeedIo\Feed\ItemInterface;
|
||||
use FeedIo\FeedInterface;
|
||||
use FeedIo\FeedIo;
|
||||
|
||||
use OCA\News\Utility\PsrLogger;
|
||||
use OCP\IL10N;
|
||||
|
||||
use OCA\News\Db\Item;
|
||||
use OCA\News\Db\Feed;
|
||||
use OCA\News\Utility\PicoFeedFaviconFactory;
|
||||
use OCA\News\Utility\PicoFeedReaderFactory;
|
||||
use OCA\News\Utility\Time;
|
||||
|
||||
class FeedFetcher implements IFeedFetcher
|
||||
|
|
@ -45,27 +33,26 @@ class FeedFetcher implements IFeedFetcher
|
|||
private $reader;
|
||||
private $l10n;
|
||||
private $time;
|
||||
private $clientService;
|
||||
private $logger;
|
||||
|
||||
public function __construct(
|
||||
Reader $reader,
|
||||
PicoFeedFaviconFactory $faviconFactory,
|
||||
IL10N $l10n,
|
||||
Time $time,
|
||||
IClientService $clientService
|
||||
) {
|
||||
$this->faviconFactory = $faviconFactory;
|
||||
$this->reader = $reader;
|
||||
$this->time = $time;
|
||||
$this->l10n = $l10n;
|
||||
$this->clientService = $clientService;
|
||||
public function __construct(FeedIo $fetcher, Favicon $favicon, IL10N $l10n, Time $time, PsrLogger $logger)
|
||||
{
|
||||
$this->reader = $fetcher;
|
||||
$this->faviconFactory = $favicon;
|
||||
$this->l10n = $l10n;
|
||||
$this->time = $time;
|
||||
$this->logger = $logger;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* This fetcher handles all the remaining urls therefore always returns true
|
||||
* This fetcher handles all the remaining urls therefore always returns true.
|
||||
*
|
||||
* @param string $url The URL to check
|
||||
*
|
||||
* @return bool
|
||||
*/
|
||||
public function canHandle($url)
|
||||
public function canHandle($url): bool
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
|
@ -74,177 +61,56 @@ class FeedFetcher implements IFeedFetcher
|
|||
/**
|
||||
* Fetch a feed from remote
|
||||
*
|
||||
* @param string $url remote url of the feed
|
||||
* @param boolean $getFavicon if the favicon should also be fetched, defaults to true
|
||||
* @param string $lastModified a last modified value from an http header defaults to false.
|
||||
* If lastModified matches the http header from the feed no results are fetched
|
||||
* @param string $etag an etag from an http header.
|
||||
* If lastModified matches the http header from the feed no results are fetched
|
||||
* @param bool $fullTextEnabled if true tells the fetcher to enhance the articles by fetching more content
|
||||
* @param string $basicAuthUser if given, basic auth is set for this feed
|
||||
* @param string $basicAuthPassword if given, basic auth is set for this feed. Ignored if user is empty
|
||||
*
|
||||
* @throws FetcherException if it fails
|
||||
* @return array an array containing the new feed and its items, first
|
||||
* element being the Feed and second element being an array of Items
|
||||
* @inheritdoc
|
||||
*/
|
||||
public function fetch(
|
||||
$url,
|
||||
$getFavicon = true,
|
||||
$lastModified = null,
|
||||
$etag = null,
|
||||
$fullTextEnabled = false,
|
||||
$basicAuthUser = null,
|
||||
$basicAuthPassword = null
|
||||
) {
|
||||
try {
|
||||
if ($basicAuthUser !== null && trim($basicAuthUser) !== '') {
|
||||
$resource = $this->reader->discover(
|
||||
$url,
|
||||
$lastModified,
|
||||
$etag,
|
||||
$basicAuthUser,
|
||||
$basicAuthPassword
|
||||
);
|
||||
} else {
|
||||
$resource = $this->reader->discover($url, $lastModified, $etag);
|
||||
}
|
||||
|
||||
if (!$resource->isModified()) {
|
||||
return [null, null];
|
||||
}
|
||||
|
||||
$location = $resource->getUrl();
|
||||
$etag = $resource->getEtag();
|
||||
$content = $resource->getContent();
|
||||
$encoding = $resource->getEncoding();
|
||||
$lastModified = $resource->getLastModified();
|
||||
|
||||
$parser = $this->reader->getParser($location, $content, $encoding);
|
||||
|
||||
if ($fullTextEnabled) {
|
||||
$parser->enableContentGrabber();
|
||||
$parser->getItemPostProcessor()->register(
|
||||
new LWNProcessor(
|
||||
$basicAuthUser,
|
||||
$basicAuthPassword,
|
||||
$this->clientService
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
$parsedFeed = $parser->execute();
|
||||
|
||||
$feed = $this->buildFeed(
|
||||
$parsedFeed,
|
||||
$url,
|
||||
$getFavicon,
|
||||
$lastModified,
|
||||
$etag,
|
||||
$location
|
||||
);
|
||||
|
||||
$items = [];
|
||||
foreach ($parsedFeed->getItems() as $item) {
|
||||
$items[] = $this->buildItem($item, $parsedFeed);
|
||||
}
|
||||
|
||||
return [$feed, $items];
|
||||
} catch (Exception $ex) {
|
||||
$this->handleError($ex, $url);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private function handleError(Exception $ex, $url)
|
||||
public function fetch(string $url, bool $favicon, $lastModified, $user, $password): array
|
||||
{
|
||||
$msg = $ex->getMessage();
|
||||
|
||||
if ($ex instanceof MalFormedXmlException) {
|
||||
$msg = $this->l10n->t('Feed contains invalid XML');
|
||||
} elseif ($ex instanceof SubscriptionNotFoundException) {
|
||||
$msg = $this->l10n->t(
|
||||
'Feed not found: Either the website ' .
|
||||
'does not provide a feed or blocks access. To rule out ' .
|
||||
'blocking, try to download the feed on your server\'s ' .
|
||||
'command line using curl: curl ' . $url
|
||||
);
|
||||
} elseif ($ex instanceof UnsupportedFeedFormatException) {
|
||||
$msg = $this->l10n->t('Detected feed format is not supported');
|
||||
} elseif ($ex instanceof InvalidCertificateException) {
|
||||
$msg = $this->buildCurlSslErrorMessage($ex->getCode());
|
||||
} elseif ($ex instanceof InvalidUrlException) {
|
||||
$msg = $this->l10n->t('Website not found');
|
||||
} elseif ($ex instanceof MaxRedirectException) {
|
||||
$msg = $this->l10n->t('More redirects than allowed, aborting');
|
||||
} elseif ($ex instanceof MaxSizeException) {
|
||||
$msg = $this->l10n->t('Bigger than maximum allowed size');
|
||||
} elseif ($ex instanceof TimeoutException) {
|
||||
$msg = $this->l10n->t('Request timed out');
|
||||
} elseif ($ex instanceof UnauthorizedException) {
|
||||
$msg = $this->l10n->t(
|
||||
'Required credentials for feed were ' .
|
||||
'either missing or incorrect'
|
||||
);
|
||||
} elseif ($ex instanceof ForbiddenException) {
|
||||
$msg = $this->l10n->t('Forbidden to access feed');
|
||||
if (!empty($user) && !empty(trim($user))) {
|
||||
$url = explode('://', $url);
|
||||
$url = $url[0] . '://' . $user . ':' . $password . '@' . $url[1];
|
||||
}
|
||||
if (is_null($lastModified) || !is_string($lastModified)) {
|
||||
$resource = $this->reader->read($url);
|
||||
} else {
|
||||
$resource = $this->reader->readSince($url, new DateTime($lastModified));
|
||||
}
|
||||
|
||||
throw new FetcherException($msg);
|
||||
}
|
||||
|
||||
private function buildCurlSslErrorMessage($errorCode)
|
||||
{
|
||||
switch ($errorCode) {
|
||||
case 35: // CURLE_SSL_CONNECT_ERROR
|
||||
return $this->l10n->t(
|
||||
'Certificate error: A problem occurred ' .
|
||||
'somewhere in the SSL/TLS handshake. Could be ' .
|
||||
'certificates (file formats, paths, permissions), ' .
|
||||
'passwords, and others.'
|
||||
);
|
||||
case 51: // CURLE_PEER_FAILED_VERIFICATION
|
||||
return $this->l10n->t(
|
||||
'Certificate error: The remote server\'s SSL ' .
|
||||
'certificate or SSH md5 fingerprint was deemed not OK.'
|
||||
);
|
||||
case 58: // CURLE_SSL_CERTPROBLEM
|
||||
return $this->l10n->t(
|
||||
'Certificate error: Problem with the local client ' .
|
||||
'certificate.'
|
||||
);
|
||||
case 59: // CURLE_SSL_CIPHER
|
||||
return $this->l10n->t(
|
||||
'Certificate error: Couldn\'t use specified cipher.'
|
||||
);
|
||||
case 60: // CURLE_SSL_CACERT
|
||||
return $this->l10n->t(
|
||||
'Certificate error: Peer certificate cannot be ' .
|
||||
'authenticated with known CA certificates.'
|
||||
);
|
||||
case 64: // CURLE_USE_SSL_FAILED
|
||||
return $this->l10n->t(
|
||||
'Certificate error: Requested FTP SSL level failed.'
|
||||
);
|
||||
case 66: // CURLE_SSL_ENGINE_INITFAILED
|
||||
return $this->l10n->t(
|
||||
'Certificate error: Initiating the SSL engine failed.'
|
||||
);
|
||||
case 77: // CURLE_SSL_CACERT_BADFILE
|
||||
return $this->l10n->t(
|
||||
'Certificate error: Problem with reading the SSL CA ' .
|
||||
'cert (path? access rights?)'
|
||||
);
|
||||
case 83: // CURLE_SSL_ISSUER_ERROR
|
||||
return $this->l10n->t(
|
||||
'Certificate error: Issuer check failed'
|
||||
);
|
||||
default:
|
||||
return $this->l10n->t('Unknown SSL certificate error!');
|
||||
$response = $resource->getResponse();
|
||||
if (!$response->isModified()) {
|
||||
$this->logger->debug('Feed {url} was not modified since last fetch. old: {old}, new: {new}', [
|
||||
'url' => $url,
|
||||
'old' => print_r($lastModified, true),
|
||||
'new' => print_r($response->getLastModified(), true),
|
||||
]);
|
||||
return [null, []];
|
||||
}
|
||||
|
||||
$location = $resource->getUrl();
|
||||
$parsedFeed = $resource->getFeed();
|
||||
$feed = $this->buildFeed(
|
||||
$parsedFeed,
|
||||
$url,
|
||||
$favicon,
|
||||
$location
|
||||
);
|
||||
|
||||
$items = [];
|
||||
$this->logger->debug('Feed ' . $url . ' was modified since last fetch. #' . count($parsedFeed) . ' items');
|
||||
foreach ($parsedFeed as $item) {
|
||||
$items[] = $this->buildItem($item, $parsedFeed);
|
||||
}
|
||||
|
||||
return [$feed, $items];
|
||||
}
|
||||
|
||||
private function decodeTwice($string)
|
||||
/**
|
||||
* Decode the string twice
|
||||
*
|
||||
* @param string $string String to decode
|
||||
*
|
||||
* @return string
|
||||
*/
|
||||
private function decodeTwice($string): string
|
||||
{
|
||||
return html_entity_decode(
|
||||
html_entity_decode(
|
||||
|
|
@ -257,37 +123,79 @@ class FeedFetcher implements IFeedFetcher
|
|||
);
|
||||
}
|
||||
|
||||
|
||||
protected function determineRtl($parsedItem, $parsedFeed)
|
||||
/**
|
||||
* Check if a feed is RTL or not
|
||||
*
|
||||
* @param FeedInterface $parsedFeed The feed that was parsed
|
||||
*
|
||||
* @return bool
|
||||
*/
|
||||
protected function determineRtl(FeedInterface $parsedFeed): bool
|
||||
{
|
||||
$itemLang = $parsedItem->getLanguage();
|
||||
$feedLang = $parsedFeed->getLanguage();
|
||||
$language = $parsedFeed->getLanguage();
|
||||
|
||||
if ($itemLang) {
|
||||
return Parser::isLanguageRTL($itemLang);
|
||||
} else {
|
||||
return Parser::isLanguageRTL($feedLang);
|
||||
$language = strtolower($language);
|
||||
$rtl_languages = array(
|
||||
'ar', // Arabic (ar-**)
|
||||
'fa', // Farsi (fa-**)
|
||||
'ur', // Urdu (ur-**)
|
||||
'ps', // Pashtu (ps-**)
|
||||
'syr', // Syriac (syr-**)
|
||||
'dv', // Divehi (dv-**)
|
||||
'he', // Hebrew (he-**)
|
||||
'yi', // Yiddish (yi-**)
|
||||
);
|
||||
foreach ($rtl_languages as $prefix) {
|
||||
if (strpos($language, $prefix) === 0) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
protected function buildItem($parsedItem, $parsedFeed)
|
||||
/**
|
||||
* Build an item based on a feed.
|
||||
*
|
||||
* @param ItemInterface $parsedItem The item to use
|
||||
* @param FeedInterface $parsedFeed The feed to use
|
||||
*
|
||||
* @return Item
|
||||
*/
|
||||
protected function buildItem(ItemInterface $parsedItem, FeedInterface $parsedFeed): Item
|
||||
{
|
||||
$item = new Item();
|
||||
$item->setUnread(true);
|
||||
$item->setUrl($parsedItem->getUrl());
|
||||
$item->setGuid($parsedItem->getId());
|
||||
$item->setGuidHash($item->getGuid());
|
||||
$item->setPubDate($parsedItem->getPublishedDate()->getTimestamp());
|
||||
$item->setUpdatedDate($parsedItem->getUpdatedDate()->getTimestamp());
|
||||
$item->setRtl($this->determineRtl($parsedItem, $parsedFeed));
|
||||
$item->setUrl($parsedItem->getLink());
|
||||
$item->setGuid($parsedItem->getPublicId());
|
||||
$item->setGuidHash(md5($item->getGuid()));
|
||||
|
||||
$lastmodified = $parsedItem->getLastModified() ?? new \DateTime();
|
||||
if ($parsedItem->getValue('pubDate') !== null) {
|
||||
$pubDT = new DateTime($parsedItem->getValue('pubDate'));
|
||||
} elseif ($parsedItem->getValue('published') !== null) {
|
||||
$pubDT = new DateTime($parsedItem->getValue('published'));
|
||||
} else {
|
||||
$pubDT = $lastmodified;
|
||||
}
|
||||
|
||||
$item->setPubDate(
|
||||
$pubDT->getTimestamp()
|
||||
);
|
||||
|
||||
$item->setLastModified(
|
||||
$lastmodified->getTimestamp()
|
||||
);
|
||||
$item->setRtl($this->determineRtl($parsedFeed));
|
||||
|
||||
// unescape content because angularjs helps against XSS
|
||||
$item->setTitle($this->decodeTwice($parsedItem->getTitle()));
|
||||
$item->setAuthor($this->decodeTwice($parsedItem->getAuthor()));
|
||||
$author = $parsedItem->getAuthor();
|
||||
if (!is_null($author)) {
|
||||
$item->setAuthor($this->decodeTwice($author->getName()));
|
||||
}
|
||||
|
||||
// purification is done in the service layer
|
||||
$body = $parsedItem->getContent();
|
||||
$body = $parsedItem->getDescription();
|
||||
$body = mb_convert_encoding(
|
||||
$body,
|
||||
'HTML-ENTITIES',
|
||||
|
|
@ -295,55 +203,57 @@ class FeedFetcher implements IFeedFetcher
|
|||
);
|
||||
$item->setBody($body);
|
||||
|
||||
$enclosureUrl = $parsedItem->getEnclosureUrl();
|
||||
if ($enclosureUrl) {
|
||||
$enclosureType = $parsedItem->getEnclosureType();
|
||||
if (stripos($enclosureType, 'audio/') !== false
|
||||
|| stripos($enclosureType, 'video/') !== false
|
||||
) {
|
||||
$item->setEnclosureMime($enclosureType);
|
||||
$item->setEnclosureLink($enclosureUrl);
|
||||
if ($parsedItem->hasMedia()) {
|
||||
// TODO: Fix multiple media support
|
||||
foreach ($parsedItem->getMedias() as $media) {
|
||||
if (!$item->isSupportedMime($media->getType())) {
|
||||
continue;
|
||||
}
|
||||
$item->setEnclosureMime($media->getType());
|
||||
$item->setEnclosureLink($media->getUrl());
|
||||
}
|
||||
}
|
||||
|
||||
$item->generateSearchIndex();
|
||||
|
||||
$this->logger->debug('Added item {title} for feed {feed} publishdate: {datetime}', [
|
||||
'title' => $item->getTitle(),
|
||||
'feed' => $parsedFeed->getTitle(),
|
||||
'datetime' => $item->getLastModified(),
|
||||
]);
|
||||
return $item;
|
||||
}
|
||||
|
||||
|
||||
protected function buildFeed(
|
||||
$parsedFeed,
|
||||
$url,
|
||||
$getFavicon,
|
||||
$modified,
|
||||
$etag,
|
||||
$location
|
||||
) {
|
||||
$feed = new Feed();
|
||||
|
||||
$link = $parsedFeed->getSiteUrl();
|
||||
|
||||
if (!$link) {
|
||||
$link = $location;
|
||||
}
|
||||
/**
|
||||
* Build a feed based on provided info
|
||||
*
|
||||
* @param FeedInterface $feed Feed to build from
|
||||
* @param string $url URL to use
|
||||
* @param boolean $getFavicon To get the favicon
|
||||
* @param string $location String base URL
|
||||
*
|
||||
* @return Feed
|
||||
*/
|
||||
protected function buildFeed(FeedInterface $feed, string $url, bool $getFavicon, string $location): Feed
|
||||
{
|
||||
$newFeed = new Feed();
|
||||
|
||||
// unescape content because angularjs helps against XSS
|
||||
$title = strip_tags($this->decodeTwice($parsedFeed->getTitle()));
|
||||
$feed->setTitle($title);
|
||||
$feed->setUrl($url); // the url used to add the feed
|
||||
$feed->setLocation($location); // the url where the feed was found
|
||||
$feed->setLink($link); // <link> attribute in the feed
|
||||
$feed->setHttpLastModified($modified);
|
||||
$feed->setHttpEtag($etag);
|
||||
$feed->setAdded($this->time->getTime());
|
||||
$title = strip_tags($this->decodeTwice($feed->getTitle()));
|
||||
$newFeed->setTitle($title);
|
||||
$newFeed->setUrl($url); // the url used to add the feed
|
||||
$newFeed->setLocation($location); // the url where the feed was found
|
||||
$newFeed->setLink($feed->getLink()); // <link> attribute in the feed
|
||||
$lastmodified = $feed->getLastModified() ?? new DateTime();
|
||||
$newFeed->setLastModified($lastmodified->getTimestamp());
|
||||
$newFeed->setAdded($this->time->getTime());
|
||||
|
||||
if ($getFavicon) {
|
||||
$faviconFetcher = $this->faviconFactory->build();
|
||||
$favicon = $faviconFetcher->find($feed->getLink());
|
||||
$feed->setFaviconLink($favicon);
|
||||
if (!$getFavicon) {
|
||||
return $newFeed;
|
||||
}
|
||||
$favicon = $this->faviconFactory->get($url);
|
||||
$newFeed->setFaviconLink($favicon);
|
||||
|
||||
return $feed;
|
||||
return $newFeed;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,6 +16,10 @@ namespace OCA\News\Fetcher;
|
|||
class Fetcher
|
||||
{
|
||||
|
||||
/**
|
||||
* List of fetchers.
|
||||
* @var IFeedFetcher[]
|
||||
*/
|
||||
private $fetchers;
|
||||
|
||||
public function __construct()
|
||||
|
|
@ -39,39 +43,28 @@ class Fetcher
|
|||
*
|
||||
* @param string $url remote url of the feed
|
||||
* @param boolean $getFavicon if the favicon should also be fetched, defaults to true
|
||||
* @param string $lastModified a last modified value from an http header defaults to false.
|
||||
* @param string $lastModified a last modified value from an http header defaults to false.
|
||||
* If lastModified matches the http header from the feed no results are fetched
|
||||
* @param string $etag an etag from an http header.
|
||||
* If lastModified matches the http header from the feed no results are fetched
|
||||
* @param bool $fullTextEnabled if true tells the fetcher to enhance the articles by fetching more content
|
||||
* @param string $basicAuthUser if given, basic auth is set for this feed
|
||||
* @param string $basicAuthPassword if given, basic auth is set for this feed. Ignored if user is empty
|
||||
* @param string $user if given, basic auth is set for this feed
|
||||
* @param string $password if given, basic auth is set for this feed. Ignored if user is empty
|
||||
*
|
||||
* @throws FetcherException if simple pie fails
|
||||
* @throws FetcherException if FeedIO fails
|
||||
* @return array an array containing the new feed and its items, first
|
||||
* element being the Feed and second element being an array of Items
|
||||
*/
|
||||
public function fetch(
|
||||
$url,
|
||||
$getFavicon = true,
|
||||
$lastModified = null,
|
||||
$etag = null,
|
||||
$fullTextEnabled = false,
|
||||
$basicAuthUser = null,
|
||||
$basicAuthPassword = null
|
||||
) {
|
||||
public function fetch($url, $getFavicon = true, $lastModified = null, $user = null, $password = null)
|
||||
{
|
||||
foreach ($this->fetchers as $fetcher) {
|
||||
if ($fetcher->canHandle($url)) {
|
||||
return $fetcher->fetch(
|
||||
$url,
|
||||
$getFavicon,
|
||||
$lastModified,
|
||||
$etag,
|
||||
$fullTextEnabled,
|
||||
$basicAuthUser,
|
||||
$basicAuthPassword
|
||||
);
|
||||
if (!$fetcher->canHandle($url)) {
|
||||
continue;
|
||||
}
|
||||
return $fetcher->fetch(
|
||||
$url,
|
||||
$getFavicon,
|
||||
$lastModified,
|
||||
$user,
|
||||
$password
|
||||
);
|
||||
}
|
||||
|
||||
return [null, []];
|
||||
|
|
|
|||
|
|
@ -19,29 +19,18 @@ interface IFeedFetcher
|
|||
/**
|
||||
* Fetch feed content.
|
||||
*
|
||||
* @param string $url remote url of the feed
|
||||
* @param boolean $getFavicon if the favicon should also be fetched, defaults to true
|
||||
* @param string $lastModified a last modified value from an http header defaults to false.
|
||||
* @param string $url remote url of the feed
|
||||
* @param boolean $favicon if the favicon should also be fetched, defaults to true
|
||||
* @param string|null $lastModified a last modified value from an http header defaults to false.
|
||||
* If lastModified matches the http header from the feed no results are fetched
|
||||
* @param string $etag an etag from an http header.
|
||||
* If lastModified matches the http header from the feed no results are fetched
|
||||
* @param bool $fullTextEnabled if true tells the fetcher to enhance the articles by fetching more content
|
||||
* @param string $basicAuthUser if given, basic auth is set for this feed
|
||||
* @param string $basicAuthPassword if given, basic auth is set for this feed. Ignored if user is empty
|
||||
* @param string|null $user if given, basic auth is set for this feed
|
||||
* @param string|null $password if given, basic auth is set for this feed. Ignored if user is empty
|
||||
*
|
||||
* @throws FetcherException if the fetcher encounters a problem
|
||||
* @return array an array containing the new feed and its items, first
|
||||
* element being the Feed and second element being an array of Items
|
||||
* @throws FetcherException if the fetcher encounters a problem
|
||||
*/
|
||||
public function fetch(
|
||||
$url,
|
||||
$getFavicon = true,
|
||||
$lastModified = null,
|
||||
$etag = null,
|
||||
$fullTextEnabled = false,
|
||||
$basicAuthUser = null,
|
||||
$basicAuthPassword = null
|
||||
);
|
||||
public function fetch(string $url, bool $favicon, $lastModified, $user, $password): array;
|
||||
|
||||
/**
|
||||
* Can a fetcher handle a feed.
|
||||
|
|
@ -51,5 +40,5 @@ interface IFeedFetcher
|
|||
* @return boolean if the fetcher can handle the url. This fetcher will be
|
||||
* used exclusively to fetch the feed and the items of the page
|
||||
*/
|
||||
public function canHandle($url);
|
||||
public function canHandle($url): bool;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ class YoutubeFetcher implements IFeedFetcher
|
|||
/**
|
||||
* This fetcher handles all the remaining urls therefore always returns true
|
||||
*/
|
||||
public function canHandle($url)
|
||||
public function canHandle($url): bool
|
||||
{
|
||||
return $this->buildUrl($url) !== $url;
|
||||
}
|
||||
|
|
@ -48,39 +48,18 @@ class YoutubeFetcher implements IFeedFetcher
|
|||
/**
|
||||
* Fetch a feed from remote
|
||||
*
|
||||
* @param string $url remote url of the feed
|
||||
* @param boolean $getFavicon if the favicon should also be fetched, defaults to true
|
||||
* @param string $lastModified a last modified value from an http header defaults to false.
|
||||
* If lastModified matches the http header from the feed no results are fetched
|
||||
* @param string $etag an etag from an http header.
|
||||
* If lastModified matches the http header from the feed no results are fetched
|
||||
* @param bool $fullTextEnabled if true tells the fetcher to enhance the articles by fetching more content
|
||||
* @param string $basicAuthUser if given, basic auth is set for this feed
|
||||
* @param string $basicAuthPassword if given, basic auth is set for this feed. Ignored if user is empty
|
||||
*
|
||||
* @throws FetcherException if it fails
|
||||
* @return array an array containing the new feed and its items, first
|
||||
* element being the Feed and second element being an array of Items
|
||||
* @inheritdoc
|
||||
*/
|
||||
public function fetch(
|
||||
$url,
|
||||
$getFavicon = true,
|
||||
$lastModified = null,
|
||||
$etag = null,
|
||||
$fullTextEnabled = false,
|
||||
$basicAuthUser = null,
|
||||
$basicAuthPassword = null
|
||||
) {
|
||||
public function fetch(string $url, bool $favicon, $lastModified, $user, $password): array
|
||||
{
|
||||
$transformedUrl = $this->buildUrl($url);
|
||||
|
||||
$result = $this->feedFetcher->fetch(
|
||||
$transformedUrl,
|
||||
$getFavicon,
|
||||
$favicon,
|
||||
$lastModified,
|
||||
$etag,
|
||||
$fullTextEnabled,
|
||||
$basicAuthUser,
|
||||
$basicAuthPassword
|
||||
$user,
|
||||
$password
|
||||
);
|
||||
|
||||
// reset feed url so we know the correct added url for the feed
|
||||
|
|
|
|||
|
|
@ -1,117 +0,0 @@
|
|||
<?php
|
||||
/**
|
||||
* Nextcloud - News
|
||||
*
|
||||
* This file is licensed under the Affero General Public License version 3 or
|
||||
* later. See the COPYING file.
|
||||
*
|
||||
* @author Robin Appelman <robin@icewind.nl>
|
||||
*/
|
||||
|
||||
namespace OCA\News\PostProcessor;
|
||||
|
||||
use GuzzleHttp\Cookie\CookieJar;
|
||||
use OCP\Http\Client\IClientService;
|
||||
use PicoFeed\Parser\Feed;
|
||||
use PicoFeed\Parser\Item;
|
||||
use PicoFeed\Processor\ItemProcessorInterface;
|
||||
use PicoFeed\Scraper\RuleParser;
|
||||
|
||||
class LWNProcessor implements ItemProcessorInterface
|
||||
{
|
||||
private $user;
|
||||
|
||||
private $password;
|
||||
|
||||
private $clientService;
|
||||
|
||||
private $cookieJar;
|
||||
|
||||
/**
|
||||
* @param $user
|
||||
* @param $password
|
||||
*/
|
||||
public function __construct($user, $password, IClientService $clientService)
|
||||
{
|
||||
$this->user = $user;
|
||||
$this->password = $password;
|
||||
$this->clientService = $clientService;
|
||||
$this->cookieJar = new CookieJar();
|
||||
}
|
||||
|
||||
private function login()
|
||||
{
|
||||
if ($this->cookieJar->count() > 0) {
|
||||
return true;
|
||||
}
|
||||
if (!$this->user || !$this->password) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$client = $this->clientService->newClient();
|
||||
$response = $client->post(
|
||||
'https://lwn.net/login',
|
||||
[
|
||||
'cookies' => $this->cookieJar,
|
||||
'body' => [
|
||||
'Username' => $this->user,
|
||||
'Password' => $this->password,
|
||||
'target' => '/'
|
||||
]
|
||||
]
|
||||
);
|
||||
return ($response->getStatusCode() === 200 && $this->cookieJar->count() > 0);
|
||||
}
|
||||
|
||||
private function getBody($url)
|
||||
{
|
||||
$client = $this->clientService->newClient();
|
||||
$response = $client->get(
|
||||
$url,
|
||||
[
|
||||
'cookies' => $this->cookieJar
|
||||
]
|
||||
);
|
||||
$parser = new RuleParser(
|
||||
$response->getBody(),
|
||||
[
|
||||
'body' => array(
|
||||
'//div[@class="ArticleText"]',
|
||||
),
|
||||
'strip' => array(
|
||||
'//div[@class="FeatureByline"]'
|
||||
)
|
||||
]
|
||||
);
|
||||
$articleBody = $parser->execute();
|
||||
// make all links absolute
|
||||
return str_replace('href="/', 'href="https://lwn.net/', $articleBody);
|
||||
}
|
||||
|
||||
private function canHandle($url)
|
||||
{
|
||||
$regex = '%(?:https?://|//)?(?:www.)?lwn.net%';
|
||||
|
||||
return (bool)preg_match($regex, $url);
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute Item Processor
|
||||
*
|
||||
* @access public
|
||||
* @param Feed $feed
|
||||
* @param Item $item
|
||||
* @return bool
|
||||
*/
|
||||
public function execute(Feed $feed, Item $item)
|
||||
{
|
||||
if ($this->canHandle($item->getUrl())) {
|
||||
$loggedIn = $this->login();
|
||||
|
||||
$item->setUrl(str_replace('/rss', '', $item->getUrl()));
|
||||
if ($loggedIn) {
|
||||
$item->setContent($this->getBody($item->getUrl()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -58,8 +58,8 @@ class FeedService extends Service
|
|||
$this->logger = $logger;
|
||||
$this->l10n = $l10n;
|
||||
$this->timeFactory = $timeFactory;
|
||||
$this->autoPurgeMinimumInterval =
|
||||
$config->getAutoPurgeMinimumInterval();
|
||||
$this->autoPurgeMinimumInterval = $config->getAutoPurgeMinimumInterval(
|
||||
);
|
||||
$this->purifier = $purifier;
|
||||
$this->feedMapper = $feedMapper;
|
||||
$this->loggerParams = $LoggerParameters;
|
||||
|
|
@ -69,6 +69,7 @@ class FeedService extends Service
|
|||
* Finds all feeds of a user
|
||||
*
|
||||
* @param string $userId the name of the user
|
||||
*
|
||||
* @return Feed[]
|
||||
*/
|
||||
public function findAll($userId)
|
||||
|
|
@ -96,57 +97,43 @@ class FeedService extends Service
|
|||
* folder
|
||||
* @param string $userId for which user the feed should be created
|
||||
* @param string $title if given, this is used for the opml feed title
|
||||
* @param string $basicAuthUser if given, basic auth is set for this feed
|
||||
* @param string $basicAuthPassword if given, basic auth is set for this
|
||||
* @param string $user if given, basic auth is set for this feed
|
||||
* @param string $password if given, basic auth is set for this
|
||||
* feed. Ignored if user is null or an empty string
|
||||
*
|
||||
* @throws ServiceConflictException if the feed exists already
|
||||
* @throws ServiceNotFoundException if the url points to an invalid feed
|
||||
* @return Feed the newly created feed
|
||||
*/
|
||||
public function create(
|
||||
$feedUrl,
|
||||
$folderId,
|
||||
$userId,
|
||||
$title = null,
|
||||
$basicAuthUser = null,
|
||||
$basicAuthPassword = null
|
||||
) {
|
||||
public function create($feedUrl, $folderId, $userId, $title = null, $user = null, $password = null)
|
||||
{
|
||||
// first try if the feed exists already
|
||||
try {
|
||||
/**
|
||||
* @var Feed $feed
|
||||
* @var Feed $feed
|
||||
* @var Item[] $items
|
||||
*/
|
||||
list($feed, $items) = $this->feedFetcher->fetch(
|
||||
$feedUrl,
|
||||
true,
|
||||
null,
|
||||
null,
|
||||
false,
|
||||
$basicAuthUser,
|
||||
$basicAuthPassword
|
||||
);
|
||||
|
||||
list($feed, $items) = $this->feedFetcher->fetch($feedUrl, true, null, $user, $password);
|
||||
// try again if feed exists depending on the reported link
|
||||
try {
|
||||
$this->feedMapper->findByUrlHash($feed->getUrlHash(), $userId);
|
||||
$hash = $feed->getUrlHash();
|
||||
$this->feedMapper->findByUrlHash($hash, $userId);
|
||||
throw new ServiceConflictException(
|
||||
$this->l10n->t('Can not add feed: Exists already')
|
||||
);
|
||||
|
||||
// If no matching feed was found everything was ok
|
||||
} catch (DoesNotExistException $ex) {
|
||||
// If no matching feed was found everything was ok
|
||||
}
|
||||
|
||||
// insert feed
|
||||
$itemCount = count($items);
|
||||
$feed->setBasicAuthUser($basicAuthUser);
|
||||
$feed->setBasicAuthPassword($basicAuthPassword);
|
||||
$feed->setBasicAuthUser($user);
|
||||
$feed->setBasicAuthPassword($password);
|
||||
$feed->setFolderId($folderId);
|
||||
$feed->setUserId($userId);
|
||||
$feed->setArticlesPerUpdate($itemCount);
|
||||
|
||||
if ($title !== null && $title !== '') {
|
||||
if (!empty($title)) {
|
||||
$feed->setTitle($title);
|
||||
}
|
||||
|
||||
|
|
@ -155,8 +142,7 @@ class FeedService extends Service
|
|||
// insert items in reverse order because the first one is usually
|
||||
// the newest item
|
||||
$unreadCount = 0;
|
||||
for ($i = $itemCount - 1; $i >= 0; $i--) {
|
||||
$item = $items[$i];
|
||||
foreach (array_reverse($items) as $item) {
|
||||
$item->setFeedId($feed->getId());
|
||||
|
||||
// check if item exists (guidhash is the same)
|
||||
|
|
@ -213,6 +199,7 @@ class FeedService extends Service
|
|||
* @param int $feedId the id of the feed that should be updated
|
||||
* @param string $userId the id of the user
|
||||
* @param bool $forceUpdate update even if the article exists already
|
||||
*
|
||||
* @throws ServiceNotFoundException if the feed does not exist
|
||||
* @return Feed the updated feed entity
|
||||
*/
|
||||
|
|
@ -237,8 +224,6 @@ class FeedService extends Service
|
|||
$location,
|
||||
false,
|
||||
$existingFeed->getHttpLastModified(),
|
||||
$existingFeed->getHttpEtag(),
|
||||
$existingFeed->getFullTextEnabled(),
|
||||
$existingFeed->getBasicAuthUser(),
|
||||
$existingFeed->getBasicAuthPassword()
|
||||
);
|
||||
|
|
@ -332,6 +317,7 @@ class FeedService extends Service
|
|||
*
|
||||
* @param array $json the array with json
|
||||
* @param string $userId the username
|
||||
*
|
||||
* @return Feed if one had to be created for nonexistent feeds
|
||||
*/
|
||||
public function importArticles($json, $userId)
|
||||
|
|
@ -406,6 +392,7 @@ class FeedService extends Service
|
|||
*
|
||||
* @param int $feedId the id of the feed that should be deleted
|
||||
* @param string $userId the name of the user for security reasons
|
||||
*
|
||||
* @throws ServiceNotFoundException when feed does not exist
|
||||
*/
|
||||
public function markDeleted($feedId, $userId)
|
||||
|
|
@ -421,6 +408,7 @@ class FeedService extends Service
|
|||
*
|
||||
* @param int $feedId the id of the feed that should be restored
|
||||
* @param string $userId the name of the user for security reasons
|
||||
*
|
||||
* @throws ServiceNotFoundException when feed does not exist
|
||||
*/
|
||||
public function unmarkDeleted($feedId, $userId)
|
||||
|
|
@ -471,13 +459,14 @@ class FeedService extends Service
|
|||
* @param $feedId
|
||||
* @param $userId
|
||||
* @param $diff an array containing the fields to update, e.g.:
|
||||
* [
|
||||
* 'ordering' => 1,
|
||||
* 'fullTextEnabled' => true,
|
||||
* 'pinned' => true,
|
||||
* 'updateMode' => 0,
|
||||
* 'title' => 'title'
|
||||
* ]
|
||||
* [
|
||||
* 'ordering' => 1,
|
||||
* 'fullTextEnabled' => true,
|
||||
* 'pinned' => true,
|
||||
* 'updateMode' => 0,
|
||||
* 'title' => 'title'
|
||||
* ]
|
||||
*
|
||||
* @throws ServiceNotFoundException if feed does not exist
|
||||
*/
|
||||
public function patch($feedId, $userId, $diff = [])
|
||||
|
|
|
|||
|
|
@ -1,42 +0,0 @@
|
|||
<?php
|
||||
/**
|
||||
* Nextcloud - News
|
||||
*
|
||||
* This file is licensed under the Affero General Public License version 3 or
|
||||
* later. See the COPYING file.
|
||||
*
|
||||
* @author Alessandro Cosentino <cosenal@gmail.com>
|
||||
* @author Bernhard Posselt <dev@bernhard-posselt.com>
|
||||
* @copyright 2012 Alessandro Cosentino
|
||||
* @copyright 2012-2014 Bernhard Posselt
|
||||
*/
|
||||
|
||||
|
||||
namespace OCA\News\Utility;
|
||||
|
||||
use \PicoFeed\Config\Config;
|
||||
use \PicoFeed\Client\Client;
|
||||
|
||||
class PicoFeedClientFactory
|
||||
{
|
||||
|
||||
private $config;
|
||||
|
||||
public function __construct(Config $config)
|
||||
{
|
||||
$this->config = $config;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Returns a new instance of an PicoFeed Http client
|
||||
*
|
||||
* @return \PicoFeed\Client instance
|
||||
*/
|
||||
public function build()
|
||||
{
|
||||
$client = Client::getInstance();
|
||||
$client->setConfig($this->config);
|
||||
return $client;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,40 +0,0 @@
|
|||
<?php
|
||||
/**
|
||||
* Nextcloud - News
|
||||
*
|
||||
* This file is licensed under the Affero General Public License version 3 or
|
||||
* later. See the COPYING file.
|
||||
*
|
||||
* @author Alessandro Cosentino <cosenal@gmail.com>
|
||||
* @author Bernhard Posselt <dev@bernhard-posselt.com>
|
||||
* @copyright 2012 Alessandro Cosentino
|
||||
* @copyright 2012-2014 Bernhard Posselt
|
||||
*/
|
||||
|
||||
|
||||
namespace OCA\News\Utility;
|
||||
|
||||
use \PicoFeed\Config\Config;
|
||||
use \PicoFeed\Reader\Favicon;
|
||||
|
||||
class PicoFeedFaviconFactory
|
||||
{
|
||||
|
||||
private $config;
|
||||
|
||||
public function __construct(Config $config)
|
||||
{
|
||||
$this->config = $config;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Returns a new instance of an PicoFeed Http client
|
||||
*
|
||||
* @return \PicoFeed\Favicon instance
|
||||
*/
|
||||
public function build()
|
||||
{
|
||||
return new Favicon($this->config);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,97 @@
|
|||
<?php
|
||||
/**
|
||||
* Nextcloud - News
|
||||
*
|
||||
* This file is licensed under the Affero General Public License version 3 or
|
||||
* later. See the COPYING file.
|
||||
*
|
||||
* @author Sean Molenaar <sean@seanmolenaar.eu>
|
||||
* @copyright 2018 Sean Molenaar
|
||||
*/
|
||||
|
||||
namespace OCA\News\Utility;
|
||||
|
||||
use \OCP\ILogger;
|
||||
|
||||
/**
|
||||
* This is a wrapper to make OC\Log conform to Psr\Log\LoggerInterface
|
||||
*
|
||||
* @package OCA\News\Utility
|
||||
*/
|
||||
class PsrLogger implements \Psr\Log\LoggerInterface
|
||||
{
|
||||
private $logger;
|
||||
private $appName;
|
||||
|
||||
/**
|
||||
* PsrLogger constructor.
|
||||
*
|
||||
* @param ILogger $logger The logger
|
||||
* @param string $appName Name of the app
|
||||
*/
|
||||
public function __construct(ILogger $logger, $appName)
|
||||
{
|
||||
$this->logger = $logger;
|
||||
$this->appName = $appName;
|
||||
}
|
||||
|
||||
public function logException($exception, array $context = [])
|
||||
{
|
||||
$context['app'] = $this->appName;
|
||||
$this->logger->logException($exception, $context);
|
||||
}
|
||||
|
||||
public function emergency($message, array $context = [])
|
||||
{
|
||||
$context['app'] = $this->appName;
|
||||
$this->logger->emergency($message, $context);
|
||||
}
|
||||
|
||||
public function alert($message, array $context = [])
|
||||
{
|
||||
$context['app'] = $this->appName;
|
||||
$this->logger->alert($message, $context);
|
||||
}
|
||||
|
||||
public function critical($message, array $context = [])
|
||||
{
|
||||
$context['app'] = $this->appName;
|
||||
$this->logger->critical($message, $context);
|
||||
}
|
||||
|
||||
public function error($message, array $context = [])
|
||||
{
|
||||
$context['app'] = $this->appName;
|
||||
$this->logger->error($message, $context);
|
||||
}
|
||||
|
||||
public function warning($message, array $context = [])
|
||||
{
|
||||
$context['app'] = $this->appName;
|
||||
$this->logger->warning($message, $context);
|
||||
}
|
||||
|
||||
public function notice($message, array $context = [])
|
||||
{
|
||||
$context['app'] = $this->appName;
|
||||
$this->logger->notice($message, $context);
|
||||
}
|
||||
|
||||
public function info($message, array $context = [])
|
||||
{
|
||||
$context['app'] = $this->appName;
|
||||
$this->logger->info($message, $context);
|
||||
}
|
||||
|
||||
public function debug($message, array $context = [])
|
||||
{
|
||||
$context['app'] = $this->appName;
|
||||
$this->logger->debug($message, $context);
|
||||
}
|
||||
|
||||
public function log($level, $message, array $context = [])
|
||||
{
|
||||
$context['app'] = $this->appName;
|
||||
$this->logger->log($level, $message, $context);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
{
|
||||
"lockfileVersion": 1
|
||||
}
|
||||
|
|
@ -29,7 +29,7 @@ class ConfigTest extends TestCase
|
|||
|
||||
public function setUp()
|
||||
{
|
||||
$this->logger = $this->getMockBuilder(ILogger::class)
|
||||
$this->logger = $this->getMockBuilder('OCA\News\Utility\PsrLogger')
|
||||
->disableOriginalConstructor()
|
||||
->getMock();
|
||||
$this->fileSystem = $this->getMockBuilder(Folder::class)->getMock();
|
||||
|
|
|
|||
|
|
@ -13,13 +13,20 @@
|
|||
|
||||
namespace OCA\News\Tests\Unit\Fetcher;
|
||||
|
||||
use \OCA\News\Db\Item;
|
||||
use FeedIo\Feed\Item\Author;
|
||||
use FeedIo\Feed\Item\MediaInterface;
|
||||
use FeedIo\Feed\ItemInterface;
|
||||
use FeedIo\FeedInterface;
|
||||
use OC\L10N\L10N;
|
||||
use OCA\AdminAudit\Actions\Auth;
|
||||
use \OCA\News\Db\Feed;
|
||||
use \OCA\News\Db\Item;
|
||||
use OCA\News\Fetcher\FeedFetcher;
|
||||
use OCA\News\Utility\PicoFeedFaviconFactory;
|
||||
use OCA\News\Utility\PsrLogger;
|
||||
use OCA\News\Utility\Time;
|
||||
use OCP\Http\Client\IClientService;
|
||||
use OCP\IL10N;
|
||||
|
||||
use PHPUnit\Framework\TestCase;
|
||||
use PicoFeed\Client\Client;
|
||||
use PicoFeed\Parser\Parser;
|
||||
|
|
@ -27,121 +34,175 @@ use PicoFeed\Processor\ItemPostProcessor;
|
|||
use PicoFeed\Reader\Favicon;
|
||||
use PicoFeed\Reader\Reader;
|
||||
|
||||
|
||||
/**
|
||||
* Class FeedFetcherTest
|
||||
*
|
||||
* @package OCA\News\Tests\Unit\Fetcher
|
||||
*/
|
||||
class FeedFetcherTest extends TestCase
|
||||
{
|
||||
|
||||
/**
|
||||
* The class to test
|
||||
*
|
||||
* @var FeedFetcher
|
||||
*/
|
||||
private $fetcher;
|
||||
private $parser;
|
||||
|
||||
/**
|
||||
* Feed reader
|
||||
*
|
||||
* @var \FeedIo\FeedIo
|
||||
*/
|
||||
private $reader;
|
||||
private $client;
|
||||
private $faviconFetcher;
|
||||
private $parsedFeed;
|
||||
private $faviconFactory;
|
||||
|
||||
/**
|
||||
* Feed reader result
|
||||
*
|
||||
* @var \FeedIo\Reader\Result
|
||||
*/
|
||||
private $result;
|
||||
|
||||
/**
|
||||
* Feed reader result object
|
||||
*
|
||||
* @var \FeedIo\Adapter\ResponseInterface
|
||||
*/
|
||||
private $response;
|
||||
|
||||
/**
|
||||
* @var \Favicon\Favicon
|
||||
*/
|
||||
private $favicon;
|
||||
|
||||
/**
|
||||
* @var L10N
|
||||
*/
|
||||
private $l10n;
|
||||
private $url;
|
||||
|
||||
/**
|
||||
* @var ItemInterface
|
||||
*/
|
||||
private $item_mock;
|
||||
|
||||
/**
|
||||
* @var FeedInterface
|
||||
*/
|
||||
private $feed_mock;
|
||||
|
||||
/**
|
||||
* @var PsrLogger
|
||||
*/
|
||||
private $logger;
|
||||
|
||||
//metadata
|
||||
/**
|
||||
* @var integer
|
||||
*/
|
||||
private $time;
|
||||
private $item;
|
||||
private $content;
|
||||
|
||||
/**
|
||||
* @var string
|
||||
*/
|
||||
private $encoding;
|
||||
|
||||
/**
|
||||
* @var string
|
||||
*/
|
||||
private $url;
|
||||
|
||||
// items
|
||||
private $permalink;
|
||||
private $title;
|
||||
private $guid;
|
||||
private $guid_hash;
|
||||
private $pub;
|
||||
private $updated;
|
||||
private $body;
|
||||
/**
|
||||
* @var Author
|
||||
*/
|
||||
private $author;
|
||||
private $enclosureLink;
|
||||
private $enclosure;
|
||||
private $rtl;
|
||||
private $language;
|
||||
|
||||
// feed
|
||||
private $feedTitle;
|
||||
private $feedLink;
|
||||
private $feedImage;
|
||||
private $webFavicon;
|
||||
private $feed_title;
|
||||
private $feed_link;
|
||||
private $feed_image;
|
||||
private $web_favicon;
|
||||
private $modified;
|
||||
private $etag;
|
||||
private $location;
|
||||
private $feedLanguage;
|
||||
|
||||
protected function setUp()
|
||||
{
|
||||
$this->l10n = $this->getMockBuilder(IL10N::class)
|
||||
$this->l10n = $this->getMockBuilder(\OCP\IL10N::class)
|
||||
->disableOriginalConstructor()
|
||||
->getMock();
|
||||
$this->reader = $this->getMockBuilder(Reader::class)
|
||||
$this->reader = $this->getMockBuilder(\FeedIo\FeedIo::class)
|
||||
->disableOriginalConstructor()
|
||||
->getMock();
|
||||
$this->parser = $this->getMockBuilder(Parser::class)
|
||||
$this->favicon = $this->getMockBuilder(\Favicon\Favicon::class)
|
||||
->disableOriginalConstructor()
|
||||
->getMock();
|
||||
$this->client = $this->getMockBuilder(Client::class)
|
||||
$this->result = $this->getMockBuilder(\FeedIo\Reader\Result::class)
|
||||
->disableOriginalConstructor()
|
||||
->getMock();
|
||||
$this->parsedFeed = $this->getMockBuilder(\PicoFeed\Parser\Feed::class)
|
||||
$this->response = $this->getMockBuilder(\FeedIo\Adapter\ResponseInterface::class)
|
||||
->disableOriginalConstructor()
|
||||
->getMock();
|
||||
$this->item = $this->getMockBuilder(\PicoFeed\Parser\Item::class)
|
||||
|
||||
$this->item_mock = $this->getMockBuilder(\FeedIo\Feed\ItemInterface::class)
|
||||
->disableOriginalConstructor()
|
||||
->getMock();
|
||||
$this->faviconFetcher = $this->getMockBuilder(Favicon::class)
|
||||
->disableOriginalConstructor()
|
||||
->getMock();
|
||||
$this->faviconFactory = $this->getMockBuilder(PicoFeedFaviconFactory::class)
|
||||
|
||||
$this->feed_mock = $this->getMockBuilder(\FeedIo\FeedInterface::class)
|
||||
->disableOriginalConstructor()
|
||||
->getMock();
|
||||
|
||||
$this->time = 2323;
|
||||
$timeFactory = $this->getMockBuilder(Time::class)
|
||||
$timeFactory = $this->getMockBuilder(\OCA\News\Utility\Time::class)
|
||||
->disableOriginalConstructor()
|
||||
->getMock();
|
||||
$timeFactory->expects($this->any())
|
||||
->method('getTime')
|
||||
->will($this->returnValue($this->time));
|
||||
$postProcessor = $this->getMockBuilder(ItemPostProcessor::class)
|
||||
->getMock();
|
||||
$this->parser->expects($this->any())
|
||||
->method('getItemPostProcessor')
|
||||
->will($this->returnValue($postProcessor));
|
||||
$clientService = $this->getMockBuilder(IClientService::class)
|
||||
$this->logger = $this->getMockBuilder(PsrLogger::class)
|
||||
->disableOriginalConstructor()
|
||||
->getMock();
|
||||
$this->fetcher = new FeedFetcher(
|
||||
$this->reader,
|
||||
$this->faviconFactory,
|
||||
$this->favicon,
|
||||
$this->l10n,
|
||||
$timeFactory,
|
||||
$clientService
|
||||
$this->logger
|
||||
);
|
||||
$this->url = 'http://tests';
|
||||
$this->url = 'http://tests';
|
||||
|
||||
$this->permalink = 'http://permalink';
|
||||
$this->title = 'my&lt;' title';
|
||||
$this->guid = 'hey guid here';
|
||||
$this->body = 'let the bodies hit the floor <a href="test">test</a>';
|
||||
$this->body2 = 'let the bodies hit the floor ' .
|
||||
'<a target="_blank" href="test">test</a>';
|
||||
$this->pub = 23111;
|
||||
$this->updated = 23444;
|
||||
$this->author = '<boogieman';
|
||||
$this->enclosureLink = 'http://enclosure.you';
|
||||
$this->title = 'my&lt;' title';
|
||||
$this->guid = 'hey guid here';
|
||||
$this->guid_hash = 'df9a5f84e44bfe38cf44f6070d5b0250';
|
||||
$this->body = 'let the bodies hit the floor <a href="test">test</a>';
|
||||
$this->pub = 23111;
|
||||
$this->updated = 23444;
|
||||
$this->author = new Author();
|
||||
$this->author->setName('<boogieman');
|
||||
$this->enclosure = 'http://enclosure.you';
|
||||
|
||||
$this->feedTitle = '<a>&its a</a> title';
|
||||
$this->feedLink = 'http://goatse';
|
||||
$this->feedImage = '/an/image';
|
||||
$this->webFavicon = 'http://anon.google.com';
|
||||
$this->authorMail = 'doe@joes.com';
|
||||
$this->modified = 3;
|
||||
$this->etag = 'yo';
|
||||
$this->content = 'some content';
|
||||
$this->encoding = 'UTF-8';
|
||||
$this->language = 'de-DE';
|
||||
$this->feedLanguage = 'de-DE';
|
||||
$this->feed_title = '<a>&its a</a> title';
|
||||
$this->feed_link = 'http://tests';
|
||||
$this->feed_image = '/an/image';
|
||||
$this->web_favicon = 'http://anon.google.com';
|
||||
$this->modified = $this->getMockBuilder('\DateTime')->getMock();
|
||||
$this->modified->expects($this->any())
|
||||
->method('getTimestamp')
|
||||
->will($this->returnValue(3));
|
||||
$this->encoding = 'UTF-8';
|
||||
$this->language = 'de-DE';
|
||||
$this->rtl = false;
|
||||
}
|
||||
|
||||
|
||||
public function testCanHandle()
|
||||
{
|
||||
$url = 'google.de';
|
||||
|
|
@ -149,163 +210,26 @@ class FeedFetcherTest extends TestCase
|
|||
$this->assertTrue($this->fetcher->canHandle($url));
|
||||
}
|
||||
|
||||
private function setUpReader($url='', $modified=true, $noParser=false)
|
||||
{
|
||||
$this->reader->expects($this->once())
|
||||
->method('discover')
|
||||
->with($this->equalTo($url))
|
||||
->will($this->returnValue($this->client));
|
||||
$this->client->expects($this->once())
|
||||
->method('isModified')
|
||||
->will($this->returnValue($modified));
|
||||
|
||||
if (!$modified) {
|
||||
$this->reader->expects($this->never())
|
||||
->method('getParser');
|
||||
} else {
|
||||
$this->client->expects($this->once())
|
||||
->method('getLastModified')
|
||||
->will($this->returnValue($this->modified));
|
||||
$this->client->expects($this->once())
|
||||
->method('getEtag')
|
||||
->will($this->returnValue($this->etag));
|
||||
$this->client->expects($this->once())
|
||||
->method('getUrl')
|
||||
->will($this->returnValue($this->location));
|
||||
$this->client->expects($this->once())
|
||||
->method('getContent')
|
||||
->will($this->returnValue($this->content));
|
||||
$this->client->expects($this->once())
|
||||
->method('getEncoding')
|
||||
->will($this->returnValue($this->encoding));
|
||||
|
||||
if ($noParser) {
|
||||
$this->reader->expects($this->once())
|
||||
->method('getParser')
|
||||
->will(
|
||||
$this->throwException(
|
||||
new \PicoFeed\Reader\SubscriptionNotFoundException()
|
||||
)
|
||||
);
|
||||
} else {
|
||||
$this->reader->expects($this->once())
|
||||
->method('getParser')
|
||||
->with(
|
||||
$this->equalTo($this->location),
|
||||
$this->equalTo($this->content),
|
||||
$this->equalTo($this->encoding)
|
||||
)
|
||||
->will($this->returnValue($this->parser));
|
||||
}
|
||||
|
||||
$this->parser->expects($this->once())
|
||||
->method('execute')
|
||||
->will($this->returnValue($this->parsedFeed));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
private function expectFeed($method, $return, $count = 1)
|
||||
{
|
||||
$this->parsedFeed->expects($this->exactly($count))
|
||||
->method($method)
|
||||
->will($this->returnValue($return));
|
||||
}
|
||||
|
||||
private function expectItem($method, $return, $count = 1)
|
||||
{
|
||||
$this->item->expects($this->exactly($count))
|
||||
->method($method)
|
||||
->will($this->returnValue($return));
|
||||
}
|
||||
|
||||
|
||||
private function createItem($enclosureType=null)
|
||||
{
|
||||
$this->expectItem('getUrl', $this->permalink);
|
||||
$this->expectItem('getTitle', $this->title);
|
||||
$this->expectItem('getId', $this->guid);
|
||||
$this->expectItem('getContent', $this->body);
|
||||
|
||||
$item = new Item();
|
||||
|
||||
date_default_timezone_set('America/Los_Angeles');
|
||||
|
||||
$pubdate = \Datetime::createFromFormat('U', $this->pub);
|
||||
$this->expectItem('getPublishedDate', $pubdate);
|
||||
$item->setPubDate($this->pub);
|
||||
|
||||
$update = \Datetime::createFromFormat('U', $this->updated);
|
||||
$this->expectItem('getUpdatedDate', $update);
|
||||
$item->setUpdatedDate($this->updated);
|
||||
|
||||
$item->setStatus(0);
|
||||
$item->setUnread(true);
|
||||
$item->setUrl($this->permalink);
|
||||
$item->setTitle('my<\' title');
|
||||
$item->setGuid($this->guid);
|
||||
$item->setGuidHash($this->guid);
|
||||
$item->setBody($this->body);
|
||||
$item->setRtl(false);
|
||||
|
||||
$this->expectItem('getAuthor', $this->author);
|
||||
$item->setAuthor(html_entity_decode($this->author));
|
||||
|
||||
if($enclosureType === 'audio/ogg' || $enclosureType === 'video/ogg') {
|
||||
$this->expectItem('getEnclosureUrl', $this->enclosureLink);
|
||||
$this->expectItem('getEnclosureType', $enclosureType);
|
||||
|
||||
$item->setEnclosureMime($enclosureType);
|
||||
$item->setEnclosureLink($this->enclosureLink);
|
||||
}
|
||||
$item->generateSearchIndex();
|
||||
|
||||
return $item;
|
||||
}
|
||||
|
||||
|
||||
private function createFeed($hasFavicon=false)
|
||||
{
|
||||
$this->expectFeed('getTitle', $this->feedTitle);
|
||||
$this->expectFeed('getSiteUrl', $this->feedLink);
|
||||
|
||||
$feed = new Feed();
|
||||
$feed->setTitle('&its a title');
|
||||
$feed->setUrl($this->url);
|
||||
$feed->setLink($this->feedLink);
|
||||
$feed->setAdded($this->time);
|
||||
$feed->setHttpLastModified($this->modified);
|
||||
$feed->setHttpEtag($this->etag);
|
||||
$feed->setLocation($this->location);
|
||||
|
||||
if($hasFavicon) {
|
||||
$this->faviconFactory->expects($this->once())
|
||||
->method('build')
|
||||
->will($this->returnValue($this->faviconFetcher));
|
||||
$this->faviconFetcher->expects($this->once())
|
||||
->method('find')
|
||||
->with($this->equalTo($this->feedLink))
|
||||
->will($this->returnValue($this->webFavicon));
|
||||
$feed->setFaviconLink($this->webFavicon);
|
||||
}
|
||||
|
||||
return $feed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Test if empty is logged when the feed remain the same.
|
||||
*/
|
||||
public function testNoFetchIfNotModified()
|
||||
{
|
||||
$this->setUpReader($this->url, false);;
|
||||
$result = $this->fetcher->fetch($this->url, false);
|
||||
$this->__setUpReader($this->url, false);
|
||||
$this->logger->expects($this->once())
|
||||
->method('debug')
|
||||
->with('Feed {url} was not modified since last fetch. old: {old}, new: {new}');
|
||||
$result = $this->fetcher->fetch($this->url, false, null, null, null);
|
||||
$this->assertSame([null, []], $result);
|
||||
}
|
||||
|
||||
public function testFetch()
|
||||
{
|
||||
$this->setUpReader($this->url);
|
||||
$item = $this->createItem();
|
||||
$feed = $this->createFeed();
|
||||
$this->expectFeed('getItems', [$this->item]);
|
||||
$result = $this->fetcher->fetch($this->url, false);
|
||||
$this->__setUpReader($this->url);
|
||||
$item = $this->_createItem();
|
||||
$feed = $this->_createFeed();
|
||||
$this->_mockIterator($this->feed_mock, [$this->item_mock]);
|
||||
$result = $this->fetcher->fetch($this->url, false, null, null, null);
|
||||
|
||||
$this->assertEquals([$feed, [$item]], $result);
|
||||
}
|
||||
|
|
@ -313,11 +237,11 @@ class FeedFetcherTest extends TestCase
|
|||
|
||||
public function testAudioEnclosure()
|
||||
{
|
||||
$this->setUpReader($this->url);
|
||||
$item = $this->createItem('audio/ogg');
|
||||
$feed = $this->createFeed();
|
||||
$this->expectFeed('getItems', [$this->item]);
|
||||
$result = $this->fetcher->fetch($this->url, false);
|
||||
$this->__setUpReader($this->url);
|
||||
$item = $this->_createItem('audio/ogg');
|
||||
$feed = $this->_createFeed();
|
||||
$this->_mockIterator($this->feed_mock, [$this->item_mock]);
|
||||
$result = $this->fetcher->fetch($this->url, false, null, null, null);
|
||||
|
||||
$this->assertEquals([$feed, [$item]], $result);
|
||||
}
|
||||
|
|
@ -325,105 +249,284 @@ class FeedFetcherTest extends TestCase
|
|||
|
||||
public function testVideoEnclosure()
|
||||
{
|
||||
$this->setUpReader($this->url);
|
||||
$item = $this->createItem('video/ogg');
|
||||
$feed = $this->createFeed();
|
||||
$this->expectFeed('getItems', [$this->item]);
|
||||
$result = $this->fetcher->fetch($this->url, false);
|
||||
$this->__setUpReader($this->url);
|
||||
$item = $this->_createItem('video/ogg');
|
||||
$feed = $this->_createFeed();
|
||||
$this->_mockIterator($this->feed_mock, [$this->item_mock]);
|
||||
$result = $this->fetcher->fetch($this->url, false, null, null, null);
|
||||
|
||||
$this->assertEquals([$feed, [$item]], $result);
|
||||
}
|
||||
|
||||
|
||||
|
||||
public function testFavicon()
|
||||
{
|
||||
$this->setUpReader($this->url);
|
||||
$this->__setUpReader($this->url);
|
||||
|
||||
$feed = $this->createFeed(true);
|
||||
$item = $this->createItem();
|
||||
$this->expectFeed('getItems', [$this->item]);
|
||||
$result = $this->fetcher->fetch($this->url);
|
||||
$feed = $this->_createFeed('de-DE', true);
|
||||
$item = $this->_createItem();
|
||||
$this->_mockIterator($this->feed_mock, [$this->item_mock]);
|
||||
$result = $this->fetcher->fetch($this->url, true, null, null, null);
|
||||
|
||||
$this->assertEquals([$feed, [$item]], $result);
|
||||
}
|
||||
|
||||
public function testFullText()
|
||||
{
|
||||
$this->setUpReader($this->url);
|
||||
|
||||
$feed = $this->createFeed();
|
||||
$item = $this->createItem();
|
||||
$this->parser->expects($this->once())
|
||||
->method('enableContentGrabber');
|
||||
$this->expectFeed('getItems', [$this->item]);
|
||||
$this->fetcher->fetch($this->url, false, null, null, true);
|
||||
}
|
||||
|
||||
|
||||
public function testNoFavicon()
|
||||
{
|
||||
$this->setUpReader($this->url);
|
||||
$this->__setUpReader($this->url);
|
||||
|
||||
$feed = $this->createFeed(false);
|
||||
$feed = $this->_createFeed(false);
|
||||
|
||||
$this->faviconFetcher->expects($this->never())
|
||||
->method('find');
|
||||
$this->favicon->expects($this->never())
|
||||
->method('get');
|
||||
|
||||
$item = $this->createItem();
|
||||
$this->expectFeed('getItems', [$this->item]);
|
||||
$result = $this->fetcher->fetch($this->url, false);
|
||||
$item = $this->_createItem();
|
||||
$this->_mockIterator($this->feed_mock, [$this->item_mock]);
|
||||
$result = $this->fetcher->fetch($this->url, false, null, null, null);
|
||||
|
||||
$this->assertEquals([$feed, [$item]], $result);
|
||||
}
|
||||
|
||||
|
||||
public function testRtl()
|
||||
{
|
||||
$this->setUpReader($this->url);
|
||||
$this->expectFeed('getLanguage', 'he-IL');
|
||||
$this->expectItem('getLanguage', '');
|
||||
$feed = $this->createFeed();
|
||||
$item = $this->createItem(null);
|
||||
$this->expectFeed('getItems', [$this->item]);
|
||||
list($feed, $items) = $this->fetcher->fetch(
|
||||
$this->url, false, null,
|
||||
null, true
|
||||
);
|
||||
$this->__setUpReader($this->url);
|
||||
$this->_createFeed('he-IL');
|
||||
$this->_createItem();
|
||||
$this->_mockIterator($this->feed_mock, [$this->item_mock]);
|
||||
list($feed, $items) = $this->fetcher->fetch($this->url, false, null, null, null);
|
||||
$this->assertTrue($items[0]->getRtl());
|
||||
}
|
||||
|
||||
|
||||
public function testRtlItemPrecedence()
|
||||
public function testRssPubDate()
|
||||
{
|
||||
$this->setUpReader($this->url);
|
||||
$this->expectFeed('getLanguage', 'de-DE');
|
||||
$this->expectItem('getLanguage', 'he-IL');
|
||||
$this->__setUpReader($this->url);
|
||||
$this->_createFeed('he-IL');
|
||||
$this->_createItem();
|
||||
|
||||
$feed = $this->createFeed();
|
||||
$item = $this->createItem(null);
|
||||
$this->expectFeed('getItems', [$this->item]);
|
||||
list($feed, $items) = $this->fetcher->fetch(
|
||||
$this->url, false, null,
|
||||
null, true
|
||||
);
|
||||
$this->assertTrue($items[0]->getRtl());
|
||||
$this->item_mock->expects($this->exactly(2))
|
||||
->method('getValue')
|
||||
->will($this->returnValueMap([
|
||||
['pubDate', '2018-03-27T19:50:29Z'],
|
||||
['published', NULL],
|
||||
]));
|
||||
|
||||
|
||||
$this->_mockIterator($this->feed_mock, [$this->item_mock]);
|
||||
list($feed, $items) = $this->fetcher->fetch($this->url, false, null, null, null);
|
||||
$this->assertSame($items[0]->getPubDate(), 1522180229);
|
||||
}
|
||||
|
||||
public function testNegativeRtlItemPrecedence()
|
||||
public function testAtomPubDate()
|
||||
{
|
||||
$this->setUpReader($this->url);
|
||||
$this->expectFeed('getLanguage', 'he-IL');
|
||||
$this->expectItem('getLanguage', 'de-DE');
|
||||
$this->__setUpReader($this->url);
|
||||
$this->_createFeed('he-IL');
|
||||
$this->_createItem();
|
||||
|
||||
$feed = $this->createFeed();
|
||||
$item = $this->createItem(null);
|
||||
$this->expectFeed('getItems', [$this->item]);
|
||||
list($feed, $items) = $this->fetcher->fetch(
|
||||
$this->url, false, null,
|
||||
null, true
|
||||
);
|
||||
$this->assertFalse($items[0]->getRtl());
|
||||
$this->item_mock->expects($this->exactly(3))
|
||||
->method('getValue')
|
||||
->will($this->returnValueMap([
|
||||
['pubDate', NULL],
|
||||
['published', '2018-02-27T19:50:29Z'],
|
||||
]));
|
||||
|
||||
|
||||
$this->_mockIterator($this->feed_mock, [$this->item_mock]);
|
||||
list($feed, $items) = $this->fetcher->fetch($this->url, false, null, null, null);
|
||||
$this->assertSame($items[0]->getPubDate(), 1519761029);
|
||||
}
|
||||
|
||||
/**
|
||||
* Mock an iteration option on an existing mock
|
||||
*
|
||||
* @param object $iteratorMock The mock to enhance
|
||||
* @param array $items The items to make available
|
||||
*
|
||||
* @return mixed
|
||||
*/
|
||||
private function _mockIterator($iteratorMock, array $items)
|
||||
{
|
||||
$iteratorData = new \stdClass();
|
||||
$iteratorData->array = $items;
|
||||
$iteratorData->position = 0;
|
||||
|
||||
$iteratorMock->expects($this->any())
|
||||
->method('rewind')
|
||||
->will(
|
||||
$this->returnCallback(
|
||||
function () use ($iteratorData) {
|
||||
$iteratorData->position = 0;
|
||||
}
|
||||
)
|
||||
);
|
||||
|
||||
$iteratorMock->expects($this->any())
|
||||
->method('current')
|
||||
->will(
|
||||
$this->returnCallback(
|
||||
function () use ($iteratorData) {
|
||||
return $iteratorData->array[$iteratorData->position];
|
||||
}
|
||||
)
|
||||
);
|
||||
|
||||
$iteratorMock->expects($this->any())
|
||||
->method('key')
|
||||
->will(
|
||||
$this->returnCallback(
|
||||
function () use ($iteratorData) {
|
||||
return $iteratorData->position;
|
||||
}
|
||||
)
|
||||
);
|
||||
|
||||
$iteratorMock->expects($this->any())
|
||||
->method('next')
|
||||
->will(
|
||||
$this->returnCallback(
|
||||
function () use ($iteratorData) {
|
||||
$iteratorData->position++;
|
||||
}
|
||||
)
|
||||
);
|
||||
|
||||
$iteratorMock->expects($this->any())
|
||||
->method('valid')
|
||||
->will(
|
||||
$this->returnCallback(
|
||||
function () use ($iteratorData) {
|
||||
return isset($iteratorData->array[$iteratorData->position]);
|
||||
}
|
||||
)
|
||||
);
|
||||
|
||||
$iteratorMock->expects($this->any())
|
||||
->method('count')
|
||||
->will(
|
||||
$this->returnCallback(
|
||||
function () use ($iteratorData) {
|
||||
return sizeof($iteratorData->array);
|
||||
}
|
||||
)
|
||||
);
|
||||
|
||||
return $iteratorMock;
|
||||
}
|
||||
|
||||
private function __setUpReader($url = '', $modified = true)
|
||||
{
|
||||
$this->reader->expects($this->once())
|
||||
->method('read')
|
||||
->with($this->equalTo($url))
|
||||
->will($this->returnValue($this->result));
|
||||
$this->result->expects($this->once())
|
||||
->method('getResponse')
|
||||
->will($this->returnValue($this->response));
|
||||
$this->response->expects($this->once())
|
||||
->method('isModified')
|
||||
->will($this->returnValue($modified));
|
||||
$this->location = $url;
|
||||
|
||||
if (!$modified) {
|
||||
$this->result->expects($this->never())
|
||||
->method('getUrl');
|
||||
} else {
|
||||
$this->result->expects($this->once())
|
||||
->method('getUrl')
|
||||
->will($this->returnValue($this->location));
|
||||
$this->result->expects($this->once())
|
||||
->method('getFeed')
|
||||
->will($this->returnValue($this->feed_mock));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private function _expectFeed($method, $return, $count = 1)
|
||||
{
|
||||
$this->feed_mock->expects($this->exactly($count))
|
||||
->method($method)
|
||||
->will($this->returnValue($return));
|
||||
}
|
||||
|
||||
private function _expectItem($method, $return, $count = 1)
|
||||
{
|
||||
$this->item_mock->expects($this->exactly($count))
|
||||
->method($method)
|
||||
->will($this->returnValue($return));
|
||||
}
|
||||
|
||||
|
||||
private function _createItem($enclosureType=null)
|
||||
{
|
||||
$this->_expectItem('getLink', $this->permalink);
|
||||
$this->_expectItem('getTitle', $this->title);
|
||||
$this->_expectItem('getPublicId', $this->guid);
|
||||
$this->_expectItem('getDescription', $this->body);
|
||||
$this->_expectItem('getLastModified', $this->modified);
|
||||
$this->_expectItem('getAuthor', $this->author);
|
||||
|
||||
$item = new Item();
|
||||
|
||||
$item->setStatus(0);
|
||||
$item->setUnread(true);
|
||||
$item->setUrl($this->permalink);
|
||||
$item->setTitle('my<\' title');
|
||||
$item->setGuid($this->guid);
|
||||
$item->setGuidHash($this->guid_hash);
|
||||
$item->setBody($this->body);
|
||||
$item->setRtl(false);
|
||||
$item->setLastModified(3);
|
||||
$item->setPubDate(3);
|
||||
$item->setAuthor(html_entity_decode($this->author->getName()));
|
||||
|
||||
if ($enclosureType === 'audio/ogg' || $enclosureType === 'video/ogg') {
|
||||
$media = $this->getMockbuilder(MediaInterface::class)->getMock();
|
||||
$media->expects($this->once())
|
||||
->method('getType')
|
||||
->will($this->returnValue('sounds'));
|
||||
$media2 = $this->getMockbuilder(MediaInterface::class)->getMock();
|
||||
$media2->expects($this->exactly(2))
|
||||
->method('getType')
|
||||
->will($this->returnValue($enclosureType));
|
||||
$media2->expects($this->once())
|
||||
->method('getUrl')
|
||||
->will($this->returnValue($this->enclosure));
|
||||
$this->_expectItem('hasMedia', true);
|
||||
$this->_expectItem('getMedias', [$media, $media2]);
|
||||
|
||||
$item->setEnclosureMime($enclosureType);
|
||||
$item->setEnclosureLink($this->enclosure);
|
||||
}
|
||||
$item->generateSearchIndex();
|
||||
|
||||
return $item;
|
||||
}
|
||||
|
||||
|
||||
private function _createFeed($lang='de-DE', $favicon=false)
|
||||
{
|
||||
$this->_expectFeed('getTitle', $this->feed_title, 2);
|
||||
$this->_expectFeed('getLink', $this->feed_link);
|
||||
$this->_expectFeed('getLastModified', $this->modified);
|
||||
$this->_expectFeed('getLanguage', $lang);
|
||||
|
||||
$feed = new Feed();
|
||||
|
||||
$feed->setTitle('&its a title');
|
||||
$feed->setLink($this->feed_link);
|
||||
$feed->setLocation($this->location);
|
||||
$feed->setUrl($this->url);
|
||||
$feed->setLastModified(3);
|
||||
$feed->setAdded($this->time);
|
||||
if ($favicon) {
|
||||
$feed->setFaviconLink('http://anon.google.com');
|
||||
$this->favicon->expects($this->exactly(1))
|
||||
->method('get')
|
||||
->with($this->equalTo($this->feed_link))
|
||||
->will($this->returnValue($this->web_favicon));
|
||||
} else {
|
||||
$this->favicon->expects($this->never())
|
||||
->method('get');
|
||||
}
|
||||
|
||||
return $feed;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -118,7 +118,7 @@ class FetcherTest extends TestCase
|
|||
public function testMultipleFetchersOnlyOneShouldHandle()
|
||||
{
|
||||
$url = 'hi';
|
||||
$return = 'zeas';
|
||||
$return = [];
|
||||
$mockFetcher = $this->getMockBuilder(IFeedFetcher::class)
|
||||
->disableOriginalConstructor()
|
||||
->getMock();
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ namespace OCA\News\Tests\Unit\Fetcher;
|
|||
|
||||
use \OCA\News\Db\Feed;
|
||||
use OCA\News\Fetcher\FeedFetcher;
|
||||
use OCA\News\Fetcher\Fetcher;
|
||||
use OCA\News\Fetcher\YoutubeFetcher;
|
||||
|
||||
use PHPUnit\Framework\TestCase;
|
||||
|
|
@ -20,7 +21,18 @@ use PHPUnit\Framework\TestCase;
|
|||
class YoutubeFetcherTest extends TestCase
|
||||
{
|
||||
|
||||
/**
|
||||
* Mocked fetcher.
|
||||
*
|
||||
* @var Fetcher
|
||||
*/
|
||||
private $fetcher;
|
||||
|
||||
/**
|
||||
* Mocked Feed Fetcher.
|
||||
*
|
||||
* @var FeedFetcher
|
||||
*/
|
||||
private $feedFetcher;
|
||||
|
||||
public function setUp()
|
||||
|
|
@ -52,7 +64,8 @@ class YoutubeFetcherTest extends TestCase
|
|||
$transformedUrl = 'http://gdata.youtube.com/feeds/api/playlists/sobo3';
|
||||
$favicon = true;
|
||||
$modified = 3;
|
||||
$etag = 5;
|
||||
$user = 5;
|
||||
$password = 5;
|
||||
$feed = new Feed();
|
||||
$feed->setUrl('http://google.de');
|
||||
$result = [$feed, []];
|
||||
|
|
@ -63,10 +76,10 @@ class YoutubeFetcherTest extends TestCase
|
|||
$this->equalTo($transformedUrl),
|
||||
$this->equalTo($favicon),
|
||||
$this->equalTo($modified),
|
||||
$this->equalTo($etag)
|
||||
$this->equalTo($user)
|
||||
)
|
||||
->will($this->returnValue($result));
|
||||
$feed = $this->fetcher->fetch($url, $favicon, $modified, $etag);
|
||||
$feed = $this->fetcher->fetch($url, $favicon, $modified, $user, $password);
|
||||
|
||||
$this->assertEquals($url, $result[0]->getUrl());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -311,7 +311,8 @@ class FeedServiceTest extends TestCase
|
|||
$this->equalTo('http://test'),
|
||||
$this->equalTo(false),
|
||||
$this->equalTo(3),
|
||||
$this->equalTo(4)
|
||||
$this->equalTo(''),
|
||||
$this->equalTo('')
|
||||
)
|
||||
->will($this->returnValue($fetchReturn));
|
||||
$this->feedMapper->expects($this->at(1))
|
||||
|
|
@ -377,7 +378,8 @@ class FeedServiceTest extends TestCase
|
|||
$this->equalTo('http://test'),
|
||||
$this->equalTo(false),
|
||||
$this->equalTo(3),
|
||||
$this->equalTo(4)
|
||||
$this->equalTo(''),
|
||||
$this->equalTo('')
|
||||
)
|
||||
->will($this->returnValue($fetchReturn));
|
||||
$this->feedMapper->expects($this->at(1))
|
||||
|
|
@ -635,7 +637,6 @@ class FeedServiceTest extends TestCase
|
|||
$feed = new Feed();
|
||||
$feed->setId(3);
|
||||
$feed->setUrl('https://goo.com');
|
||||
$feed->setHttpEtag('abc');
|
||||
$feed->setHttpLastModified(123);
|
||||
$feed->setFullTextEnabled(true);
|
||||
|
||||
|
|
@ -654,9 +655,7 @@ class FeedServiceTest extends TestCase
|
|||
->with(
|
||||
$this->equalTo($feed->getUrl()),
|
||||
$this->equalTo(false),
|
||||
$this->equalTo($feed->getHttpLastModified()),
|
||||
$this->equalTo($feed->getHttpEtag()),
|
||||
$this->equalTo($feed->getFullTextEnabled())
|
||||
$this->equalTo($feed->getHttpLastModified())
|
||||
)
|
||||
->will($this->throwException($ex));
|
||||
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче