Skip to content

Commit

Permalink
v2.0.1
Browse files Browse the repository at this point in the history
  • Loading branch information
zalevsk1y committed Nov 10, 2023
1 parent e62ce09 commit ca93592
Show file tree
Hide file tree
Showing 68 changed files with 4,589 additions and 1,818 deletions.
18 changes: 17 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,13 @@ News-parser is a plugin for Wordpress that allows you to easily receive the full
* Visual content extractor
* Flexible template creation system to speed up parsing
* Ability to parse not only from RSS XML source but also from url
* WP_CLI support

### Future plans

* Parsing videos from other(than YouTube) sources
* Saving images to the Media Library
* Add AI support
* Have an idea? – Please, feel free to let me know. Let’s make News-Parser even better!

### Installing
Expand Down Expand Up @@ -88,7 +90,7 @@ Watch this short video to learn HOW TO PARSE SINGLE PAGE with news-parser plugin
[![Alt text](http://img.youtube.com/vi/Sbke_LF-TFA/0.jpg)](https://www.youtube.com/watch?v=Sbke_LF-TFA)


## Autopilot Function for Automatic RSS Parsing
### Autopilot Function for Automatic RSS Parsing

The autopilot function is now available to automatically parse posts from an RSS feed. Please note that the wp-cron (`https://developer.wordpress.org/plugins/cron/`) is used for scheduling the autopilot function, which triggers the task scheduler only when the website is visited. If you encounter issues with this function, you can add the following option to the wp-config.php file: `define('ALTERNATE_WP_CRON', true);`

Expand All @@ -110,6 +112,20 @@ To configure the autopilot settings, follow these steps:
Watch this short video to learn HOW TO USE AUTOPILOT with news-parser plugin:

[![Alt text](http://img.youtube.com/vi/Eu_5GR32nB0/0.jpg)](https://www.youtube.com/watch?v=Eu_5GR32nB0)

### WP-CLI Support

With the latest update, a new feature has been introduced that leverages wp-cli. This feature enables users to activate an autopilot function, allowing for automated parsing and saving of posts from RSS feeds. The autopilot function can now be accessed directly from the command-line interface, providing a convenient way to manage this process.

To utilize this functionality, you'll need to install wp-cli and execute the command `wp autopilot` in the command-line interface. Additionally, you'll need to specify the desired interval at which the autopilot function should be triggered by including the additional parameter `wp autopilot --interval=`. This allows you to customize the frequency of the autopilot function according to your specific needs.

By incorporating wp-cli and the new "wp autopilot" command, managing the automatic parsing and saving of posts from RSS feeds becomes more efficient and streamlined. This feature provides enhanced control and flexibility, empowering users to automate their post management tasks with ease.

Example:

`wp autopilot --interval=hourly`


## Dependencies

* php-simple-html-dom-parser https://github.com/sunra/php-simple-html-dom-parser
Expand Down
22 changes: 19 additions & 3 deletions bootstrap.php
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,14 @@ function news_parser_init(){

// Load script, style, and global variable configurations

//$scripts_config= include NEWS_PARSER_PLUGIN_DIR.'inc/Config/scripts-config.php';
//$styles_config= include NEWS_PARSER_PLUGIN_DIR.'inc/Config/styles-config.php';
//$scripts_config= include NEWS_PARSER_PLUGIN_DIR.'inc/Config/scripts-config-dev.php';
//$styles_config= include NEWS_PARSER_PLUGIN_DIR.'inc/Config/styles-config-dev.php';

$scripts_config= include NEWS_PARSER_PLUGIN_DIR.'inc/Config/scripts-config.php';
$styles_config= include NEWS_PARSER_PLUGIN_DIR.'inc/Config/styles-config.php';
$global_variables_config= include NEWS_PARSER_PLUGIN_DIR.'inc/Config/global-variables-config.php';

$scripts_translation_config= include NEWS_PARSER_PLUGIN_DIR.'inc/Config/scripts-translation-config.php';

$app=Core\App::start($container);

// Initialize the ScriptLoadingManager
Expand All @@ -30,6 +31,7 @@ function news_parser_init(){
$loading_manager->setScriptsConfig($scripts_config);
$loading_manager->setStylesConfig($styles_config);
$loading_manager->setGlobalVariablesConfig($global_variables_config);
$loading_manager->setScriptsTranslationConfig($scripts_translation_config);
$loading_manager->init();

// Set up modifiers middleware for html parser
Expand All @@ -54,5 +56,19 @@ function news_parser_init(){
$app->event->on('list:get',array(Controller\ListController::class,'get'));
$app->event->on('html:get',array(Controller\VisualConstructorController::class,'get'));
$app->event->on('post:create',array(Controller\PostControllerUserID::class,'create'));

// Add cron action

add_action(NEWS_PARSER_CRON_ACTION_PREFIX.'hourly',array($app->cronTaskController,'cronTaskCallback'));
add_action(NEWS_PARSER_CRON_ACTION_PREFIX.'twicedaily',array($app->cronTaskController,'cronTaskCallback'));
add_action(NEWS_PARSER_CRON_ACTION_PREFIX.'daily',array($app->cronTaskController,'cronTaskCallback'));
add_action(NEWS_PARSER_CRON_ACTION_PREFIX.'weekly',array($app->cronTaskController,'cronTaskCallback'));

// Add WP_CLI commands

if ( defined( 'WP_CLI' ) && WP_CLI ) {
$invoke_autopilot=new \NewsParserPlugin\CLI\InvokeAutopilot($app->cronTaskController);
\WP_CLI::add_command( 'autopilot', array($invoke_autopilot,'commandCallback'));
}

}
2 changes: 1 addition & 1 deletion composer.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"description": "Wordpress plugin allows you to easily and quickly receive text and images from the site and from rss feed.",

"require": {
"php": ">=7.2",
"php": ">=7.4",
"caophihung94/php-simple-html-dom-parser":"1.7.3",
"zalevsk1y/container-builder":"^1.0.2"
},
Expand Down
44 changes: 44 additions & 0 deletions inc/CLI/InvokeAutopilot.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
<?php

namespace NewsParserPlugin\CLI;
use \NewsParserPlugin\Controller\CronTaskController;

class InvokeAutopilot {
protected $cronTaskController;
protected $intervals=array('hourly','daily','weekly','monthly','yearly');
protected $intervalErrorMessage='The --interval option must be one of the following values:
* hourly
* daily
* weekly
* monthly
* yearly
For example:
autopilot --interval=hourly
autopilot --interval=daily
autopilot --interval=weekly
autopilot --interval=monthly
autopilot --interval=yearly
';
public function __construct(CronTaskController $cronTaskController) {
$this->cronTaskController = $cronTaskController;
}

public function commandCallback( $args,$assoc_args ) {
if(!array_key_exists('interval',$assoc_args)){
\WP_CLI::log( $this->intervalErrorMessage );
return ;
}
$interval=$assoc_args['interval'];
if (!$interval||!in_array($interval,$this->intervals)){
\WP_CLI::log( $intervalErrorMessage );
return ;
}
$parsed_posts=$this->cronTaskController->cronTaskCallback($interval);
\WP_CLI::log( sprintf( '%s posts were saved.', $parsed_posts) );

}
}

35 changes: 23 additions & 12 deletions inc/Config/global-variables-config.php
Original file line number Diff line number Diff line change
Expand Up @@ -30,41 +30,52 @@
return array(
'global'=>array(),
'shared'=> array(
array(
'script_name'=>NEWS_PARSER_PLUGIN_SLUG.'-google-analitics',
'position'=>'after',
'data'=>"
window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments);}
gtag('js', new Date());
gtag('config', 'G-ZPX4NQDFKG');
"
)
),
$menu_config->menu->subs[0]->menu_slug=>array(
array(
'script_name'=>'main-parser-rss-bundle',
'name'=>'newsParserSettings',
'value'=>$nonce
'position'=>'before',
'data'=>"window.newsParserSettings=".json_encode($nonce)
),
array(
'script_name'=>'main-parser-rss-bundle',
'name'=>'newsParserApiEndpoints',
'value'=>$rest_api_endpoints
'position'=>'before',
'data'=>"window.newsParserApiEndpoints=".json_encode($rest_api_endpoints)
)
),
$menu_config->menu->subs[1]->menu_slug=>array(
array(
'script_name'=>'main-parser-page-bundle',
'name'=>'newsParserSettings',
'value'=>$nonce
'position'=>'before',
'data'=>"window.newsParserSettings=".json_encode($nonce)
),
array(
'script_name'=>'main-parser-page-bundle',
'name'=>'newsParserApiEndpoints',
'value'=>$rest_api_endpoints
'position'=>'before',
'data'=>"window.newsParserApiEndpoints=".json_encode($rest_api_endpoints)
)
),
$menu_config->menu->subs[2]->menu_slug=>array(
array(
'script_name'=>'main-parser-autopilot-bundle',
'name'=>'newsParserSettings',
'value'=>$nonce
'position'=>'before',
'data'=>"window.newsParserSettings=".json_encode($nonce)
),
array(
'script_name'=>'main-parser-autopilot-bundle',
'name'=>'newsParserApiEndpoints',
'value'=>$rest_api_endpoints
'position'=>'before',
'data'=>"window.newsParserApiEndpoints=".json_encode($rest_api_endpoints)
)
),
$menu_config->menu->subs[3]->menu_slug=>array(
Expand Down
20 changes: 16 additions & 4 deletions inc/Config/scripts-config-dev.php
Original file line number Diff line number Diff line change
Expand Up @@ -9,16 +9,28 @@
return array(
'global'=>array(),
'shared'=> array(

NEWS_PARSER_PLUGIN_SLUG.'-google-analitics'=>array(
'path'=>'https://www.googletagmanager.com/gtag/js?id=G-ZPX4NQDFKG',
'depends_on'=>array()
)
),
$menu_config->menu->subs[0]->menu_slug=>array(
'main-parser-rss-bundle'=> NEWS_PARSER_PLUGIN_URL.'/public/js/parser_rss-'.NEWS_PARSER_PLUGIN_VERSION.'.bundle.js'
'main-parser-rss-bundle'=> array(
'path'=>NEWS_PARSER_PLUGIN_URL.'/public/js/parser_rss-'.NEWS_PARSER_PLUGIN_VERSION.'.bundle.js',
'depends_on'=>array('wp-i18n')
)
),
$menu_config->menu->subs[1]->menu_slug=>array(
'main-parser-page-bundle'=>NEWS_PARSER_PLUGIN_URL.'/public/js/parser_page-'.NEWS_PARSER_PLUGIN_VERSION.'.bundle.js'
'main-parser-page-bundle'=>array(
'path'=>NEWS_PARSER_PLUGIN_URL.'/public/js/parser_page-'.NEWS_PARSER_PLUGIN_VERSION.'.bundle.js',
'depends_on'=>array('wp-i18n')
)
),
$menu_config->menu->subs[2]->menu_slug=>array(
'main-parser-autopilot-bundle'=>NEWS_PARSER_PLUGIN_URL.'/public/js/autopilot-'.NEWS_PARSER_PLUGIN_VERSION.'.bundle.js'
'main-parser-autopilot-bundle'=>array(
'path'=>NEWS_PARSER_PLUGIN_URL.'/public/js/autopilot-'.NEWS_PARSER_PLUGIN_VERSION.'.bundle.js',
'depends_on'=>array('wp-i18n')
)
),
$menu_config->menu->subs[3]->menu_slug=>array(

Expand Down
39 changes: 32 additions & 7 deletions inc/Config/scripts-config.php
Original file line number Diff line number Diff line change
Expand Up @@ -9,19 +9,44 @@
return array(
'global'=>array(),
'shared'=> array(
NEWS_PARSER_PLUGIN_SLUG . '-152-chunk'=>NEWS_PARSER_PLUGIN_URL . '/public/js/152-'.NEWS_PARSER_PLUGIN_VERSION.'.bundle.js',
NEWS_PARSER_PLUGIN_SLUG . '-607-chunk'=>NEWS_PARSER_PLUGIN_URL . '/public/js/607-'.NEWS_PARSER_PLUGIN_VERSION.'.bundle.js',
NEWS_PARSER_PLUGIN_SLUG . '-748-chunk'=>NEWS_PARSER_PLUGIN_URL . '/public/js/748-'.NEWS_PARSER_PLUGIN_VERSION.'.bundle.js',
NEWS_PARSER_PLUGIN_SLUG . '-771-chunk'=>NEWS_PARSER_PLUGIN_URL . '/public/js/771-'.NEWS_PARSER_PLUGIN_VERSION.'.bundle.js',
NEWS_PARSER_PLUGIN_SLUG . '-388-chunk'=>array(
'path'=>NEWS_PARSER_PLUGIN_URL . '/public/js/388-'.NEWS_PARSER_PLUGIN_VERSION.'.bundle.js',
'depends_on'=>array()
),
NEWS_PARSER_PLUGIN_SLUG . '-589-chunk'=>array(
'path'=>NEWS_PARSER_PLUGIN_URL . '/public/js/589-'.NEWS_PARSER_PLUGIN_VERSION.'.bundle.js',
'depends_on'=>array()
),
NEWS_PARSER_PLUGIN_SLUG . '-748-chunk'=>array(
'path'=>NEWS_PARSER_PLUGIN_URL . '/public/js/748-'.NEWS_PARSER_PLUGIN_VERSION.'.bundle.js',
'depends_on'=>array()
),
NEWS_PARSER_PLUGIN_SLUG . '-771-chunk'=>array(
'path'=>NEWS_PARSER_PLUGIN_URL . '/public/js/771-'.NEWS_PARSER_PLUGIN_VERSION.'.bundle.js',
'depends_on'=>array()
),
NEWS_PARSER_PLUGIN_SLUG.'-google-analitics'=>array(
'path'=>'https://www.googletagmanager.com/gtag/js?id=G-ZPX4NQDFKG',
'depends_on'=>array()
)
),
$menu_config->menu->subs[0]->menu_slug=>array(
'main-parser-rss-bundle'=> NEWS_PARSER_PLUGIN_URL . '/public/js/parser_rss-'.NEWS_PARSER_PLUGIN_VERSION.'.bundle.js'
'main-parser-rss-bundle'=> array(
'path'=>NEWS_PARSER_PLUGIN_URL . '/public/js/parser_rss-'.NEWS_PARSER_PLUGIN_VERSION.'.bundle.js',
'depends_on'=>array('wp-i18n')
)
),
$menu_config->menu->subs[1]->menu_slug=>array(
'main-parser-page-bundle'=>NEWS_PARSER_PLUGIN_URL . '/public/js/parser_page-'.NEWS_PARSER_PLUGIN_VERSION.'.bundle.js'
'main-parser-page-bundle'=>array(
'path'=>NEWS_PARSER_PLUGIN_URL . '/public/js/parser_page-'.NEWS_PARSER_PLUGIN_VERSION.'.bundle.js',
'depends_on'=>array('wp-i18n')
)
),
$menu_config->menu->subs[2]->menu_slug=>array(
'main-parser-autopilot-bundle'=>NEWS_PARSER_PLUGIN_URL . '/public/js/autopilot-'.NEWS_PARSER_PLUGIN_VERSION.'.bundle.js'
'main-parser-autopilot-bundle'=>array(
'path'=>NEWS_PARSER_PLUGIN_URL . '/public/js/autopilot-'.NEWS_PARSER_PLUGIN_VERSION.'.bundle.js',
'depends_on'=>array('wp-i18n')
)
),
$menu_config->menu->subs[3]->menu_slug=>array(

Expand Down
17 changes: 17 additions & 0 deletions inc/Config/scripts-translation-config.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
<?php

return array(
'global'=>array(),
'shared'=> array(),
$menu_config->menu->subs[0]->menu_slug=>array(
'main-parser-rss-bundle'=> NEWS_PARSER_PLUGIN_SLUG
),
$menu_config->menu->subs[1]->menu_slug=>array(
'main-parser-page-bundle'=>NEWS_PARSER_PLUGIN_SLUG
),
$menu_config->menu->subs[2]->menu_slug=>array(
'main-parser-autopilot-bundle'=>NEWS_PARSER_PLUGIN_SLUG
),
$menu_config->menu->subs[3]->menu_slug=>array(
)
);
4 changes: 1 addition & 3 deletions inc/Config/styles-config.php
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,7 @@
'shared'=> array(
NEWS_PARSER_PLUGIN_SLUG . '-bootstrap'=>'https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css',
NEWS_PARSER_PLUGIN_SLUG . '-bootstrap-icons'=> 'https://cdn.jsdelivr.net/npm/[email protected]/font/bootstrap-icons.css',
NEWS_PARSER_PLUGIN_SLUG . '-media_views'=>NEWS_PARSER_PLUGIN_URL . '/public/css/media-views.css',
NEWS_PARSER_PLUGIN_SLUG . '-152'=>NEWS_PARSER_PLUGIN_URL.'/public/css/152-'.NEWS_PARSER_PLUGIN_VERSION.'.css',
NEWS_PARSER_PLUGIN_SLUG . '-607'=>NEWS_PARSER_PLUGIN_URL.'/public/css/607-'.NEWS_PARSER_PLUGIN_VERSION.'.css',
NEWS_PARSER_PLUGIN_SLUG . '-media_views'=>NEWS_PARSER_PLUGIN_URL . '/public/css/media-views.css'
),
$menu_config->menu->subs[0]->menu_slug=>array(
NEWS_PARSER_PLUGIN_SLUG . '-parser-rss'=> NEWS_PARSER_PLUGIN_URL.'/public/css/parser_rss-'.NEWS_PARSER_PLUGIN_VERSION.'.css'
Expand Down
22 changes: 16 additions & 6 deletions inc/Controller/CronTaskController.php
Original file line number Diff line number Diff line change
Expand Up @@ -8,26 +8,34 @@
class CronTaskController {

protected const CRON_TABLE_NAME=NEWS_PURSER_PLUGIN_CRON_OPTIONS_NAME;
public function __construct(EventController $event){
public function __construct(EventController $event)
{
$this->event=$event;
}
public function cronTaskCallback($interval){
public function cronTaskCallback($interval)
{
$parsed_posts_counter=0;
$cron_options=$this->getCronOptions(array('interval',$interval));
foreach ($cron_options as $cron_options_data){
foreach ($cron_options as $cron_options_data)
{
$cron_options_model=$this->getCronOptionsModel($cron_options_data);
if($cron_options_model->getCronCalls()<$cron_options_model->getMaxCronCalls())
{
$cron_options_post_counter=$cron_options_model->getParsedPosts();
$rss_list=$this->event->trigger('list:get',array($cron_options_model->getUrl()));
$last_parsed_post_timestamp=$cron_options_model->getTimestamp();
$this->parsePosts(array_filter($rss_list,function($post_data) use ($last_parsed_post_timestamp){
return strtotime($post_data->pubDate)>$last_parsed_post_timestamp;
}),$cron_options_model);
$cron_options_model->increaseCronCalls();
$cron_options_model->save();
$parsed_posts_counter+=$cron_options_model->getParsedPosts()-$cron_options_post_counter;
}
};
return $parsed_posts_counter;
}
protected function parsePosts($posts_rss_data,$cron_options_model){
protected function parsePosts($posts_rss_data,$cron_options_model)
{
//to avoid sorting data by pubDate use $latest_timestamp
$latest_timestamp=$cron_options_model->getTimestamp();
foreach (array_reverse($posts_rss_data) as $post_data){
Expand All @@ -49,10 +57,12 @@ protected function parsePosts($posts_rss_data,$cron_options_model){
}
return $cron_options_model->setTimestamp($latest_timestamp);
}
protected function getCronOptionsModel($cron_options_data){
protected function getCronOptionsModel($cron_options_data)
{
return new CronDataModel($cron_options_data);
}
protected function getCronOptions($filter_data=[]){
protected function getCronOptions($filter_data=[])
{
if($crons_options=$this->getOption(self::CRON_TABLE_NAME)){
return $crons_options;
if(count($filter_data)){
Expand Down
7 changes: 0 additions & 7 deletions inc/Core/App.php
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@ protected function __construct(ContainerInterface $DI_container){
$this->cronApiController=CronApiController::create($this->event);
$this->middleware=MiddlewareController::getInstance($this->event);
$this->cronTaskController=new CronTaskController($this->event);
$this->addActions();
}
static public function start(ContainerInterface $DI_container){
if(self::$instance==null){
Expand All @@ -38,10 +37,4 @@ static public function start(ContainerInterface $DI_container){
}
return self::$instance;
}
protected function addActions(){
add_action(NEWS_PARSER_CRON_ACTION_PREFIX.'hourly',array($this->cronTaskController,'cronTaskCallback'));
add_action(NEWS_PARSER_CRON_ACTION_PREFIX.'twicedaily',array($this->cronTaskController,'cronTaskCallback'));
add_action(NEWS_PARSER_CRON_ACTION_PREFIX.'daily',array($this->cronTaskController,'cronTaskCallback'));
add_action(NEWS_PARSER_CRON_ACTION_PREFIX.'weekly',array($this->cronTaskController,'cronTaskCallback'));
}
}
Loading

0 comments on commit ca93592

Please sign in to comment.