Merge MOODLE_310_STABLE branch

Author: TomoTsuyuki

.github/workflows/ci.yml

@@ -0,0 +1,12 @@
+# .github/workflows/ci.yml
+name: ci
+
+on: [push, pull_request]
+
+jobs:
+  ci:
+    uses: catalyst/catalyst-moodle-workflows/.github/workflows/ci.yml@main
+    secrets:
+      moodle_org_token: ${{ secrets.MOODLE_ORG_TOKEN }}
+    with:
+      disable_behat: true

.travis.yml

@@ -1,4 +1,6 @@
-[![Build Status](https://travis-ci.org/catalyst/moodle-tool_crawler.svg?branch=master)](https://travis-ci.org/catalyst/moodle-tool_crawler)
+[![ci](https://github.com/catalyst/moodle-tool_crawler/actions/workflows/ci.yml/badge.svg?branch=MOODLE_310_STABLE)](https://github.com/catalyst/moodle-tool_crawler/actions/workflows/ci.yml?branch=MOODLE_310_STABLE)
+
+# moodle-tool_crawler
 
 * [What is this?](#what-is-this)
 * [How does it work?](#how-does-it-work)
@@ -30,23 +32,24 @@ Since the plugin cronjob comes in from outside it needs to authenticate in Moodl
 
 # Branches
 
-| Moodle verion     | Branch      |
-| ----------------- | ----------- |
-| Moodle 3.4 to 3.8 | master      |
-| Totara 12+        | master      |
+| Moodle version    | Branch                |
+| ----------------- | --------------------- |
+| Moodle 3.10+      | MOODLE_310_STABLE     |
+| Moodle 3.4 to 3.9 | master                |
+| Totara 12+        | master                |
 
 # Installation
 
 The plugin has a dependency on the [moodle-auth_basic](https://moodle.org/plugins/auth_basic).
 To install the dependency plugin as a git submodule:
 ```
-git submodule add https://github.com/catalyst/moodle-auth_basic auth/basic
+git submodule add git@github.com:catalyst/moodle-auth_basic.git auth/basic
 ```
 
 
 Install plugin moodle-tool_crawler as a git submodule:
 ```
-git submodule add https://github.com/central-queensland-uni/moodle-tool_crawler.git admin/tool/crawler
+git submodule add git@github.com:catalyst/moodle-tool_crawler.git admin/tool/crawler
 ```
 # Configuration
 
@@ -156,7 +159,7 @@ be able to see the line "You are logged in as ".
 Once Basic HTTP auth works test running the robot task from the CLI:
 
 ```
-php admin/tool/task/cli/schedule_task.php --execute='\tool_crawler\task\crawl_task'
+php admin/cli/scheduled_task.php --execute='\tool_crawler\task\crawl_task'
 Execute scheduled task: Parallel crawling task (tool_crawler\task\crawl_task)
 ... used 22 dbqueries
 ... used 0.039698123931885 seconds
@@ -168,7 +171,7 @@ will run in parallel, depending on the crawl_task setting.
 
 You can manually run the adhoc tasks from the CLI with:
 ```
-php admin/tool/task/cli/adhoc_task.php --execute
+php admin/cli/adhoc_task.php --execute
 Execute adhoc task: tool_crawler\task\adhoc_crawl_task
 ... used 5733 dbqueries
 ... used 58.239180088043 seconds

README.md

@@ -529,6 +529,7 @@ public function process_queue($verbose = false) {
         // Iterate through the queue.
         $cronstart = time();
         $cronstop = $cronstart + $config->maxcrontime;
+        $hastime = true;
 
         // Get an instance of the currently configured lock_factory.
         $lockfactory = \core\lock\lock_config::get_lock_factory('tool_crawler_process_queue');
@@ -550,7 +551,16 @@ public function process_queue($verbose = false) {
             }
         }
         // While we are not exceeding the maxcron time, and the queue is not empty.
-        while (time() < $cronstop) {
+        while ($hastime) {
+
+            if (\core\local\cli\shutdown::should_gracefully_exit() ||
+                \core\task\manager::static_caches_cleared_since($cronstart)) {
+                if ($verbose) {
+                    echo "Shutting down crawler early\n";
+                }
+                return true;
+            }
+
             if (empty($nodes)) {
                 // Grab a list of items from the front of the queue. We need the first 1000
                 // in case other workers are already locked and processing items at the front of the queue.
@@ -625,6 +635,8 @@ public function process_queue($verbose = false) {
             } finally {
                 $lock->release();
             }
+
+            $hastime = time() < $cronstop;
         }
         if ($courselock) {
             $courselock->release();
@@ -906,8 +918,9 @@ public function parse_html($node, $external, $verbose = false) {
             } while ($walk);
 
             $text = self::clean_html_node_content($e);
+            $text = trim($text);
             if ($verbose > 1) {
-                printf (" - Found link to: %-20s / %-50s => %-50s\n", $text, $e->href, $href);
+                printf (" - Found link to: %-30s -> %s\n", "'$text'", $href);
             }
             $this->link_from_node_to_url($node, $href, $text, $idattr);
         }
@@ -1134,7 +1147,12 @@ private static function determine_filesize($curlhandle, $method, $success, $body
     public function scrape($url) {
 
         global $CFG;
-        $cookiefilelocation = $CFG->dataroot . '/tool_crawler_cookies.txt';
+
+        static $cookiefilelocation = '';
+        if (!$cookiefilelocation) {
+            $cookiefilelocation = make_request_directory() . '/tool_crawler_cookies.txt';
+        }
+
         $config = self::get_config();
 
         $version = moodle_major_version();

classes/robot/crawler.php

@@ -52,6 +52,8 @@
     die();
 }
 
+\core\local\cli\shutdown::script_supports_graceful_exit();
+
 tool_crawler_crawl($options['verbose']);
 exit(0);
 

cli/crawler.php

@@ -18,6 +18,7 @@
  * Link checker robot plugin uninstall script.
  *
  * @package    tool_crawler
+ * @copyright  2019 Nicolas Roeser
  * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
  */
 

db/uninstall.php

@@ -80,6 +80,11 @@
 $string['crawlend'] = 'Crawl end';
 $string['crawlstart'] = 'Crawl start';
 $string['cronticks'] = 'Cron ticks';
+$string['debugging'] = 'Verbose debugging';
+$string['debugoff'] = 'Debugging off';
+$string['debugnormal'] = 'Normal debugging';
+$string['debugverbose'] = 'Verbose debugging';
+$string['debuggingdesc'] = 'This turns on debugging in the task output';
 $string['disablebot'] = 'Disable the link crawler robot';
 $string['disablebotdesc'] = 'Make the crawler do nothing when a scheduled task is executed. This effectively prevents crawling of links and running of bot cleanup functions. Intended to deactivate or temporarily pause the crawler without having to disable all its scheduled tasks.';
 $string['duration'] = 'Duration';

lang/en/tool_crawler.php

@@ -45,6 +45,11 @@ function tool_crawler_crawl($verbose = false) {
     $robot = new crawler();
     $url = new url();
     $config = $robot::get_config();
+
+    if ($config->debugging) {
+        $verbose = $config->debugging;
+    }
+
     $crawlstart = $config->crawlstart;
     $crawlend   = $config->crawlend;
 
@@ -179,7 +184,7 @@ function tool_crawler_extend_navigation_course($navigation, $course, $coursecont
     if ($coursereports && ($siteconfig || $courseconfig)) {
         $node = $coursereports->add(
             get_string('pluginname', 'tool_crawler'),
-            null,
+            new moodle_url('/admin/tool/crawler/report.php', array('report' => 'queued', 'course' => $course->id)),
             navigation_node::TYPE_CONTAINER,
             null,
             'linkchecker',

lib.php

@@ -45,14 +45,17 @@
  * @return string HTML snippet which can be used in output.
  */
 function tool_crawler_link($url, $label, $redirect = '', $labelishtml = false, $courseid = 0) {
+    if (empty($label)) {
+        // Ensure that label is always at least a string.
+        $label = '';
+    }
     if (!$labelishtml) {
         $label = htmlspecialchars($label, ENT_NOQUOTES | ENT_HTML401);
     }
 
-    $html = html_writer::link(new moodle_url('url.php', array('courseid' => $courseid, 'url' => $url)), $label) .
-            ' ' .
-            html_writer::link($url, '↗', array('target' => 'link')) .
-            '<br><small>' . htmlspecialchars($url, ENT_NOQUOTES | ENT_HTML401) . '</small>';
+    $canviewsitelevelreports = has_capability('moodle/site:config', context_system::instance());
+    $html = $canviewsitelevelreports ? html_writer::link(new moodle_url('url.php', array('courseid' => $courseid, 'url' => $url)), $label) : $label;
+    $html .= '<br><small>' . html_writer::link($url, htmlspecialchars($url, ENT_NOQUOTES | ENT_HTML401), ['target' => 'link']) . '</small>';
 
     if ($redirect) {
         $linkhtmlsnippet = html_writer::link($redirect, htmlspecialchars($redirect, ENT_NOQUOTES | ENT_HTML401));
@@ -175,11 +178,11 @@ function tool_crawler_sql_oversize_filter($tablealias = null) {
         $tbl = '';
     }
 
-    $where = "( ${tbl}filesize > ?
-             OR ( ${tbl}filesize IS NULL
-                  AND ${tbl}lastcrawled IS NOT NULL
+    $where = "( {$tbl}filesize > ?
+             OR ( {$tbl}filesize IS NULL
+                  AND {$tbl}lastcrawled IS NOT NULL
                 )
-             OR ${tbl}filesizestatus = ?
+             OR {$tbl}filesizestatus = ?
               )";
 
     $bigfilesize = get_config('tool_crawler', 'bigfilesize');

locallib.php

@@ -54,10 +54,13 @@
     $coursecontext = context_course::instance($courseid);
     require_capability('moodle/course:update', $coursecontext);
 
+    $coursename = format_string($course->fullname, true, array('context' => $coursecontext));
     $PAGE->set_context($coursecontext);
     $PAGE->set_url($navurl);
-    $PAGE->set_pagelayout('admin');
     $PAGE->set_title( get_string($report, 'tool_crawler') );
+    $PAGE->set_heading($coursename);
+    $PAGE->set_pagelayout('incourse');
+    $PAGE->add_body_class('limitedwidth');
     $sqlfilter = ' AND c.id = '.$courseid;
 
 } else {