Add discovery of linked URLs during crawling

This works but it doesn't loop around, so new
URLs may not be crawled until the next job run.

Author: john-shaffer

src/Crawler.php

@@ -107,12 +107,14 @@ public static function wp2staticCrawl( string $crawler_slug ) : void {
         if ( 'wp2static' === $crawler_slug ) {
             $paths = CrawlQueue::getPathsIter();
             $crawler = new Crawler();
+            $url_discovery = new URLDiscovery();
             $crawled = $crawler->crawlIter( $paths );
             $crawled = CrawlCache::remove404s( $crawled );
             $crawled = CrawlCache::writeFilesIter( $crawled );
             if ( $crawler->use_crawl_cache ) {
                 $crawled = CrawlCache::addPathsIter( $crawled );
             }
+            $crawled = $url_discovery->discoverURLs( $crawled );
             foreach ( $crawled as $crawled ) {}
             $crawler->crawlComplete();
         }

src/DirectDeployer.php

@@ -11,6 +11,7 @@ class DirectDeployer {
     private $crawler;
     private $deployer;
     private $processor;
+    private $url_discovery;
     private $use_crawl_cache;
 
     public function __construct() {
@@ -30,6 +31,7 @@ public function __construct() {
 
         $this->deployer = new $deployer_class();
         $this->crawler = new Crawler();
+        $this->url_discovery = new URLDiscovery();
         $this->processor = new PostProcessor();
 
         $this->use_crawl_cache = CoreOptions::getValue( 'useCrawlCaching' );
@@ -57,6 +59,8 @@ public function deployPaths( \Iterator $paths ) : void {
             $crawled = CrawlCache::addPathsIter( $crawled );
         }
 
+        $crawled = $this->url_discovery->discoverURLs( $crawled );
+
         $processed = $this->processor->processIter( $crawled );
         $this->deployer->uploadFilesIter( $processed );
     }

src/URLDiscovery.php

@@ -0,0 +1,106 @@
+<?php
+
+namespace WP2Static;
+
+use WP2Static\FileFiltering;
+use WP2Static\ParseHTML;
+
+class URLDiscovery {
+    private string $destination_host;
+    private string $destination_url;
+    private bool $discover_complete = false;
+    private FileFiltering $file_filtering;
+    private string $site_host;
+
+    public function __construct( ) {
+        $this->destination_url = untrailingslashit(
+            apply_filters(
+                'wp2static_set_destination_url',
+                CoreOptions::getValue( 'deploymentURL' )
+            )
+        );
+        $this->destination_host = \Wa72\Url\Url::parse( $this->destination_url )->getHost();
+        $this->file_filtering = new FileFiltering();
+    }
+
+    public function discoverURLs( \Iterator $iterator ) : \Iterator {
+        global $wpdb;
+
+        $table_name = CrawlQueue::getTableName();
+
+        foreach ( $iterator as $arr ) {
+            if ( isset( $arr['content_type'] ) && str_starts_with( $arr['content_type'], 'text/html' ) ) {
+                $urls = [];
+                foreach ( $this->parseURLs( $arr ) as $url ) {
+                    $urls[$url] = true;
+                }
+                if ( empty( $urls ) ) {
+                    yield $arr;
+                    continue;
+                }
+                $placeholders = array_fill( 0, count( $urls ), '(%s)' );
+                $sql = "INSERT IGNORE INTO $table_name (url) VALUES " . implode( ',', $placeholders );
+                $result = $wpdb->query( $wpdb->prepare( $sql, ...array_keys( $urls ) ) );
+                if ( false === $result ) {
+                    WsLog::w( 'Error inserting into crawl queue: ' . $wpdb->last_error );
+                }
+                yield $arr;
+            } else {
+                yield $arr;
+            }
+        }
+
+        $this->discover_complete = true;
+    }
+
+    public function isURLLocal( \Wa72\Url\Url $base_url, \Wa72\Url\Url $url ) : bool {
+        if ( ! $url->getPath() && ! $url->getHost() ) {
+            // fragment-only URL
+            return false;
+        }
+
+        $scheme = $url->getScheme();
+
+        if ( ! $scheme ) {
+            $url = $url->makeAbsolute( $base_url );
+            $scheme = $url->getScheme();
+        }
+
+        $path = $url->getPath();
+
+        if ( ( 'http' === $scheme || 'https' === $scheme ) &&
+                $url->equalsHost( $this->destination_host ) && 0 < strlen( $url->getPath() ) &&
+                $this->file_filtering->pathLooksCrawlable( $path ) ) {
+            return true;
+        }
+
+        return false;
+    }
+
+    public function parseURLs( array $arr ) : \Iterator {
+        $body = null;
+        if ( isset( $arr['body'] ) ) {
+            $body = $arr['body'];
+        } else if ( isset( $arr['filename'] ) ) {
+            $body = file_get_contents( $arr['filename'] );
+        }
+
+        if ( ! $body ) {
+            return;
+        }
+
+        $page_url = \Wa72\Url\Url::parse( $this->destination_url . $arr['path'] );
+        foreach ( ParseHTML::parseURLsString( $body ) as $url ) {
+            $discovered_url = \Wa72\Url\Url::parse( $url );
+            $discovered_url->setFragment( '' );
+            $discovered_url->setQuery( '' );
+            $is_local = $this->isURLLocal( $page_url, $discovered_url );
+            $discovered_url->setHost( '' );
+            $discovered_url->setScheme( '' );
+            if ( $is_local ) {
+                yield $discovered_url->write();
+            }
+        }
+    }
+
+}