Add recursive discovery during crawl job

Author: john-shaffer

src/CrawlQueue.php

@@ -168,16 +168,17 @@ public static function withPathsIter( \Iterator $paths ) : \Iterator {
     /**
      * Yields all paths in the table.
      *
+     * @param string $detected_since default to '0000-00-00 00:00:00'
      */
-    public static function getPathsIter() : \Iterator {
+    public static function getPathsIter( string $detected_since = '0000-00-00 00:00:00' ) : \Iterator {
         global $wpdb;
 
         $table_name = self::getTableName();
         $batch_size = 1000;
         $last_id = 0;
         while ( true ) {
-            $qs = "SELECT id, url AS path, filename FROM $table_name WHERE id > %d ORDER BY id ASC LIMIT %d";
-            $q = $wpdb->prepare( $qs, $last_id, $batch_size );
+            $qs = "SELECT id, url AS path, filename FROM $table_name WHERE id > %d AND detected_at >= %s ORDER BY id ASC LIMIT %d";
+            $q = $wpdb->prepare( $qs, $last_id, $detected_since, $batch_size );
             $rows = $wpdb->get_results( $q, ARRAY_A );
 
             foreach ( $rows as $row ) {

src/Crawler.php

@@ -104,18 +104,38 @@ public function __construct() {
     }
 
     public static function wp2staticCrawl( string $crawler_slug ) : void {
+        global $wpdb;
+
         if ( 'wp2static' === $crawler_slug ) {
-            $paths = CrawlQueue::getPathsIter();
             $crawler = new Crawler();
             $url_discovery = new URLDiscovery();
-            $crawled = $crawler->crawlIter( $paths );
+
+            $detected = CrawlQueue::getPathsIter();
+            $last_now = $wpdb->get_var( 'SELECT NOW()' );
+            $crawled = $crawler->crawlIter( $detected );
             $crawled = CrawlCache::remove404s( $crawled );
             $crawled = CrawlCache::writeFilesIter( $crawled );
             if ( $crawler->use_crawl_cache ) {
                 $crawled = CrawlCache::addPathsIter( $crawled );
             }
             $crawled = $url_discovery->discoverURLs( $crawled );
-            foreach ( $crawled as $crawled ) {}
+            foreach ( $crawled as $_ ) { }
+
+            $has_new = true;
+            while ( $has_new ) {
+                $detected = CrawlQueue::getPathsIter( $last_now );
+                $last_now = $wpdb->get_var( 'SELECT NOW()' );
+                $crawled = $crawler->crawlIter( $detected );
+                $crawled = CrawlCache::writeFilesIter( $crawled );
+                if ( $crawler->use_crawl_cache ) {
+                    $crawled = CrawlCache::addPathsIter( $crawled );
+                }
+                $crawled = $url_discovery->discoverURLs( $crawled );
+                $has_new = false;
+                foreach ( $crawled as $_ ) {
+                    $has_new = true;
+                }
+            }
             $crawler->crawlComplete();
         }
     }