Recursive fetch & parse

rssmonster.php

@@ -1,16 +1,32 @@
 <?php
-
 declare(strict_types = 1);
 
+// Some dependencies haven't been updated for PHP 8
+// This can probably be removed for PHP 7.3
+error_reporting(E_ALL & ~E_DEPRECATED);
+
 require('vendor/autoload.php');
 
 use GuzzleHttp\Client as HTTPClient;
 use PHPHtmlParser\Dom as DOM;
 
 class Log {
+    private static $enabled = [
+        'silly' => false,
+        'debug' => true,
+        'info' => true
+    ];
+    
     private static function emit(string $level, $message) {
+        if (!Log::$enabled[$level]) {
+            return;
+        };
         $now = date("c");
-        print("$now [$level] $message");
+        print("$now [$level] $message\n");
+    }
+
+    public static function silly($message) {
+        Log::emit('silly', $message);
     }
 
     public static function debug($message) {
@@ -28,6 +44,9 @@ function fetchUrl(string $url) {
     $res = $client->request(
         'GET', $url, [
             'http_errors' => false,
+            'headers' => [
+                'User-Agent' => 'Lexie\'s RSS Monster (for Cial) (lexie@alexis-marie-wright.me)'
+            ]
         ]
     );
 
@@ -36,7 +55,45 @@ function fetchUrl(string $url) {
         throw "Request for $url returned {$res->getStatusCode()}";
     };
 
-    return $res->getBody();
+    return strval($res->getBody());
 };
 
-Log::info(fetchUrl('http://inhuman-comic.com'));
+$baseUrl = 'http://inhuman-comic.com';
+
+$nextLinkSelector = 'div.body a.prev';
+$urls = [$baseUrl];
+$fetchedUrls = [];
+
+while (count($urls) > 0) {
+    $url = array_shift($urls);
+    
+    $res = fetchUrl($url);
+    Log::info("$url: fetched " . strlen($res) . " bytes");
+
+    Log::silly($res);
+    
+    array_push($fetchedUrls, $url);
+
+    $dom = new DOM;
+    $dom->loadStr($res);
+
+    foreach (array($dom->find($nextLinkSelector)) as $el) {
+        $nextUrl = $baseUrl . '/' . $el->href;
+        
+        if (array_search($nextUrl, $fetchedUrls)) {
+            Log::debug("$url: already fetched next link $nextUrl");
+            continue;
+        };
+
+        if (array_search($nextUrl, $urls)) {
+            Log::debug("$url: duplicate next link $nextUrl");
+            continue;
+        };
+        
+        Log::info("$url: next link $nextUrl");
+        array_push($urls, $nextUrl);
+    };
+
+    usleep(500000); // wait half a second, to be minimally polite
+};
+Log::info("Out of next links!");