Crawl images and other HTML linked resources as well

lib/parklife/utils.rb

@@ -39,8 +39,17 @@ def save_page(path, content, config)
 
     def scan_for_links(html)
       doc = Nokogiri::HTML.parse(html)
-      doc.css('a').each do |a|
-        uri = URI.parse(a[:href])
+
+      # In this very exciting exercise we'll go through a list of elements to collect a list of candidate
+      # urls that *maybe* we'll want to crawl.
+      urls = doc.css('a').map { |v| v[:href] }
+      urls.concat(doc.css('img').map { |v| v[:src] })
+
+      # Source elements come in two flavors, for images the srcset attribute is used, for audio / video the src is used
+      urls.concat(doc.css('source').map { |v| [v[:srcset], v[:src]].compact.reject(&:empty?).first })
+
+      urls.each do |url|
+        uri = URI.parse(url)
 
         # Don't visit a URL that belongs to a different domain - for now this is
         # a guess that it's not an internal link but it also covers mailto/ftp

spec/parklife/utils_spec.rb

@@ -144,6 +144,12 @@ def build_files
         ❌ <a href="">empty</a>
         ✅ <a href="/baz">baz</a>
         ❌ <a href="ftp://example.com/foo/bar">ftp</a>
+
+        ✅ <img src="/foo.jpg" alt="Foo is life">
+        ❌ <img src="https://www.example.com/foo.jpg" alt="Foo is life">
+
+        ✅ <audio><source src="/foo.mp3" type="audio/mp3"></audio>
+        ✅ <picture><source srcset="/foo.webp"><source srcset="/foo-squared.jpg"><img src="/default-foo.jpg"></picture>
       HTML
     }
 
@@ -153,7 +159,14 @@ def build_files
       }.to yield_successive_args(
         '/foo',
         '/bar',
-        '/baz'
+        '/baz',
+
+        '/foo.jpg',
+        '/default-foo.jpg',
+
+        '/foo.mp3',
+        '/foo.webp',
+        '/foo-squared.jpg',
       )
     end
   end