Refactor fetcher, add specs

db/scrape.rb

@@ -3,8 +3,8 @@ require 'sequel'
 
 require_relative '../db/connect'
 require_relative '../lib/auth'
-require_relative '../lib/fetch'
 require_relative '../lib/parse'
+require_relative '../lib/fetcher'
 require_relative '../lib/models/post'
 require_relative '../lib/models/thread'
 
@@ -19,7 +19,7 @@ class Scraper
 
   def scrape
     (@first..@last).each_with_index do |page_number, page_index|
-      page = Fetch.page(page_number, @cookie)
+      page = Fetcher.new(cookie: @cookie).page(page_number)
       threads = Parse.threads(page)
 
       threads.each do |t|
@@ -44,7 +44,7 @@ class Scraper
   def scrape_thread(t)
     log t[:title]
 
-    page = Fetch.thread(t, @cookie)
+    page = Fetcher.new(cookie: @cookie).thread(t)
     first_post = page.at_css('.postinfo:first-child')
     return false if first_post.nil?
 

lib/fetch.rb

@@ -1,24 +0,0 @@
-require 'net/http'
-
-module Fetch
-  def self.page(page_number, cookie)
-    url = URI("http://board.vivalavinyl.com/thread/list/#{page_number}")
-    http = Net::HTTP.new(url.host, url.port)
-    request = Net::HTTP::Get.new(url)
-    request['cookie'] = cookie
-    response = http.request(request)
-    Nokogiri.HTML(response.body)
-  end
-
-  def self.thread(thread, cookie)
-    url =
-      URI(
-        "http://board.vivalavinyl.com/thread/view/#{thread[:remote_id]}&ajax=true"
-      )
-    http = Net::HTTP.new(url.host, url.port)
-    request = Net::HTTP::Get.new(url)
-    request['cookie'] = cookie
-    response = http.request(request)
-    Nokogiri.HTML(response.body)
-  end
-end

lib/fetcher.rb

@@ -0,0 +1,29 @@
+require 'net/http'
+require 'nokogiri'
+
+class Fetcher
+  BASE_URL = 'http://board.vivalavinyl.com'
+
+  def initialize(cookie:)
+    @cookie = cookie
+  end
+
+  def page(page_number)
+    authenticated_request("/thread/list/#{page_number}")
+  end
+
+  def thread(thread)
+    authenticated_request("/thread/view/#{thread[:remote_id]}&ajax=true")
+  end
+
+  private
+
+  def authenticated_request(path)
+    uri = URI("#{BASE_URL}#{path}")
+    http = Net::HTTP.new(uri.host, uri.port)
+    request = Net::HTTP::Get.new(uri)
+    request['cookie'] = @cookie
+    response = http.request(request)
+    Nokogiri.HTML(response.body)
+  end
+end

spec/fetcher_spec.rb

@@ -0,0 +1,32 @@
+require 'dotenv/load'
+require_relative '../lib/auth'
+require_relative '../lib/fetcher'
+
+RSpec.describe Fetcher do
+  let(:cookie) { login(ENV['VLV_USERNAME'], ENV['VLV_PASSWORD']) }
+  let(:thread_id) { 49668 }
+
+  subject { Fetcher.new(cookie: cookie) }
+
+  describe '#thread' do
+    it "fetches a thread" do
+      result = subject.thread({ remote_id: 49668 })
+      aggregate_failures do
+        expect(result).to be_a(Nokogiri::HTML::Document)
+        expect(result.at_css('.post:first-child').text)
+          .to include('reddwarf posted this March 30th, 2020 @ 3:29:23 am')
+      end
+    end
+  end
+
+  describe '#page' do
+    it "fetches a page of threads" do
+      result = subject.page(0)
+      aggregate_failures do
+        expect(result).to be_a(Nokogiri::HTML::Document)
+        expect(result.at_css('h3').text).to eq('VLV: politics, food, and aging')
+        expect(result.css('.even, .odd').length).to eq(110)
+      end
+    end
+  end
+end