Refactor parser and add specs

db/scrape.rb

@@ -3,8 +3,8 @@ require 'sequel'
 
 require_relative '../db/connect'
 require_relative '../lib/auth'
-require_relative '../lib/parse'
 require_relative '../lib/fetcher'
+require_relative '../lib/parser'
 require_relative '../lib/models/post'
 require_relative '../lib/models/thread'
 
@@ -20,7 +20,7 @@ class Scraper
   def scrape
     (@first..@last).each_with_index do |page_number, page_index|
       page = Fetcher.new(cookie: @cookie).page(page_number)
-      threads = Parse.threads(page)
+      threads = Parser.new.threads(page)
 
       threads.each do |t|
         next if page_index > 0 && t[:is_sticky]
@@ -48,10 +48,9 @@ class Scraper
     first_post = page.at_css('.postinfo:first-child')
     return false if first_post.nil?
 
-    t[:created_at] = Parse.thread_created_at(first_post)
-
     thread = DB.from(:threads).first(remote_id: t[:remote_id])
     if thread.nil?
+      t[:created_at] = Parser.new.thread_created_at(first_post)
       thread = VLV::Thread.create(t.delete_if { |k| k == :is_sticky })
       log '  Inserting thread'
     end
@@ -60,7 +59,7 @@ class Scraper
   end
 
   def scrape_posts(thread, page)
-    posts = Parse.posts(thread, page)
+    posts = Parser.new.posts(thread, page)
     last_post = posts.last
     unless DB.from(:posts).first(remote_id: last_post[:remote_id]).nil?
       log '  No new posts'

lib/parse.rb → lib/parser.rb

@@ -1,7 +1,8 @@
 require 'nokogiri'
+require 'time'
 
-module Parse
-  def self.threads(page)
+class Parser
+  def threads(page)
     threads = Array.new
 
     page.css('.even, .odd').each do |row|
@@ -20,15 +21,14 @@ module Parse
     threads
   end
 
-  def self.posts(thread, page)
+  def posts(thread, page)
     posts = Array.new
 
     page.css('.post').each do |_post|
       post = Hash.new
       post[:remote_id] = _post.at_css('ul.view')[:id].split('_')[1].to_i
       post[:creator] = _post.at_css('.memberlink').text.strip
-      date, time =
-        _post.at_css('.postinfo').text.split('posted this')[1].split('@')
+      date, time = _post.at_css('.postinfo').text.split('posted this')[1].split('@')
       post[:created_at] = Time.parse("#{date} #{time}")
       post[:body] = _post.at_css('.postbody').children.map(&:to_html).join.strip
       post[:thread_id] = thread[:id]
@@ -38,7 +38,7 @@ module Parse
     posts
   end
 
-  def self.thread_created_at(first_post)
+  def thread_created_at(first_post)
     post_info = first_post.text.split('posted this')
     date, time = post_info[1].split('@')
     Time.parse("#{date} #{time}")

spec/parser_spec.rb

@@ -1,47 +1,123 @@
-require_relative '../lib/parse'
+require_relative '../lib/parser'
 
-RSpec.describe 'Parser' do
-  it 'should parse threads' do
-    html = Nokogiri::HTML(<<~HTML)
-      <div class="even" id="thread_12345">
-        <ul class="list read">
-          <li class="member">
-            <span>Thread By: </span>
-            <a href="/member/view/creator1/" class="memberlink">creator1</a>
-          </li>
-          <li class="subject">
-            <span>Subject: </span>
-            <a href="/thread/view/12345/&p=999">
-              <strong>Sticky:</sticky> Thread title 1
-            </a>
-          </li>
-          <li class="posts"><span>Posts: </span>999</li>
-          <li class="lastpost">
-            <span>Last Post By:</span>
-            <a href="/member/view/lastposter1/" class="memberlink">lastposter1</a> on Fri&nbsp;Apr&nbsp;10&nbsp;2020&nbsp;01:23&nbsp;am</li>
-        </ul>
-      </div>
-      <div class="even" id="thread_123456">
-        <ul class="list read">
-          <li class="member">
-            <span>Thread By: </span>
-            <a href="/member/view/creator2/" class="memberlink">creator2</a>
-          </li>
-          <li class="subject">
-            <span>Subject: </span>
-            <a href="/thread/view/123456/&p=999">Thread title 2</a>
-          </li>
-          <li class="posts"><span>Posts: </span>999</li>
-          <li class="lastpost">
-            <span>Last Post By:</span>
-            <a href="/member/view/lastposter2/" class="memberlink">lastposter2</a> on Fri&nbsp;Apr&nbsp;10&nbsp;2020&nbsp;01:23&nbsp;am</li>
-        </ul>
+RSpec.describe Parser do
+  subject { Parser.new }
+
+  describe '#threads' do
+    let(:html) do
+      Nokogiri::HTML(<<~HTML)
+        <div class="even" id="thread_12345">
+          <ul class="list read">
+            <li class="member">
+              <span>Thread By: </span>
+              <a href="/member/view/creator1/" class="memberlink">creator1</a>
+            </li>
+            <li class="subject">
+              <span>Subject: </span>
+              <a href="/thread/view/12345/&p=999">
+                <strong>Sticky:</sticky> Thread title 1
+              </a>
+            </li>
+            <li class="posts"><span>Posts: </span>999</li>
+            <li class="lastpost">
+              <span>Last Post By:</span>
+              <a href="/member/view/lastposter1/" class="memberlink">lastposter1</a> on Fri&nbsp;Apr&nbsp;10&nbsp;2020&nbsp;01:23&nbsp;am</li>
+          </ul>
+        </div>
+        <div class="even" id="thread_123456">
+          <ul class="list read">
+            <li class="member">
+              <span>Thread By: </span>
+              <a href="/member/view/creator2/" class="memberlink">creator2</a>
+            </li>
+            <li class="subject">
+              <span>Subject: </span>
+              <a href="/thread/view/123456/&p=999">Thread title 2</a>
+            </li>
+            <li class="posts"><span>Posts: </span>999</li>
+            <li class="lastpost">
+              <span>Last Post By:</span>
+              <a href="/member/view/lastposter2/" class="memberlink">lastposter2</a> on Fri&nbsp;Apr&nbsp;10&nbsp;2020&nbsp;01:23&nbsp;am</li>
+          </ul>
+        </div>
+      HTML
+    end
+
+    it 'parses threads' do
+      expect(subject.threads(html)).to eq([
+        {remote_id: '12345', title: 'Sticky: Thread title 1', creator: 'creator1', is_sticky: true},
+        {remote_id: '123456', title: 'Thread title 2', creator: 'creator2', is_sticky: false},
+      ])
+    end
+  end
+
+  describe '#posts' do
+    let(:html) do
+      Nokogiri::HTML(<<~HTML)
+        <div>
+          <div class="post">
+            <ul class="view" id="post_69">
+              <li class="info">
+                <div class="postinfo">
+                  <a class="memberlink" href="/member/view/User1">User1</a>
+                  posted this October 26th, 2021 @ 12:34:56 am
+                </div>
+              </li>
+              <li class="postbody">
+                This is the body of the first post
+              </li>
+            </ul>
+          </div>
+          <div class="post">
+            <ul class="view" id="post_420">
+              <li class="info">
+                <div class="postinfo">
+                  <a class="memberlink" href="/member/view/User2">User2</a>
+                  posted this October 27th, 2021 @ 12:34:56 am
+                </div>
+              </li>
+              <li class="postbody">
+                This is the body of the second post
+              </li>
+            </ul>
+          </div>
+        </div>
+      HTML
+    end
+
+    it 'parses posts' do
+      expect(subject.posts({ id: 666 }, html)).to match_array([
+        {
+          remote_id: 69,
+          creator: 'User1',
+          thread_id: 666,
+          created_at: Time.new(2021, 10, 26, 0, 34, 56),
+          body: 'This is the body of the first post'
+        },
+        {
+          remote_id: 420,
+          creator: 'User2',
+          thread_id: 666,
+          created_at: Time.new(2021, 10, 27, 0, 34, 56),
+          body: 'This is the body of the second post'
+        }
+      ])
+    end
+  end
+
+  describe '#thread_created_at' do
+    let(:html) do
+      Nokogiri::HTML(<<~HTML)
+      <div class="postinfo">
+        <a class="memberlink" href="/member/view/User1">User1</a>
+        posted this October 27th, 2021 @ 12:34:56 am
       </div>
-    HTML
+      HTML
+    end
 
-    expect(Parse.threads(html)).to eq([
-      {remote_id: '12345', title: 'Sticky: Thread title 1', creator: 'creator1', is_sticky: true},
-      {remote_id: '123456', title: 'Thread title 2', creator: 'creator2', is_sticky: false},
-    ])
+    it 'parses the timestamp of the first post' do
+      expect(subject.thread_created_at(html))
+        .to eq(Time.new(2021, 10, 27, 00, 34, 56))
+    end
   end
 end