Merge branch 'specs'

* specs:
  Specs
  Rename scraper
  Refactor scraper
  Allow passing DB into migration

.gitignore

@@ -3,3 +3,4 @@ vendor/
 web/public/*.css
 web/public/*.js
 db/log
+spec/vcr

Gemfile

@@ -13,5 +13,6 @@ gem 'rspec', '~> 3.9'
 gem 'sassc', '~> 2.2'
 gem 'sequel', '~> 5.30'
 gem 'sinatra', '~> 2.0'
-
 gem "truncato", "~> 0.7.11"
+gem "vcr", "~> 6.0"
+gem "webmock", "~> 3.14"

Gemfile.lock

@@ -1,9 +1,14 @@
 GEM
   remote: https://rubygems.org/
   specs:
+    addressable (2.8.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    crack (0.4.5)
+      rexml
     diff-lcs (1.3)
     dotenv (2.7.5)
     ffi (1.12.2)
+    hashdiff (1.0.1)
     htmlentities (4.3.4)
     httparty (0.18.0)
       mime-types (~> 3.0)
@@ -18,10 +23,12 @@ GEM
     nokogiri (1.10.9)
       mini_portile2 (~> 2.4.0)
     pg (1.2.3)
+    public_suffix (4.0.6)
     rack (2.2.2)
     rack-protection (2.0.8.1)
       rack
     rake (13.0.1)
+    rexml (3.2.5)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)
@@ -48,6 +55,11 @@ GEM
     truncato (0.7.11)
       htmlentities (~> 4.3.1)
       nokogiri (>= 1.7.0, <= 2.0)
+    vcr (6.0.0)
+    webmock (3.14.0)
+      addressable (>= 2.8.0)
+      crack (>= 0.3.2)
+      hashdiff (>= 0.4.0, < 2.0.0)
 
 PLATFORMS
   ruby
@@ -64,6 +76,8 @@ DEPENDENCIES
   sequel (~> 5.30)
   sinatra (~> 2.0)
   truncato (~> 0.7.11)
+  vcr (~> 6.0)
+  webmock (~> 3.14)
 
 BUNDLED WITH
    2.1.4

Rakefile

@@ -12,8 +12,22 @@ task 'migrate' do
   migrate
 end
 
+task 'migrate_test' do
+  require_relative './db/migrate'
+
+  TEST_DB = Sequel.connect(
+    adapter: :postgres,
+    database: ENV['DB_DATABASE'] + '_test',
+    user: ENV['DB_USERNAME'],
+    password: ENV['DB_PASSWORD'],
+    logger: ENV['APP_ENV'] == 'development' ? Logger.new('db/log') : nil
+  )
+
+  migrate(db: TEST_DB)
+end
+
 task 'scrape' do
-  require_relative './db/scrape'
+  require_relative './db/scraper'
 
   should_log = ENV['APP_ENV'] == 'development' || ARGV.include?('--log')
   scraper = Scraper.new(log: should_log)

db/migrate.rb

@@ -3,8 +3,7 @@ require 'sequel'
 
 require_relative 'connect'
 
-def migrate
-  db = DB
+def migrate(db: DB)
   db.create_table? :threads do
     primary_key :id
     String :title

db/scrape.rb → db/scraper.rb

@@ -13,21 +13,27 @@ class Scraper
     @first = first
     @last = last
     @log = log
+    @no_new_posts = false
 
     authenticate!
   end
 
   def scrape
     (@first..@last).each_with_index do |page_number, page_index|
-      page = Fetcher.new(cookie: @cookie).page(page_number)
-      threads = Parser.new.threads(page)
+      page = fetcher.page(page_number)
+      threads = parser.threads(page)
 
       threads.each do |t|
         next if page_index > 0 && t[:is_sticky]
-        no_new_posts = scrape_thread(t)
+
+        scrape_thread(t)
+
         if no_new_posts
-          next if t[:is_sticky]
-          return
+          if t[:is_sticky]
+            next
+          else
+            return
+          end
         end
       end
     end
@@ -35,50 +41,58 @@ class Scraper
 
   private
 
-  def authenticate!
-    @cookie = login(ENV['VLV_USERNAME'], ENV['VLV_PASSWORD'])
-
-    raise "Error logging into VLV. Check your credentials." if @cookie.nil?
-  end
+  attr_reader :cookie, :fetcher, :parser, :no_new_posts
 
   def scrape_thread(t)
+    @no_new_posts = false
+
     log t[:title]
 
-    page = Fetcher.new(cookie: @cookie).thread(t)
+    page = fetcher.thread(t)
     first_post = page.at_css('.postinfo:first-child')
     return false if first_post.nil?
 
     thread = DB.from(:threads).first(remote_id: t[:remote_id])
     if thread.nil?
-      t[:created_at] = Parser.new.thread_created_at(first_post)
-      thread = VLV::Thread.create(t.delete_if { |k| k == :is_sticky })
       log '  Inserting thread'
+      t[:created_at] = parser.thread_created_at(first_post)
+      thread = VLV::Thread.create(t.delete_if { |k| k == :is_sticky })
     end
 
     scrape_posts(thread, page)
   end
 
   def scrape_posts(thread, page)
-    posts = Parser.new.posts(thread, page)
+    posts = parser.posts(thread, page)
     last_post = posts.last
-    unless DB.from(:posts).first(remote_id: last_post[:remote_id]).nil?
-      log '  No new posts'
-      return true
-    end
 
-    posts_count = posts.size
-    posts.each_with_index do |p, index|
-      msg = "  Inserting post #{index + 1}/#{posts_count}"
-      print msg if @log
-      if DB.from(:posts).first(remote_id: p[:remote_id]).nil?
-        VLV::Post.create(p)
+    if DB.from(:posts).first(remote_id: last_post[:remote_id]).nil?
+      db_posts = VLV::Post.where(remote_id: posts.map { |p| p[:remote_id] }).all
+
+      posts = posts.each_with_index.map do |post|
+        if db_posts.detect { |db_post| db_post.remote_id == post[:remote_id] }.nil?
+          post
+        end
       end
-      print "\b" * msg.size unless index == posts_count - 1 if @log
+
+      log "  Inserting #{posts.size} posts"
+      VLV::Post.multi_insert(posts.compact)
+    else
+      no_new_posts!
+      log '  No new posts'
     end
+  end
 
-    log ''
+  def no_new_posts!
+    @no_new_posts = true
+  end
+
+  def authenticate!
+    @cookie = login(ENV['VLV_USERNAME'], ENV['VLV_PASSWORD'])
+    raise "Error logging into VLV. Check your credentials." if @cookie.nil?
 
-    false
+    @fetcher = Fetcher.new(cookie: cookie)
+    @parser = Parser.new
   end
 
   def log(msg)

spec/scraper_spec.rb

@@ -0,0 +1,28 @@
+require 'dotenv/load'
+require 'spec_helper'
+
+require_relative '../db/scraper'
+
+RSpec.describe Scraper do
+  around(:each) do |example|
+    DB.transaction(rollback: :always, auto_savepoint: true) { example.run }
+  end
+
+  subject { described_class.new(log: true) }
+
+  describe '#initialize' do
+    it 'authenticates' do
+      expect(subject.send(:cookie)).to_not be_nil
+    end
+  end
+
+  describe '#scrape' do
+    it 'creates new threads and posts' do
+      VCR.use_cassette "create_new_threads_and_posts" do
+        expect { subject.scrape }
+          .to  change { VLV::Thread.count }.by(109)
+          .and change { VLV::Post.count   }.by_at_least(5000)
+      end
+    end
+  end
+end

spec/spec_helper.rb

@@ -0,0 +1,12 @@
+require 'vcr'
+require "webmock/rspec"
+
+ENV["DB_DATABASE"] = ENV["DB_DATABASE"] + '_test'
+
+WebMock.disable_net_connect!(allow_localhost: true)
+
+VCR.configure do |c|
+  c.cassette_library_dir = "spec/vcr"
+  c.hook_into :webmock
+  c.ignore_localhost = true
+end