Gemfile

@@ -1,25 +1,15 @@
 # frozen_string_literal: true
 
 source 'https://rubygems.org'
-
 git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
 
-# gem "rails"
-
+gem 'dotenv', '~> 2.7'
 gem 'httparty', '~> 0.18.0'
-
 gem 'nokogiri', '~> 1.10'
-
 gem 'pg', '~> 1.2'
-
+gem 'rack', '~> 2.2'
+gem 'rake', '~> 13.0'
+gem 'rspec', '~> 3.9'
+gem 'sassc', '~> 2.2'
 gem 'sequel', '~> 5.30'
-
 gem 'sinatra', '~> 2.0'
-
-gem 'dotenv', '~> 2.7'
-
-gem 'sassc', '~> 2.2'
-
-gem 'rake', '~> 13.0'
-
-gem 'rack', '~> 2.2'

Gemfile.lock

@@ -1,6 +1,7 @@
 GEM
   remote: https://rubygems.org/
   specs:
+    diff-lcs (1.3)
     dotenv (2.7.5)
     ffi (1.12.2)
     httparty (0.18.0)
@@ -15,21 +16,28 @@ GEM
       ruby2_keywords (~> 0.0.1)
     nokogiri (1.10.9)
       mini_portile2 (~> 2.4.0)
-    pg (1.2.2)
+    pg (1.2.3)
     rack (2.2.2)
     rack-protection (2.0.8.1)
       rack
     rake (13.0.1)
-    redis (4.1.3)
-    redis-rack (2.1.2)
-      rack (>= 2.0.8, < 3)
-      redis-store (>= 1.2, < 2)
-    redis-store (1.8.2)
-      redis (>= 4, < 5)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.1)
+      rspec-support (~> 3.9.1)
+    rspec-expectations (3.9.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.2)
     ruby2_keywords (0.0.2)
     sassc (2.2.1)
       ffi (~> 1.9)
-    sequel (5.30.0)
+    sequel (5.31.0)
     sinatra (2.0.8.1)
       mustermann (~> 1.0)
       rack (~> 2.0)
@@ -47,10 +55,10 @@ DEPENDENCIES
   pg (~> 1.2)
   rack (~> 2.2)
   rake (~> 13.0)
-  redis-rack (~> 2.1)
+  rspec (~> 3.9)
   sassc (~> 2.2)
   sequel (~> 5.30)
   sinatra (~> 2.0)
 
 BUNDLED WITH
-   1.17.3
+   2.1.4

Rakefile

@@ -9,7 +9,8 @@ task 'migrate' do
 end
 
 task 'scrape' do
-  scraper = Scraper.new(log: ENV['APP_ENV'] == 'development')
+  should_log = ENV['APP_ENV'] == 'development' || ARGV.include?('--log')
+  scraper = Scraper.new(log: should_log)
   scraper.scrape
 end
 
@@ -32,6 +33,6 @@ task 'deploy' do
   puts `rsync -rv ./Rakefile #{username}@#{hostname}:/var/www/vlv-search/`
   puts `rsync -rv ./Gemfile #{username}@#{hostname}:/var/www/vlv-search/`
   puts `rsync -rv ./Gemfile.lock #{username}@#{hostname}:/var/www/vlv-search/`
-  puts `ssh #{username}@#{hostname} 'cd /var/www/vlv-search; rake build'`
-  puts `ssh #{username}@#{hostname} '~/restart-unicorn'`
+  puts `ssh -t #{username}@#{hostname} 'bash -ic ". .profile; cd /var/www/vlv-search; rake build;"'`
+  puts `ssh -t #{username}@#{hostname} 'bash -ic ". .profile; ~/restart-unicorn"'`
 end

db/connect.rb

@@ -2,11 +2,10 @@ require 'dotenv'
 
 Dotenv.load(File.expand_path('../.env'))
 
-def connect
+DB =
   Sequel.connect(
     adapter: :postgres,
     database: ENV['DB_DATABASE'],
     user: ENV['DB_USERNAME'],
-    password: ENV['DB_PASSWORD'],
+    password: ENV['DB_PASSWORD']
   )
-end

db/scrape.rb

@@ -4,8 +4,9 @@ require 'sequel'
 require_relative '../db/connect'
 require_relative '../lib/auth'
 require_relative '../lib/fetch'
-require_relative '../lib/insert'
 require_relative '../lib/parse'
+require_relative '../lib/models/post'
+require_relative '../lib/models/thread'
 
 class Scraper
   def initialize(first: 0, last: 0, log: false)
@@ -13,7 +14,6 @@ class Scraper
     @last = last
     @log = log
     @cookie = login(ENV['VLV_USERNAME'], ENV['VLV_PASSWORD'])
-    @db = connect
   end
 
   def scrape
@@ -41,9 +41,9 @@ class Scraper
 
     t[:created_at] = Parse.thread_created_at(first_post)
 
-    thread = @db.from(:threads).first(remote_id: t[:remote_id])
+    thread = DB.from(:threads).first(remote_id: t[:remote_id])
     if thread.nil?
-      thread = Insert.thread(t, @db)
+      thread = VLV::Thread.create(t.delete_if { |k| k == :is_sticky })
       log '  Inserting thread'
     end
 
@@ -53,7 +53,7 @@ class Scraper
   def scrape_posts(thread, page)
     posts = Parse.posts(thread, page)
     last_post = posts.last
-    unless @db.from(:posts).first(remote_id: last_post[:remote_id]).nil?
+    unless DB.from(:posts).first(remote_id: last_post[:remote_id]).nil?
       log '  No new posts'
       return true
     end
@@ -62,8 +62,8 @@ class Scraper
     posts.each_with_index do |p, index|
       msg = "  Inserting post #{index + 1}/#{posts_count}"
       print msg if @log
-      if @db.from(:posts).first(remote_id: p[:remote_id]).nil?
-        Insert.post(p, @db)
+      if DB.from(:posts).first(remote_id: p[:remote_id]).nil?
+        VLV::Post.create(p)
       end
       print "\b" * msg.size unless index == posts_count - 1 if @log
     end

lib/insert.rb

@@ -1,21 +0,0 @@
-module Insert
-  def self.post(post, db)
-    db.from(:posts).insert(
-      body: post[:body],
-      created_at: post[:created_at],
-      creator: post[:creator],
-      thread_id: post[:thread_id],
-      remote_id: post[:remote_id],
-    )
-  end
-
-  def self.thread(thread, db)
-    id = db.from(:threads).insert(
-      title: thread[:title],
-      creator: thread[:creator],
-      remote_id: thread[:remote_id],
-      created_at: thread[:created_at]
-    )
-    thread.merge(id: id)
-  end
-end

lib/models/post.rb

@@ -0,0 +1,5 @@
+module VLV
+  class Post < Sequel::Model
+    many_to_one :thread
+  end
+end

lib/models/thread.rb

@@ -0,0 +1,5 @@
+module VLV
+  class Thread < Sequel::Model
+    one_to_many :posts
+  end
+end

lib/search.rb

@@ -1,9 +1,6 @@
 require 'sequel'
 
-require_relative '../db/connect'
-
 def search(params)
-  db = connect
   query = params[:q].strip
   offset = (params[:page] - 1) * 10
   username = params[:username].strip
@@ -18,16 +15,16 @@ def search(params)
       else
         'created_at DESC'
       end
-    search_threads(db, query, username, sort, offset)
+    search_threads(query, username, sort, offset)
   when 'posts'
-    search_posts(db, query, username, offset)
+    search_posts(query, username, offset)
   else
     Array.new
   end
 end
 
-def search_threads(db, query, username, sort, offset)
-  db[<<-SQL, query, username, username, offset]
+def search_threads(query, username, sort, offset)
+  DB[<<-SQL, query, username, username, offset]
     SELECT
       threads.*
     FROM threads
@@ -40,8 +37,8 @@ def search_threads(db, query, username, sort, offset)
   SQL
 end
 
-def search_posts(db, query, username, offset)
-  db[<<-SQL, query, username, username, offset]
+def search_posts(query, username, offset)
+  DB[<<-SQL, query, username, username, offset]
     SELECT
       posts.*,
       threads.title as thread_title,

spec/parser_spec.rb

@@ -0,0 +1,47 @@
+require_relative '../lib/parse'
+
+RSpec.describe 'Parser' do
+  it 'should parse threads' do
+    html = Nokogiri::HTML(<<~HTML)
+      <div class="even" id="thread_12345">
+        <ul class="list read">
+          <li class="member">
+            <span>Thread By: </span>
+            <a href="/member/view/creator1/" class="memberlink">creator1</a>
+          </li>
+          <li class="subject">
+            <span>Subject: </span>
+            <a href="/thread/view/12345/&p=999">
+              <strong>Sticky:</sticky> Thread title 1
+            </a>
+          </li>
+          <li class="posts"><span>Posts: </span>999</li>
+          <li class="lastpost">
+            <span>Last Post By:</span>
+            <a href="/member/view/lastposter1/" class="memberlink">lastposter1</a> on Fri&nbsp;Apr&nbsp;10&nbsp;2020&nbsp;01:23&nbsp;am</li>
+        </ul>
+      </div>
+      <div class="even" id="thread_123456">
+        <ul class="list read">
+          <li class="member">
+            <span>Thread By: </span>
+            <a href="/member/view/creator2/" class="memberlink">creator2</a>
+          </li>
+          <li class="subject">
+            <span>Subject: </span>
+            <a href="/thread/view/123456/&p=999">Thread title 2</a>
+          </li>
+          <li class="posts"><span>Posts: </span>999</li>
+          <li class="lastpost">
+            <span>Last Post By:</span>
+            <a href="/member/view/lastposter2/" class="memberlink">lastposter2</a> on Fri&nbsp;Apr&nbsp;10&nbsp;2020&nbsp;01:23&nbsp;am</li>
+        </ul>
+      </div>
+    HTML
+
+    expect(Parse.threads(html)).to eq([
+      {remote_id: '12345', title: 'Sticky: Thread title 1', creator: 'creator1', is_sticky: true},
+      {remote_id: '123456', title: 'Thread title 2', creator: 'creator2', is_sticky: false},
+    ])
+  end
+end

web/assets/styles/global.scss

@@ -56,3 +56,7 @@ blockquote {
 .btn--small {
   font-size: 14px;
 }
+
+.center {
+  text-align: center;
+}

web/server.rb

@@ -2,6 +2,7 @@ require 'dotenv/load'
 require 'sequel'
 require 'sinatra'
 
+require_relative '../db/connect'
 require_relative '../lib/auth'
 require_relative '../lib/search'
 
@@ -14,6 +15,11 @@ class VLVSearch < Sinatra::Base
       secret: ENV['SESSION_SECRET']
 
   set :environment, ENV['APP_ENV'] == 'production' ? :production : :development
+  set :show_exceptions, ENV['APP_ENV'] == 'development'
+
+  error 500 do
+    erb :error, { layout: :layout }
+  end
 
   get '/' do
     redirect '/login' unless signed_in?

web/views/error.erb

@@ -0,0 +1,11 @@
+<div class="center">
+  <h3>Something went wrong!</h3>
+  <p>
+    The site administrator has been notified but feel free to
+    <%= external_link(
+      'http://board.vivalavinyl.com/message/create/reddwarf',
+      'send a PM'
+    ) %>
+    as well.
+  </p>
+</div>