9 Commits

Author SHA1 Message Date
  Dylan Baker 645b249df3 Allow forcing log 4 years ago
  Dylan Baker 448cdc02db Reading comprehension 4 years ago
  Dylan Baker df4a96dc05 Creator error page 4 years ago
  Dylan Baker 88eae898ed Update web/search for DB constant 4 years ago
  Dylan Baker 3c82d382ef Add a test for parsing 4 years ago
  Dylan Baker 585828341d Install rspec 4 years ago
  Dylan Baker d4cff431b9 Use DB constant and sequel models 4 years ago
  Dylan Baker 9eb85106ac Update 4 years ago
  Dylan Baker cb1717ca65 Fix deploy script 4 years ago
13 changed files with 120 additions and 68 deletions
  1. 5
    15
      Gemfile
  2. 18
    10
      Gemfile.lock
  3. 4
    3
      Rakefile
  4. 2
    3
      db/connect.rb
  5. 7
    7
      db/scrape.rb
  6. 0
    21
      lib/insert.rb
  7. 5
    0
      lib/models/post.rb
  8. 5
    0
      lib/models/thread.rb
  9. 6
    9
      lib/search.rb
  10. 47
    0
      spec/parser_spec.rb
  11. 4
    0
      web/assets/styles/global.scss
  12. 6
    0
      web/server.rb
  13. 11
    0
      web/views/error.erb

+ 5
- 15
Gemfile View File

@@ -1,25 +1,15 @@
1 1
 # frozen_string_literal: true
2 2
 
3 3
 source 'https://rubygems.org'
4
-
5 4
 git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
6 5
 
7
-# gem "rails"
8
-
6
+gem 'dotenv', '~> 2.7'
9 7
 gem 'httparty', '~> 0.18.0'
10
-
11 8
 gem 'nokogiri', '~> 1.10'
12
-
13 9
 gem 'pg', '~> 1.2'
14
-
10
+gem 'rack', '~> 2.2'
11
+gem 'rake', '~> 13.0'
12
+gem 'rspec', '~> 3.9'
13
+gem 'sassc', '~> 2.2'
15 14
 gem 'sequel', '~> 5.30'
16
-
17 15
 gem 'sinatra', '~> 2.0'
18
-
19
-gem 'dotenv', '~> 2.7'
20
-
21
-gem 'sassc', '~> 2.2'
22
-
23
-gem 'rake', '~> 13.0'
24
-
25
-gem 'rack', '~> 2.2'

+ 18
- 10
Gemfile.lock View File

@@ -1,6 +1,7 @@
1 1
 GEM
2 2
   remote: https://rubygems.org/
3 3
   specs:
4
+    diff-lcs (1.3)
4 5
     dotenv (2.7.5)
5 6
     ffi (1.12.2)
6 7
     httparty (0.18.0)
@@ -15,21 +16,28 @@ GEM
15 16
       ruby2_keywords (~> 0.0.1)
16 17
     nokogiri (1.10.9)
17 18
       mini_portile2 (~> 2.4.0)
18
-    pg (1.2.2)
19
+    pg (1.2.3)
19 20
     rack (2.2.2)
20 21
     rack-protection (2.0.8.1)
21 22
       rack
22 23
     rake (13.0.1)
23
-    redis (4.1.3)
24
-    redis-rack (2.1.2)
25
-      rack (>= 2.0.8, < 3)
26
-      redis-store (>= 1.2, < 2)
27
-    redis-store (1.8.2)
28
-      redis (>= 4, < 5)
24
+    rspec (3.9.0)
25
+      rspec-core (~> 3.9.0)
26
+      rspec-expectations (~> 3.9.0)
27
+      rspec-mocks (~> 3.9.0)
28
+    rspec-core (3.9.1)
29
+      rspec-support (~> 3.9.1)
30
+    rspec-expectations (3.9.1)
31
+      diff-lcs (>= 1.2.0, < 2.0)
32
+      rspec-support (~> 3.9.0)
33
+    rspec-mocks (3.9.1)
34
+      diff-lcs (>= 1.2.0, < 2.0)
35
+      rspec-support (~> 3.9.0)
36
+    rspec-support (3.9.2)
29 37
     ruby2_keywords (0.0.2)
30 38
     sassc (2.2.1)
31 39
       ffi (~> 1.9)
32
-    sequel (5.30.0)
40
+    sequel (5.31.0)
33 41
     sinatra (2.0.8.1)
34 42
       mustermann (~> 1.0)
35 43
       rack (~> 2.0)
@@ -47,10 +55,10 @@ DEPENDENCIES
47 55
   pg (~> 1.2)
48 56
   rack (~> 2.2)
49 57
   rake (~> 13.0)
50
-  redis-rack (~> 2.1)
58
+  rspec (~> 3.9)
51 59
   sassc (~> 2.2)
52 60
   sequel (~> 5.30)
53 61
   sinatra (~> 2.0)
54 62
 
55 63
 BUNDLED WITH
56
-   1.17.3
64
+   2.1.4

+ 4
- 3
Rakefile View File

@@ -9,7 +9,8 @@ task 'migrate' do
9 9
 end
10 10
 
11 11
 task 'scrape' do
12
-  scraper = Scraper.new(log: ENV['APP_ENV'] == 'development')
12
+  should_log = ENV['APP_ENV'] == 'development' || ARGV.include?('--log')
13
+  scraper = Scraper.new(log: should_log)
13 14
   scraper.scrape
14 15
 end
15 16
 
@@ -32,6 +33,6 @@ task 'deploy' do
32 33
   puts `rsync -rv ./Rakefile #{username}@#{hostname}:/var/www/vlv-search/`
33 34
   puts `rsync -rv ./Gemfile #{username}@#{hostname}:/var/www/vlv-search/`
34 35
   puts `rsync -rv ./Gemfile.lock #{username}@#{hostname}:/var/www/vlv-search/`
35
-  puts `ssh #{username}@#{hostname} 'cd /var/www/vlv-search; rake build'`
36
-  puts `ssh #{username}@#{hostname} '~/restart-unicorn'`
36
+  puts `ssh -t #{username}@#{hostname} 'bash -ic ". .profile; cd /var/www/vlv-search; rake build;"'`
37
+  puts `ssh -t #{username}@#{hostname} 'bash -ic ". .profile; ~/restart-unicorn"'`
37 38
 end

+ 2
- 3
db/connect.rb View File

@@ -2,11 +2,10 @@ require 'dotenv'
2 2
 
3 3
 Dotenv.load(File.expand_path('../.env'))
4 4
 
5
-def connect
5
+DB =
6 6
   Sequel.connect(
7 7
     adapter: :postgres,
8 8
     database: ENV['DB_DATABASE'],
9 9
     user: ENV['DB_USERNAME'],
10
-    password: ENV['DB_PASSWORD'],
10
+    password: ENV['DB_PASSWORD']
11 11
   )
12
-end

+ 7
- 7
db/scrape.rb View File

@@ -4,8 +4,9 @@ require 'sequel'
4 4
 require_relative '../db/connect'
5 5
 require_relative '../lib/auth'
6 6
 require_relative '../lib/fetch'
7
-require_relative '../lib/insert'
8 7
 require_relative '../lib/parse'
8
+require_relative '../lib/models/post'
9
+require_relative '../lib/models/thread'
9 10
 
10 11
 class Scraper
11 12
   def initialize(first: 0, last: 0, log: false)
@@ -13,7 +14,6 @@ class Scraper
13 14
     @last = last
14 15
     @log = log
15 16
     @cookie = login(ENV['VLV_USERNAME'], ENV['VLV_PASSWORD'])
16
-    @db = connect
17 17
   end
18 18
 
19 19
   def scrape
@@ -41,9 +41,9 @@ class Scraper
41 41
 
42 42
     t[:created_at] = Parse.thread_created_at(first_post)
43 43
 
44
-    thread = @db.from(:threads).first(remote_id: t[:remote_id])
44
+    thread = DB.from(:threads).first(remote_id: t[:remote_id])
45 45
     if thread.nil?
46
-      thread = Insert.thread(t, @db)
46
+      thread = VLV::Thread.create(t.delete_if { |k| k == :is_sticky })
47 47
       log '  Inserting thread'
48 48
     end
49 49
 
@@ -53,7 +53,7 @@ class Scraper
53 53
   def scrape_posts(thread, page)
54 54
     posts = Parse.posts(thread, page)
55 55
     last_post = posts.last
56
-    unless @db.from(:posts).first(remote_id: last_post[:remote_id]).nil?
56
+    unless DB.from(:posts).first(remote_id: last_post[:remote_id]).nil?
57 57
       log '  No new posts'
58 58
       return true
59 59
     end
@@ -62,8 +62,8 @@ class Scraper
62 62
     posts.each_with_index do |p, index|
63 63
       msg = "  Inserting post #{index + 1}/#{posts_count}"
64 64
       print msg if @log
65
-      if @db.from(:posts).first(remote_id: p[:remote_id]).nil?
66
-        Insert.post(p, @db)
65
+      if DB.from(:posts).first(remote_id: p[:remote_id]).nil?
66
+        VLV::Post.create(p)
67 67
       end
68 68
       print "\b" * msg.size unless index == posts_count - 1 if @log
69 69
     end

+ 0
- 21
lib/insert.rb View File

@@ -1,21 +0,0 @@
1
-module Insert
2
-  def self.post(post, db)
3
-    db.from(:posts).insert(
4
-      body: post[:body],
5
-      created_at: post[:created_at],
6
-      creator: post[:creator],
7
-      thread_id: post[:thread_id],
8
-      remote_id: post[:remote_id],
9
-    )
10
-  end
11
-
12
-  def self.thread(thread, db)
13
-    id = db.from(:threads).insert(
14
-      title: thread[:title],
15
-      creator: thread[:creator],
16
-      remote_id: thread[:remote_id],
17
-      created_at: thread[:created_at]
18
-    )
19
-    thread.merge(id: id)
20
-  end
21
-end

+ 5
- 0
lib/models/post.rb View File

@@ -0,0 +1,5 @@
1
+module VLV
2
+  class Post < Sequel::Model
3
+    many_to_one :thread
4
+  end
5
+end

+ 5
- 0
lib/models/thread.rb View File

@@ -0,0 +1,5 @@
1
+module VLV
2
+  class Thread < Sequel::Model
3
+    one_to_many :posts
4
+  end
5
+end

+ 6
- 9
lib/search.rb View File

@@ -1,9 +1,6 @@
1 1
 require 'sequel'
2 2
 
3
-require_relative '../db/connect'
4
-
5 3
 def search(params)
6
-  db = connect
7 4
   query = params[:q].strip
8 5
   offset = (params[:page] - 1) * 10
9 6
   username = params[:username].strip
@@ -18,16 +15,16 @@ def search(params)
18 15
       else
19 16
         'created_at DESC'
20 17
       end
21
-    search_threads(db, query, username, sort, offset)
18
+    search_threads(query, username, sort, offset)
22 19
   when 'posts'
23
-    search_posts(db, query, username, offset)
20
+    search_posts(query, username, offset)
24 21
   else
25 22
     Array.new
26 23
   end
27 24
 end
28 25
 
29
-def search_threads(db, query, username, sort, offset)
30
-  db[<<-SQL, query, username, username, offset]
26
+def search_threads(query, username, sort, offset)
27
+  DB[<<-SQL, query, username, username, offset]
31 28
     SELECT
32 29
       threads.*
33 30
     FROM threads
@@ -40,8 +37,8 @@ def search_threads(db, query, username, sort, offset)
40 37
   SQL
41 38
 end
42 39
 
43
-def search_posts(db, query, username, offset)
44
-  db[<<-SQL, query, username, username, offset]
40
+def search_posts(query, username, offset)
41
+  DB[<<-SQL, query, username, username, offset]
45 42
     SELECT
46 43
       posts.*,
47 44
       threads.title as thread_title,

+ 47
- 0
spec/parser_spec.rb View File

@@ -0,0 +1,47 @@
1
+require_relative '../lib/parse'
2
+
3
+RSpec.describe 'Parser' do
4
+  it 'should parse threads' do
5
+    html = Nokogiri::HTML(<<~HTML)
6
+      <div class="even" id="thread_12345">
7
+        <ul class="list read">
8
+          <li class="member">
9
+            <span>Thread By: </span>
10
+            <a href="/member/view/creator1/" class="memberlink">creator1</a>
11
+          </li>
12
+          <li class="subject">
13
+            <span>Subject: </span>
14
+            <a href="/thread/view/12345/&p=999">
15
+              <strong>Sticky:</sticky> Thread title 1
16
+            </a>
17
+          </li>
18
+          <li class="posts"><span>Posts: </span>999</li>
19
+          <li class="lastpost">
20
+            <span>Last Post By:</span>
21
+            <a href="/member/view/lastposter1/" class="memberlink">lastposter1</a> on Fri&nbsp;Apr&nbsp;10&nbsp;2020&nbsp;01:23&nbsp;am</li>
22
+        </ul>
23
+      </div>
24
+      <div class="even" id="thread_123456">
25
+        <ul class="list read">
26
+          <li class="member">
27
+            <span>Thread By: </span>
28
+            <a href="/member/view/creator2/" class="memberlink">creator2</a>
29
+          </li>
30
+          <li class="subject">
31
+            <span>Subject: </span>
32
+            <a href="/thread/view/123456/&p=999">Thread title 2</a>
33
+          </li>
34
+          <li class="posts"><span>Posts: </span>999</li>
35
+          <li class="lastpost">
36
+            <span>Last Post By:</span>
37
+            <a href="/member/view/lastposter2/" class="memberlink">lastposter2</a> on Fri&nbsp;Apr&nbsp;10&nbsp;2020&nbsp;01:23&nbsp;am</li>
38
+        </ul>
39
+      </div>
40
+    HTML
41
+
42
+    expect(Parse.threads(html)).to eq([
43
+      {remote_id: '12345', title: 'Sticky: Thread title 1', creator: 'creator1', is_sticky: true},
44
+      {remote_id: '123456', title: 'Thread title 2', creator: 'creator2', is_sticky: false},
45
+    ])
46
+  end
47
+end

+ 4
- 0
web/assets/styles/global.scss View File

@@ -56,3 +56,7 @@ blockquote {
56 56
 .btn--small {
57 57
   font-size: 14px;
58 58
 }
59
+
60
+.center {
61
+  text-align: center;
62
+}

+ 6
- 0
web/server.rb View File

@@ -2,6 +2,7 @@ require 'dotenv/load'
2 2
 require 'sequel'
3 3
 require 'sinatra'
4 4
 
5
+require_relative '../db/connect'
5 6
 require_relative '../lib/auth'
6 7
 require_relative '../lib/search'
7 8
 
@@ -14,6 +15,11 @@ class VLVSearch < Sinatra::Base
14 15
       secret: ENV['SESSION_SECRET']
15 16
 
16 17
   set :environment, ENV['APP_ENV'] == 'production' ? :production : :development
18
+  set :show_exceptions, ENV['APP_ENV'] == 'development'
19
+
20
+  error 500 do
21
+    erb :error, { layout: :layout }
22
+  end
17 23
 
18 24
   get '/' do
19 25
     redirect '/login' unless signed_in?

+ 11
- 0
web/views/error.erb View File

@@ -0,0 +1,11 @@
1
+<div class="center">
2
+  <h3>Something went wrong!</h3>
3
+  <p>
4
+    The site administrator has been notified but feel free to
5
+    <%= external_link(
6
+      'http://board.vivalavinyl.com/message/create/reddwarf',
7
+      'send a PM'
8
+    ) %>
9
+    as well.
10
+  </p>
11
+</div>

Loading…
Cancel
Save