Initial commit

.bundle/config

@@ -0,0 +1,2 @@
+---
+BUNDLE_PATH: "vendor/"

.gitignore

@@ -0,0 +1,4 @@
+.env
+vendor/
+web/public/*.css
+web/public/*.js

Gemfile

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# gem "rails"
+
+gem "httparty", "~> 0.18.0"
+gem "nokogiri", "~> 1.10"
+gem "sqlite3", "~> 1.4"
+
+gem "pg", "~> 1.2"
+
+gem "sequel", "~> 5.30"
+
+gem "sinatra", "~> 2.0"
+
+gem "dotenv", "~> 2.7"
+
+gem "sassc", "~> 2.2"
+
+gem "rake", "~> 13.0"

Gemfile.lock

@@ -0,0 +1,50 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    dotenv (2.7.5)
+    ffi (1.12.2)
+    httparty (0.18.0)
+      mime-types (~> 3.0)
+      multi_xml (>= 0.5.2)
+    mime-types (3.3.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2019.1009)
+    mini_portile2 (2.4.0)
+    multi_xml (0.6.0)
+    mustermann (1.1.1)
+      ruby2_keywords (~> 0.0.1)
+    nokogiri (1.10.9)
+      mini_portile2 (~> 2.4.0)
+    pg (1.2.2)
+    rack (2.2.2)
+    rack-protection (2.0.8.1)
+      rack
+    rake (13.0.1)
+    ruby2_keywords (0.0.2)
+    sassc (2.2.1)
+      ffi (~> 1.9)
+    sequel (5.30.0)
+    sinatra (2.0.8.1)
+      mustermann (~> 1.0)
+      rack (~> 2.0)
+      rack-protection (= 2.0.8.1)
+      tilt (~> 2.0)
+    sqlite3 (1.4.2)
+    tilt (2.0.10)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  dotenv (~> 2.7)
+  httparty (~> 0.18.0)
+  nokogiri (~> 1.10)
+  pg (~> 1.2)
+  rake (~> 13.0)
+  sassc (~> 2.2)
+  sequel (~> 5.30)
+  sinatra (~> 2.0)
+  sqlite3 (~> 1.4)
+
+BUNDLED WITH
+   1.17.3

Rakefile

@@ -0,0 +1,31 @@
+require 'dotenv/load'
+require 'sassc'
+
+require_relative './db/migrate'
+require_relative './db/scrape'
+
+task 'migrate' do
+  migrate
+end
+
+task 'scrape' do
+  scrape
+end
+
+task 'build' do
+  scss = File.read('./web/assets/style.scss')
+  css = SassC::Engine.new(scss, style: :compressed).render
+  File.write('./web/public/style.css', css)
+  js = File.read('./web/assets/script.js')
+  File.write('./web/public/script.js', js)
+end
+
+task 'deploy' do
+  username = ENV['PROD_USERNAME']
+  hostname = ENV['PROD_HOSTNAME']
+  puts `rsync -rv ./db #{username}@#{hostname}:/var/www/vlv-search/`
+  puts `rsync -rv ./lib #{username}@#{hostname}:/var/www/vlv-search/`
+  puts `rsync -rv ./web #{username}@#{hostname}:/var/www/vlv-search/`
+  puts `rsync -rv ./Gemfile #{username}@#{hostname}:/var/www/vlv-search/`
+  puts `rsync -rv ./Gemfile.lock #{username}@#{hostname}:/var/www/vlv-search/`
+end

db/migrate.rb

@@ -0,0 +1,43 @@
+require 'sequel'
+
+def migrate
+  db = Sequel.connect(adapter: :postgres, database: 'vlv2')
+
+  db.create_table :threads do
+    primary_key :id
+    String :title
+    String :creator, index: true
+    Integer :remote_id, index: true, unique: true
+    String :last_post_creator
+    DateTime :last_post_created_at
+    DateTime :created_at, index: true
+  end
+
+  db.create_table :posts do
+    primary_key :id
+    String :body
+    String :creator, index: true
+    Integer :remote_id, index: true, unique: true
+    foreign_key :thread_id, :threads
+    DateTime :created_at, index: true
+  end
+
+  db.create_function(:update_last_post, <<-SQL, language: :plpgsql, returns: :trigger)
+    BEGIN
+      UPDATE threads
+      SET
+        last_post_created_at = NEW.created_at,
+        last_post_creator = NEW.creator
+      WHERE
+        threads.id = NEW.thread_id;
+      RETURN NEW;
+    END
+  SQL
+
+  db.run(<<-SQL)
+    CREATE TRIGGER update_last_post
+    AFTER INSERT ON posts
+    FOR EACH ROW
+    EXECUTE PROCEDURE update_last_post();
+  SQL
+end

db/scrape.rb

@@ -0,0 +1,137 @@
+require 'dotenv/load'
+require 'net/http'
+require 'nokogiri'
+require 'sequel'
+
+require_relative '../lib/auth'
+
+def fetch_page(page_number, cookie)
+  url = URI("http://board.vivalavinyl.com/thread/list/#{page_number}")
+  http = Net::HTTP.new(url.host, url.port)
+  request = Net::HTTP::Get.new(url)
+  request['cookie'] = cookie
+  response = http.request(request)
+  Nokogiri.HTML(response.body)
+end
+
+def fetch_thread(thread, cookie)
+  url =
+    URI(
+      "http://board.vivalavinyl.com/thread/view/#{thread[:remote_id]}&ajax=true"
+    )
+  http = Net::HTTP.new(url.host, url.port)
+  request = Net::HTTP::Get.new(url)
+  request['cookie'] = cookie
+  response = http.request(request)
+  Nokogiri.HTML(response.body)
+end
+
+def parse_posts(thread, page)
+  posts = Array.new
+
+  page.css('.post').each do |_post|
+    post = Hash.new
+    post[:remote_id] = _post.at_css('ul.view')[:id].split('_')[1].to_i
+    post[:creator] = _post.at_css('.memberlink').text.strip
+    date, time =
+      _post.at_css('.postinfo').text.split('posted this')[1].split('@')
+    post[:created_at] = Time.parse("#{date} #{time}")
+    post[:body] = _post.at_css('.postbody').children.map(&:to_html).join.strip
+    post[:thread_id] = thread[:id]
+    posts << post
+  end
+
+  posts
+end
+
+def insert_post(post, db)
+  db.exec(
+    'insert into posts (body, timestamp, creator, thread_id, remote_id) values ($1, $2, $3, $4, $5)',
+    [
+      post[:body],
+      post[:created_at].to_s,
+      post[:creator],
+      post[:thread_id].to_i,
+      post[:id].to_i
+    ]
+  )
+end
+
+def parse_threads(page)
+  threads = Array.new
+  page.css('.even, .odd').each do |row|
+    thread = Hash.new
+    thread_link = row.at_css('.subject > a')
+    next if thread_link.nil?
+    thread[:remote_id] = thread_link['href'].split('/')[3]
+    thread[:title] = thread_link.text
+    creator = row.at_css('.memberlink').text.strip
+    creator = creator[0..-2] if creator.match(/\+$/)
+    thread[:creator] = creator
+    threads << thread
+  end
+  threads
+end
+
+def scrape(first: 0, last: 0)
+  cookie = login(ENV['VLV_USERNAME'], ENV['VLV_PASSWORD'])
+  db = Sequel.connect(adapter: :postgres, database: ENV['DB_DATABASE'])
+
+  (first..last).each do |page_number|
+    page = fetch_page(page_number, cookie)
+    threads = parse_threads(page)
+    threads.each do |t|
+      puts t[:title]
+
+      page = fetch_thread(t, cookie)
+      first_post = page.at_css('.postinfo:first-child')
+
+      next if first_post.nil?
+
+      post_info = first_post.text.split('posted this')
+      date, time = post_info[1].split('@')
+      t[:created_at] = Time.parse("#{date} #{time}")
+
+      thread = db.from(:threads).first(remote_id: t[:remote_id])
+      is_new_thread = thread.nil?
+      if is_new_thread
+        puts '  Inserting thread'
+        id =
+          db.from(:threads).insert(
+            title: t[:title],
+            creator: t[:creator],
+            remote_id: t[:remote_id],
+            created_at: t[:created_at]
+          )
+        t[:id] = id
+        thread = t
+      end
+
+      posts = parse_posts(thread, page)
+
+      last_post = posts.last
+      unless db.from(:posts).first(remote_id: last_post[:remote_id]).nil?
+        puts '  Up to date, skipping'
+        next
+      end
+
+      posts_count = posts.size
+      posts.each_with_index do |p, index|
+        msg = "  Inserting post #{index + 1}/#{posts_count}"
+        print msg
+        if is_new_thread || db.from(:posts).first(remote_id: p[:remote_id]).nil?
+          db.from(:posts).insert(
+            body: p[:body],
+            created_at: p[:created_at],
+            thread_id: p[:thread_id],
+            creator: p[:creator],
+            remote_id: p[:remote_id]
+          )
+        end
+        print "\b" * msg.size unless index == posts_count - 1
+      end
+
+      puts
+    end
+  end
+end

lib/auth.rb

@@ -0,0 +1,12 @@
+require 'net/http'
+
+def login(username, password)
+  url = URI('http://board.vivalavinyl.com/main/login')
+  http = Net::HTTP.new(url.host, url.port)
+  request = Net::HTTP::Post.new(url)
+  request.body = "name=#{username}&pass=#{password}&login=login"
+  response = http.request(request)
+  cookies = response.response['set-cookie']
+  return nil if cookies.nil?
+  cookies.split('; ').first
+end

lib/search.rb

@@ -0,0 +1,58 @@
+require 'sequel'
+
+def search(params)
+  db = Sequel.connect(adapter: :postgres, database: ENV['DB_DATABASE'])
+  query = params[:q].strip
+  offset = (params[:page] - 1) * 10
+  username = params[:username].strip
+  case params[:type]
+  when 'threads'
+    sort =
+      case params[:sort]
+      when 'thread'
+        'threads.created_at DESC'
+      when 'post'
+        'posts.created_at DESC'
+      else
+        'threads.created_at DESC'
+      end
+    search_threads(db, query, username, sort, offset)
+  when 'posts'
+    search_posts(db, query, username, offset)
+  else
+    Array.new
+  end
+end
+
+def search_threads(db, query, username, sort, offset)
+  db[<<-SQL, query, username, username, String.new, offset]
+    SELECT
+      threads.*
+    FROM threads
+    WHERE
+      to_tsvector(title) @@ plainto_tsquery(?)
+      AND (LOWER(threads.creator) = LOWER(?) OR ? = '')
+    ORDER BY #{sort}
+    LIMIT 10
+    OFFSET ?;
+  SQL
+end
+
+def search_posts(db, query, username, offset)
+  db[<<-SQL, query, username, username, offset]
+    SELECT
+      posts.*,
+      threads.title as thread_title,
+      threads.remote_id as remote_thread_id
+    FROM posts
+    INNER JOIN threads on posts.thread_id = threads.id
+    WHERE
+      to_tsvector(body) @@ plainto_tsquery(?)
+      AND (
+        (LOWER(posts.creator) = LOWER(?)) OR (? = '')
+      )
+    ORDER BY created_at DESC
+    LIMIT 10
+    OFFSET ?;
+  SQL
+end

web/assets/script.js

@@ -0,0 +1,6 @@
+var types = document.querySelectorAll('input[type=radio][name=type]');
+types.forEach(function (type) {
+  type.addEventListener('change', function (e) {
+    document.querySelector('.filters__section--sort').classList.toggle('open');
+  });
+});

web/assets/style.scss

@@ -0,0 +1,4 @@
+@import './web/assets/styles/auth.scss';
+@import './web/assets/styles/form.scss';
+@import './web/assets/styles/global.scss';
+@import './web/assets/styles/results.scss';

web/assets/styles/auth.scss

@@ -0,0 +1,14 @@
+.auth {
+  align-items: center;
+  display: flex;
+  font-size: 14px;
+  justify-content: space-between;
+}
+
+.auth__copy {
+  margin: 0;
+}
+
+.auth__username {
+  font-weight: bold;
+}

web/assets/styles/form.scss

@@ -0,0 +1,51 @@
+.form {
+  display: flex;
+  flex-direction: column;
+  margin: 2em auto;
+  width: 100%;
+}
+
+.form__search {
+  border: 1px solid #999;
+  border-radius: 5px;
+  font-size: 20px;
+  flex: 1;
+  padding: 0.5em;
+}
+
+.form .filters {
+  margin: 1em 0;
+}
+
+.filters__section--sort {
+  display: none;
+}
+
+.filters__section--sort.open {
+  display: block;
+}
+
+.form__field {
+  margin: 10px 0;
+}
+
+.form__text-field {
+  border: 1px solid #000000;
+  border-radius: 4px;
+  font-size: 16px;
+  padding: 5px;
+  width: 100%;
+}
+
+.form__submit {
+  font-size: 16px;
+}
+
+.form--login {
+  margin-top: 0;
+}
+
+.form--login .form__label {
+  display: block;
+  margin: 10px 0;
+}

web/assets/styles/global.scss

@@ -0,0 +1,49 @@
+* {
+  box-sizing: border-box;
+}
+
+body {
+  font-family: verdana, sans-serif;
+}
+
+img {
+  max-width: 100%;
+}
+
+blockquote {
+  font-style: italic;
+  margin: 0;
+}
+
+.error {
+  color: #DD2222;
+}
+
+.container {
+  margin: auto;
+  max-width: 800px;
+}
+
+.header__link {
+  color: black;
+  text-decoration: none;
+}
+
+.btn {
+  background-color: rgb(40, 96, 144);
+  border: 1px solid #222;
+  border-radius: 5px;
+  color: #ffffff;
+  cursor: pointer;
+  font-size: 20px;
+  padding: 0.5em 0.5em;
+  text-decoration: none;
+}
+
+.btn:hover {
+  background-color: rgb(30, 85, 135);
+}
+
+.btn--small {
+  font-size: 14px;
+}

web/assets/styles/results.scss

@@ -0,0 +1,42 @@
+.results-container {
+  margin: auto;
+  max-width: 800px;
+  padding-bottom: 1em;
+}
+
+.results__subheader {
+  border-bottom: 1px solid black;
+  margin: 1.5em 0;
+  padding: 5px 0;
+}
+
+.results__result {
+  background: #efefef;
+  border: 1px solid black;
+  border-radius: 5px;
+  margin: 1em 0;
+  padding: 0.5em;
+}
+
+.results__result.threads .result__info {
+  margin-bottom: 0;
+}
+
+.result__info {
+  display: flex;
+  font-size: 12px;
+  justify-content: space-between;
+
+  @media(max-width: 500px) {
+    flex-direction: column;
+
+    span {
+      display: inline-block;
+      margin: 5px 0;
+    }
+  }
+}
+
+.btn--prev {
+  margin-right: 1em;
+}

web/server.rb

@@ -0,0 +1,84 @@
+require 'dotenv/load'
+require 'sequel'
+require 'sinatra'
+
+require_relative '../lib/auth'
+require_relative '../lib/search'
+
+class VLVSearch < Sinatra::Base
+  enable :sessions
+
+  get '/' do
+    redirect '/login' unless signed_in?
+    params[:type] = 'threads'
+    erb :index, { locals: params, layout: :layout }
+  end
+
+  get '/search' do
+    redirect '/login' unless signed_in?
+    params[:type] = 'threads' unless params[:type]
+    params[:page] = 1 unless params[:page]
+    params[:page] = params[:page].to_i
+
+    results = search(params)
+    previous_url, next_url = build_urls(params)
+
+    locals =
+      params.merge(
+        results: results, previous_url: previous_url, next_url: next_url
+      )
+
+    erb :results, { locals: locals, layout: :layout }
+  end
+
+  get '/login' do
+    erb :login, { layout: :layout, locals: { error_message: nil } }
+  end
+
+  post '/login' do
+    username = params[:username]
+    password = params[:password]
+    cookie = login(username, password)
+
+    if cookie.nil?
+      erb :login,
+          { layout: :layout, locals: { error_message: 'Invalid credentials' } }
+    else
+      session[:user_id] = username
+      redirect '/'
+    end
+  end
+
+  post '/logout' do
+    session[:user_id] = nil
+    redirect '/'
+  end
+
+  helpers do
+    def build_urls(params)
+      current_page = params[:page].to_i
+      previous_page = current_page > 1 ? current_page - 1 : nil
+      next_page = current_page + 1
+
+      url_params = { q: params[:q], type: params[:type] }
+      url_params[:username] = params[:username] if params[:username]
+
+      [previous_page, next_page].map do |page|
+        URI::Generic.build(
+          path: '/search',
+          query: URI.encode_www_form(url_params.merge(page: page))
+        )
+      end
+    end
+
+    def current_user
+      session[:user_id]
+    end
+
+    def signed_in?
+      !!session[:user_id]
+    end
+
+    run!
+  end
+end

web/views/index.erb

@@ -0,0 +1 @@
+<%= erb :'partials/search' %>

web/views/layout.erb

@@ -0,0 +1,17 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <title>vlv search</title>
+    <link rel="stylesheet" href="/style.css">
+  </head>
+  <body>
+    <div class="container">
+      <h1 class="header">
+        <a href="/" class="header__link">vlv search</a>
+      </h1>
+      <%= erb :'partials/auth' %>
+      <%= yield %>
+    </div>
+    <script src="/script.js"></script>
+  </body>
+</html>

web/views/login.erb

@@ -0,0 +1,19 @@
+<div class="container">
+  <h2 class="container__heading">Login</h2>
+  <form class="form form--login" action="/login" method="POST">
+    <% if error_message %>
+      <p class="error"><%= error_message %></p>
+    <% end %>
+    <div class="form__field">
+      <label class="form__label">Username</label>
+      <input type="text" name="username" class="form__text-field" value="<%= params[:username] %>">
+    </div>
+    <div class="form__field">
+      <label class="form__label">Password</label>
+      <input type="password" name="password" class="form__text-field" value="<%= params[:password] %>">
+    </div>
+    <div class="form__field">
+      <input type="submit" value="Login" class="btn form__submit">
+    </div>
+  </form>
+</div>

web/views/partials/auth.erb

@@ -0,0 +1,8 @@
+<% if signed_in? %>
+  <form class="auth" method="POST" action="/logout">
+    <p class="auth__copy">
+      Logged in as <span class="auth__username"><%= current_user %></span>
+    </p>
+    <input type="submit" class="auth__logout btn btn--small" value="Logout">
+  </form>
+<% end %>

web/views/partials/search.erb

@@ -0,0 +1,34 @@
+<div class="search">
+  <form action="/search" method="get" class="form">
+    <input name="q" type="search" value="<%= params[:q] %>" placeholder="Search for..." class="form__search" required>
+    <div class="filters">
+      <div class="filters__section">
+        <p>Search in:</p>
+        <label class="form__label">
+          <input type="radio" name="type" value="threads" <% if [nil, "threads"].include?(params[:type]) %>checked<% end%>>
+          Threads
+        </label>
+        <label class="form__label">
+          <input type="radio" name="type" value="posts" <% if params[:type] == "posts" %>checked<% end %>>
+          Posts
+        </label>
+      </div>
+      <div class="filters__section filters__section--sort <%= params[:type] == 'threads' ? 'open' : '' %>">
+        <p>Sort by:</p>
+        <label class="form__label">
+          <input type="radio" name="sort" value="thread" <% if [nil, "thread"].include? params[:sort] %>checked<% end%>>
+          Thread creation
+        </label>
+        <label class="form__label">
+          <input type="radio" name="sort" value="post" <% if params[:sort] == "post" %>checked<% end %>>
+          Most recent post
+        </label>
+      </div>
+      <div class="filters__section">
+        <p>Posted by:</p>
+        <input type="text" name="username" value="<%= params[:username] %>" class="form__text-field">
+      </div>
+    </div>
+    <input type="submit" value="Search" class="btn form__submit">
+  </form>
+</div>

web/views/results.erb

@@ -0,0 +1,57 @@
+<%= erb :'partials/search' %>
+
+<div class="results-container">
+  <% if params[:q] %>
+    <p>Showing results for <strong><em><%= params[:q] %></em></strong>:</p>
+  <% end %>
+  <div class="results">
+    <% if results.empty? %>
+      <h3>No results!</h3>
+    <% end %>
+    <% results.each do |result| %>
+      <div class="results__result <%= params[:type] %>">
+        <% if params[:type] == "threads" %>
+          <a href="http://board.vivalavinyl.com/thread/view/<%= result[:remote_id] %>">
+            <%= result[:title] %>
+          </a>
+          <p class="result__info">
+            <span>
+              created by <%= result[:creator] %>
+              on <%= result[:created_at].strftime('%B %d, %Y') %>
+            </span>
+            <span>
+              last post by <%= result[:last_post_creator] %>
+              at <%= result[:last_post_created_at].strftime('%-I:%M %p EST') %>
+              on <%= result[:last_post_created_at].strftime('%B %-d, %Y') %>
+            </span>
+          </p>
+        <% elsif params[:type] == "posts" %>
+          <p>
+            In thread:
+            <a href="http://board.vivalavinyl.com/thread/view/<%= result[:remote_thread_id] %>">
+              <strong><%= result[:thread_title] %></strong>
+            </a>
+          </p>
+          <p class="result__info">
+            <%= result[:creator] %> posted this at <%= result[:created_at] %>
+          </p>
+          <p class="result__body">
+            <%= result[:body] %>
+          </p>
+        <% end %>
+      </div>
+    <% end %>
+
+    <% if params[:q] %>
+      <% if params[:page] > 1 %>
+        <a class="btn btn--small btn--prev" href="<%= previous_url %>">
+          previous page
+        </a>
+      <% end %>
+
+      <a class="btn btn--small btn--next" href="<%= next_url %>">
+        next page
+      </a>
+    <% end %>
+  </div>
+</div>