Browse Source

Add option to log or not

master
Dylan Baker 4 years ago
parent
commit
e1bbe85032
1 changed files with 9 additions and 6 deletions
  1. 9
    6
      db/scrape.rb

+ 9
- 6
db/scrape.rb View File

16
 end
16
 end
17
 
17
 
18
 def fetch_thread(thread, cookie)
18
 def fetch_thread(thread, cookie)
19
+  sleep(0.5)
19
   url =
20
   url =
20
     URI(
21
     URI(
21
       "http://board.vivalavinyl.com/thread/view/#{thread[:remote_id]}&ajax=true"
22
       "http://board.vivalavinyl.com/thread/view/#{thread[:remote_id]}&ajax=true"
74
   threads
75
   threads
75
 end
76
 end
76
 
77
 
77
-def scrape(first: 0, last: 0)
78
+def scrape(first: 0, last: 0, log: false)
78
   cookie = login(ENV['VLV_USERNAME'], ENV['VLV_PASSWORD'])
79
   cookie = login(ENV['VLV_USERNAME'], ENV['VLV_PASSWORD'])
79
   db = connect
80
   db = connect
80
 
81
 
82
     page = fetch_page(page_number, cookie)
83
     page = fetch_page(page_number, cookie)
83
     threads = parse_threads(page)
84
     threads = parse_threads(page)
84
     threads.each do |t|
85
     threads.each do |t|
85
-      sleep(1)
86
       is_sticky = t[:title].match(/^Sticky:/)
86
       is_sticky = t[:title].match(/^Sticky:/)
87
       next if index > 0 && is_sticky
87
       next if index > 0 && is_sticky
88
+
89
+      puts t[:title] if log
90
+
88
       page = fetch_thread(t, cookie)
91
       page = fetch_thread(t, cookie)
89
       first_post = page.at_css('.postinfo:first-child')
92
       first_post = page.at_css('.postinfo:first-child')
90
 
93
 
97
       thread = db.from(:threads).first(remote_id: t[:remote_id])
100
       thread = db.from(:threads).first(remote_id: t[:remote_id])
98
       is_new_thread = thread.nil?
101
       is_new_thread = thread.nil?
99
       if is_new_thread
102
       if is_new_thread
100
-        puts '  Inserting thread'
103
+        puts '  Inserting thread' if log
101
         id =
104
         id =
102
           db.from(:threads).insert(
105
           db.from(:threads).insert(
103
             title: t[:title],
106
             title: t[:title],
120
       posts_count = posts.size
123
       posts_count = posts.size
121
       posts.each_with_index do |p, index|
124
       posts.each_with_index do |p, index|
122
         msg = "  Inserting post #{index + 1}/#{posts_count}"
125
         msg = "  Inserting post #{index + 1}/#{posts_count}"
123
-        print msg
126
+        print msg if log
124
         if is_new_thread || db.from(:posts).first(remote_id: p[:remote_id]).nil?
127
         if is_new_thread || db.from(:posts).first(remote_id: p[:remote_id]).nil?
125
           db.from(:posts).insert(
128
           db.from(:posts).insert(
126
             body: p[:body],
129
             body: p[:body],
130
             remote_id: p[:remote_id]
133
             remote_id: p[:remote_id]
131
           )
134
           )
132
         end
135
         end
133
-        print "\b" * msg.size unless index == posts_count - 1
136
+        print "\b" * msg.size unless index == posts_count - 1 if log
134
       end
137
       end
135
 
138
 
136
-      puts
139
+      puts if log
137
     end
140
     end
138
   end
141
   end
139
 end
142
 end

Loading…
Cancel
Save