Browse Source

Parse stickiness in parse_threads

master
Dylan Baker 4 years ago
parent
commit
e46a9fa449
1 changed files with 4 additions and 4 deletions
  1. 4
    4
      db/scrape.rb

+ 4
- 4
db/scrape.rb View File

@@ -66,10 +66,11 @@ def parse_threads(page)
66 66
     thread_link = row.at_css('.subject > a')
67 67
     next if thread_link.nil?
68 68
     thread[:remote_id] = thread_link['href'].split('/')[3]
69
-    thread[:title] = thread_link.text
69
+    thread[:title] = thread_link.text.strip
70 70
     creator = row.at_css('.memberlink').text.strip
71 71
     creator = creator[0..-2] if creator.match(/\+$/)
72 72
     thread[:creator] = creator
73
+    thread[:is_sticky] = !!thread[:title].match(/^Sticky:/)
73 74
     threads << thread
74 75
   end
75 76
   threads
@@ -83,8 +84,7 @@ def scrape(first: 0, last: 0, log: false)
83 84
     page = fetch_page(page_number, cookie)
84 85
     threads = parse_threads(page)
85 86
     threads.each do |t|
86
-      is_sticky = t[:title].match(/^Sticky:/)
87
-      next if index > 0 && is_sticky
87
+      next if index > 0 && t[:is_sticky]
88 88
 
89 89
       puts t[:title] if log
90 90
 
@@ -116,7 +116,7 @@ def scrape(first: 0, last: 0, log: false)
116 116
 
117 117
       last_post = posts.last
118 118
       unless db.from(:posts).first(remote_id: last_post[:remote_id]).nil?
119
-        next if is_sticky
119
+        next if t[:is_sticky]
120 120
         break
121 121
       end
122 122
 

Loading…
Cancel
Save