Class: Jobs::ReindexSearch

Inherits:
Scheduled show all
Defined in:
app/jobs/scheduled/reindex_search.rb

Instance Method Summary collapse

Methods inherited from Scheduled

#perform

Methods inherited from Base

acquire_cluster_concurrency_lock!, clear_cluster_concurrency_lock!, cluster_concurrency, cluster_concurrency_redis_key, delayed_perform, #error_context, get_cluster_concurrency, #last_db_duration, #log, #perform, #perform_immediately

Instance Method Details

#clean_posts ⇒ Object 



93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
 
# File 'app/jobs/scheduled/reindex_search.rb', line 93

def clean_posts
  puts "cleaning up post search data" if @verbose

  # remove search data from deleted/empty posts

  DB.exec(<<~SQL, deleted_at: @cleanup_grace_period)
    DELETE FROM post_search_data
     WHERE post_id IN (
      SELECT post_id
        FROM post_search_data
   LEFT JOIN posts ON post_id = posts.id
        JOIN topics ON posts.topic_id = topics.id
       WHERE posts.id IS NULL
          OR posts.raw = ''
          OR (posts.deleted_at IS NOT NULL AND posts.deleted_at <= :deleted_at)
          OR (topics.deleted_at IS NOT NULL AND topics.deleted_at <= :deleted_at)
      )
  SQL
end

#clean_topics ⇒ Object 



76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
 
# File 'app/jobs/scheduled/reindex_search.rb', line 76

def clean_topics
  puts "cleaning up topic search data" if @verbose

  # remove search data from deleted topics

  DB.exec(<<~SQL, deleted_at: @cleanup_grace_period)
    DELETE FROM topic_search_data
     WHERE topic_id IN (
      SELECT topic_id
        FROM topic_search_data
   LEFT JOIN topics ON topic_id = topics.id
       WHERE topics.id IS NULL
          OR (deleted_at IS NOT NULL AND deleted_at <= :deleted_at)
      )
  SQL
end

#execute(args) ⇒ Object 



7
8
9
10
11
12
13
14
15
16
17
18
19
 
# File 'app/jobs/scheduled/reindex_search.rb', line 7

def execute(args)
  @verbose = args[:verbose]
  @cleanup_grace_period = 1.day.ago

  rebuild_categories
  rebuild_tags
  rebuild_topics
  rebuild_posts
  rebuild_users

  clean_topics
  clean_posts
end

#load_problem_category_ids(limit) ⇒ Object 



113
114
115
116
117
118
119
120
121
122
123
124
 
# File 'app/jobs/scheduled/reindex_search.rb', line 113

def load_problem_category_ids(limit)
  Category
    .joins("LEFT JOIN category_search_data ON category_id = categories.id")
    .where(
      "category_search_data.locale IS NULL OR category_search_data.locale != ? OR category_search_data.version != ?",
      SiteSetting.default_locale,
      SearchIndexer::CATEGORY_INDEX_VERSION,
    )
    .order("categories.id ASC")
    .limit(limit)
    .pluck(:id)
end

#load_problem_post_ids(limit) ⇒ Object 



152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
 
# File 'app/jobs/scheduled/reindex_search.rb', line 152

def load_problem_post_ids(limit)
  Post
    .joins(:topic)
    .joins("LEFT JOIN post_search_data ON post_id = posts.id")
    .where("posts.raw != ''")
    .where("topics.deleted_at IS NULL")
    .where(
      "post_search_data.locale IS NULL OR post_search_data.locale != ? OR post_search_data.version != ?",
      SiteSetting.default_locale,
      SearchIndexer::POST_INDEX_VERSION,
    )
    .order("posts.id DESC")
    .limit(limit)
    .pluck(:id)
end

#load_problem_tag_ids(limit) ⇒ Object 



126
127
128
129
130
131
132
133
134
135
136
137
 
# File 'app/jobs/scheduled/reindex_search.rb', line 126

def load_problem_tag_ids(limit)
  Tag
    .joins("LEFT JOIN tag_search_data ON tag_id = tags.id")
    .where(
      "tag_search_data.locale IS NULL OR tag_search_data.locale != ? OR tag_search_data.version != ?",
      SiteSetting.default_locale,
      SearchIndexer::TAG_INDEX_VERSION,
    )
    .order("tags.id ASC")
    .limit(limit)
    .pluck(:id)
end

#load_problem_topic_ids(limit) ⇒ Object 



139
140
141
142
143
144
145
146
147
148
149
150
 
# File 'app/jobs/scheduled/reindex_search.rb', line 139

def load_problem_topic_ids(limit)
  Topic
    .joins("LEFT JOIN topic_search_data ON topic_id = topics.id")
    .where(
      "topic_search_data.locale IS NULL OR topic_search_data.locale != ? OR topic_search_data.version != ?",
      SiteSetting.default_locale,
      SearchIndexer::TOPIC_INDEX_VERSION,
    )
    .order("topics.id DESC")
    .limit(limit)
    .pluck(:id)
end

#load_problem_user_ids(limit) ⇒ Object 



168
169
170
171
172
173
174
175
176
177
178
179
 
# File 'app/jobs/scheduled/reindex_search.rb', line 168

def load_problem_user_ids(limit)
  User
    .joins("LEFT JOIN user_search_data ON user_id = users.id")
    .where(
      "user_search_data.locale IS NULL OR user_search_data.locale != ? OR user_search_data.version != ?",
      SiteSetting.default_locale,
      SearchIndexer::USER_INDEX_VERSION,
    )
    .order("users.id ASC")
    .limit(limit)
    .pluck(:id)
end

#rebuild_categories(limit: 500, indexer: SearchIndexer) ⇒ Object 



21
22
23
24
25
26
27
28
29
30
 
# File 'app/jobs/scheduled/reindex_search.rb', line 21

def rebuild_categories(limit: 500, indexer: SearchIndexer)
  category_ids = load_problem_category_ids(limit)

  puts "rebuilding #{category_ids.size} categories" if @verbose

  category_ids.each do |id|
    category = Category.find_by(id: id)
    indexer.index(category, force: true) if category
  end
end

#rebuild_posts(limit: 20_000, indexer: SearchIndexer) ⇒ Object 



54
55
56
57
58
59
60
61
62
63
 
# File 'app/jobs/scheduled/reindex_search.rb', line 54

def rebuild_posts(limit: 20_000, indexer: SearchIndexer)
  post_ids = load_problem_post_ids(limit)

  puts "rebuilding #{post_ids.size} posts" if @verbose

  post_ids.each do |id|
    post = Post.find_by(id: id)
    indexer.index(post, force: true) if post
  end
end

#rebuild_tags(limit: 1_000, indexer: SearchIndexer) ⇒ Object 



32
33
34
35
36
37
38
39
40
41
 
# File 'app/jobs/scheduled/reindex_search.rb', line 32

def rebuild_tags(limit: 1_000, indexer: SearchIndexer)
  tag_ids = load_problem_tag_ids(limit)

  puts "rebuilding #{tag_ids.size} tags" if @verbose

  tag_ids.each do |id|
    tag = Tag.find_by(id: id)
    indexer.index(tag, force: true) if tag
  end
end

#rebuild_topics(limit: 10_000, indexer: SearchIndexer) ⇒ Object 



43
44
45
46
47
48
49
50
51
52
 
# File 'app/jobs/scheduled/reindex_search.rb', line 43

def rebuild_topics(limit: 10_000, indexer: SearchIndexer)
  topic_ids = load_problem_topic_ids(limit)

  puts "rebuilding #{topic_ids.size} topics" if @verbose

  topic_ids.each do |id|
    topic = Topic.find_by(id: id)
    indexer.index(topic, force: true) if topic
  end
end

#rebuild_users(limit: 5_000, indexer: SearchIndexer) ⇒ Object 



65
66
67
68
69
70
71
72
73
74
 
# File 'app/jobs/scheduled/reindex_search.rb', line 65

def rebuild_users(limit: 5_000, indexer: SearchIndexer)
  user_ids = load_problem_user_ids(limit)

  puts "rebuilding #{user_ids.size} users" if @verbose

  user_ids.each do |id|
    user = User.find_by(id: id)
    indexer.index(user, force: true) if user
  end
end