Class: Upload
- Inherits:
-
ActiveRecord::Base
- Object
- ActiveRecord::Base
- Upload
- Includes:
- ActionView::Helpers::NumberHelper, HasUrl
- Defined in:
- app/models/upload.rb
Constant Summary collapse
- SHA1_LENGTH =
40
- SEEDED_ID_THRESHOLD =
0
- MAX_IDENTIFY_SECONDS =
5
- DOMINANT_COLOR_COMMAND_TIMEOUT_SECONDS =
5
Instance Attribute Summary collapse
-
#for_export ⇒ Object
Returns the value of attribute for_export.
-
#for_gravatar ⇒ Object
Returns the value of attribute for_gravatar.
-
#for_group_message ⇒ Object
Returns the value of attribute for_group_message.
-
#for_private_message ⇒ Object
Returns the value of attribute for_private_message.
-
#for_site_setting ⇒ Object
Returns the value of attribute for_site_setting.
-
#for_theme ⇒ Object
Returns the value of attribute for_theme.
-
#validate_file_size ⇒ Object
Returns the value of attribute validate_file_size.
Class Method Summary collapse
- .add_in_use_callback(&block) ⇒ Object
- .add_unused_callback(&block) ⇒ Object
- .backfill_dominant_colors!(count) ⇒ Object
- .base62_sha1(sha1) ⇒ Object
- .consider_for_reuse(upload, post) ⇒ Object
- .extract_upload_ids(raw) ⇒ Object
- .generate_digest(path) ⇒ Object
- .in_use_callbacks ⇒ Object
- .migrate_to_new_scheme(limit: nil) ⇒ Object
- .reset_in_use_callbacks ⇒ Object
- .reset_unused_callbacks ⇒ Object
- .secure_uploads_url?(url) ⇒ Boolean
- .secure_uploads_url_from_upload_url(url) ⇒ Object
- .sha1_from_base62_encoded(encoded_sha1) ⇒ Object
- .sha1_from_short_path(path) ⇒ Object
- .sha1_from_short_url(url) ⇒ Object
- .short_path(sha1:, extension:) ⇒ Object
- .signed_url_from_secure_uploads_url(url) ⇒ Object
- .unused_callbacks ⇒ Object
- .verification_statuses ⇒ Object
- .with_no_non_post_relations ⇒ Object
Instance Method Summary collapse
-
#access_control_post ⇒ Object
when we access this post we don’t care if the post is deleted.
- #base62_sha1 ⇒ Object
- #calculate_dominant_color!(local_path = nil) ⇒ Object
- #content ⇒ Object
- #copied_from_other_post?(post) ⇒ Boolean
- #create_thumbnail!(width, height, opts = nil) ⇒ Object
- #destroy ⇒ Object
- #dominant_color(calculate_if_missing: false) ⇒ Object
- #fix_dimensions! ⇒ Object
- #fix_image_extension ⇒ Object
-
#get_dimension(key) ⇒ Object
on demand image size calculation, this allows us to null out image sizes and still handle as needed.
-
#get_optimized_image(width, height, opts = nil) ⇒ Object
this method attempts to correct old incorrect extensions.
- #has_thumbnail?(width, height) ⇒ Boolean
- #height ⇒ Object
- #human_filesize ⇒ Object
-
#initialize(*args) ⇒ Upload
constructor
A new instance of Upload.
- #local? ⇒ Boolean
- #matching_access_control_post?(post) ⇒ Boolean
- #rebake_posts_on_old_scheme ⇒ Object
- #secure_params(secure, reason, source = "unknown") ⇒ Object
- #short_path ⇒ Object
- #short_url ⇒ Object
- #target_image_quality(local_path, test_quality) ⇒ Object
- #thumbnail(width = self.thumbnail_width, height = self.thumbnail_height) ⇒ Object
- #thumbnail_height ⇒ Object
- #thumbnail_width ⇒ Object
- #to_markdown ⇒ Object
- #to_s ⇒ Object
- #update_secure_status(source: "unknown", override: nil) ⇒ Object
- #uploaded_before_secure_uploads_enabled? ⇒ Boolean
- #width ⇒ Object
Constructor Details
#initialize(*args) ⇒ Upload
Returns a new instance of Upload.
94 95 96 97 |
# File 'app/models/upload.rb', line 94 def initialize(*args) super self.validate_file_size = true end |
Instance Attribute Details
#for_export ⇒ Object
Returns the value of attribute for_export.
34 35 36 |
# File 'app/models/upload.rb', line 34 def for_export @for_export end |
#for_gravatar ⇒ Object
Returns the value of attribute for_gravatar.
36 37 38 |
# File 'app/models/upload.rb', line 36 def for_gravatar @for_gravatar end |
#for_group_message ⇒ Object
Returns the value of attribute for_group_message.
31 32 33 |
# File 'app/models/upload.rb', line 31 def @for_group_message end |
#for_private_message ⇒ Object
Returns the value of attribute for_private_message.
33 34 35 |
# File 'app/models/upload.rb', line 33 def @for_private_message end |
#for_site_setting ⇒ Object
Returns the value of attribute for_site_setting.
35 36 37 |
# File 'app/models/upload.rb', line 35 def for_site_setting @for_site_setting end |
#for_theme ⇒ Object
Returns the value of attribute for_theme.
32 33 34 |
# File 'app/models/upload.rb', line 32 def for_theme @for_theme end |
#validate_file_size ⇒ Object
Returns the value of attribute validate_file_size.
37 38 39 |
# File 'app/models/upload.rb', line 37 def validate_file_size @validate_file_size end |
Class Method Details
.add_in_use_callback(&block) ⇒ Object
76 77 78 |
# File 'app/models/upload.rb', line 76 def self.add_in_use_callback(&block) (@in_use_callbacks ||= []) << block end |
.add_unused_callback(&block) ⇒ Object
64 65 66 |
# File 'app/models/upload.rb', line 64 def self.add_unused_callback(&block) (@unused_callbacks ||= []) << block end |
.backfill_dominant_colors!(count) ⇒ Object
628 629 630 631 632 633 634 |
# File 'app/models/upload.rb', line 628 def self.backfill_dominant_colors!(count) Upload .where(dominant_color: nil) .order("id desc") .first(count) .each { |upload| upload.calculate_dominant_color! } end |
.base62_sha1(sha1) ⇒ Object
256 257 258 |
# File 'app/models/upload.rb', line 256 def self.base62_sha1(sha1) Base62.encode(sha1.hex) end |
.consider_for_reuse(upload, post) ⇒ Object
213 214 215 216 217 218 219 |
# File 'app/models/upload.rb', line 213 def self.consider_for_reuse(upload, post) return upload if !SiteSetting.secure_uploads? || upload.blank? || post.blank? if !upload.matching_access_control_post?(post) || upload.uploaded_before_secure_uploads_enabled? return nil end upload end |
.extract_upload_ids(raw) ⇒ Object
614 615 616 617 618 619 620 621 622 623 624 625 626 |
# File 'app/models/upload.rb', line 614 def self.extract_upload_ids(raw) return [] if raw.blank? sha1s = [] raw.scan(/\/(\h{40})/).each { |match| sha1s << match[0] } raw .scan(%r{/([a-zA-Z0-9]+)}) .each { |match| sha1s << Upload.sha1_from_base62_encoded(match[0]) } Upload.where(sha1: sha1s.uniq).pluck(:id) end |
.generate_digest(path) ⇒ Object
446 447 448 |
# File 'app/models/upload.rb', line 446 def self.generate_digest(path) Digest::SHA1.file(path).hexdigest end |
.in_use_callbacks ⇒ Object
80 81 82 |
# File 'app/models/upload.rb', line 80 def self.in_use_callbacks @in_use_callbacks end |
.migrate_to_new_scheme(limit: nil) ⇒ Object
491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 |
# File 'app/models/upload.rb', line 491 def self.migrate_to_new_scheme(limit: nil) problems = [] DistributedMutex.synchronize("migrate_upload_to_new_scheme") do if SiteSetting.migrate_to_new_scheme max_file_size_kb = [ SiteSetting.max_image_size_kb, SiteSetting., ].max.kilobytes local_store = FileStore::LocalStore.new db = RailsMultisite::ConnectionManagement.current_db scope = Upload .by_users .where("url NOT LIKE '%/original/_X/%' AND url LIKE ?", "%/uploads/#{db}%") .order(id: :desc) scope = scope.limit(limit) if limit if scope.count == 0 SiteSetting.migrate_to_new_scheme = false return problems end remap_scope = nil scope.each do |upload| begin # keep track of the url previous_url = upload.url.dup # where is the file currently stored? external = previous_url =~ %r{\A//} # download if external if external url = SiteSetting.scheme + ":" + previous_url begin retries ||= 0 file = FileHelper.download( url, max_file_size: max_file_size_kb, tmp_file_name: "discourse", follow_redirect: true, ) rescue OpenURI::HTTPError retry if (retries += 1) < 1 next end path = file.path else path = local_store.path_for(upload) end # compute SHA if missing upload.sha1 = Upload.generate_digest(path) if upload.sha1.blank? # store to new location & update the filesize File.open(path) do |f| upload.url = Discourse.store.store_upload(f, upload) upload.filesize = f.size upload.save!(validate: false) end # remap the URLs DbHelper.remap(UrlHelper.absolute(previous_url), upload.url) unless external DbHelper.remap( previous_url, upload.url, excluded_tables: %w[ posts post_search_data incoming_emails notifications single_sign_on_records stylesheet_cache topic_search_data users user_emails draft_sequences optimized_images ], ) remap_scope ||= begin Post .with_deleted .where( "raw ~ '/uploads/#{db}/\\d+/' OR raw ~ '/uploads/#{db}/original/(\\d|[a-z])/'", ) .select(:id, :raw, :cooked) .all end remap_scope.each do |post| post.raw.gsub!(previous_url, upload.url) post.cooked.gsub!(previous_url, upload.url) if post.changed? Post.with_deleted.where(id: post.id).update_all(raw: post.raw, cooked: post.cooked) end end upload.optimized_images.find_each(&:destroy!) upload.rebake_posts_on_old_scheme # remove the old file (when local) FileUtils.rm(path, force: true) unless external rescue => e problems << { upload: upload, ex: e } ensure file&.unlink file&.close end end end end problems end |
.reset_in_use_callbacks ⇒ Object
84 85 86 |
# File 'app/models/upload.rb', line 84 def self.reset_in_use_callbacks @in_use_callbacks = [] end |
.reset_unused_callbacks ⇒ Object
72 73 74 |
# File 'app/models/upload.rb', line 72 def self.reset_unused_callbacks @unused_callbacks = [] end |
.secure_uploads_url?(url) ⇒ Boolean
221 222 223 224 225 226 227 228 229 230 |
# File 'app/models/upload.rb', line 221 def self.secure_uploads_url?(url) # we do not want to exclude topic links that for whatever reason # have secure-uploads in the URL e.g. /t/secure-uploads-are-cool/223452 route = UrlHelper.rails_route_from_url(url) return false if route.blank? route[:action] == "show_secure" && route[:controller] == "uploads" && FileHelper.is_supported_media?(url) rescue ActionController::RoutingError false end |
.secure_uploads_url_from_upload_url(url) ⇒ Object
239 240 241 242 243 244 245 246 247 248 |
# File 'app/models/upload.rb', line 239 def self.secure_uploads_url_from_upload_url(url) return url if !url.include?(SiteSetting.Upload.absolute_base_url) uri = URI.parse(url) Rails.application.routes.url_for( controller: "uploads", action: "show_secure", path: uri.path[1..-1], only_path: true, ) end |
.sha1_from_base62_encoded(encoded_sha1) ⇒ Object
436 437 438 439 440 441 442 443 444 |
# File 'app/models/upload.rb', line 436 def self.sha1_from_base62_encoded(encoded_sha1) sha1 = Base62.decode(encoded_sha1).to_s(16) if sha1.length > SHA1_LENGTH nil else sha1.rjust(SHA1_LENGTH, "0") end end |
.sha1_from_short_path(path) ⇒ Object
428 429 430 |
# File 'app/models/upload.rb', line 428 def self.sha1_from_short_path(path) self.sha1_from_base62_encoded($2) if path =~ %r{(/uploads/short-url/)([a-zA-Z0-9]+)(\..*)?} end |
.sha1_from_short_url(url) ⇒ Object
432 433 434 |
# File 'app/models/upload.rb', line 432 def self.sha1_from_short_url(url) self.sha1_from_base62_encoded($2) if url =~ %r{(upload://)?([a-zA-Z0-9]+)(\..*)?} end |
.short_path(sha1:, extension:) ⇒ Object
250 251 252 253 254 |
# File 'app/models/upload.rb', line 250 def self.short_path(sha1:, extension:) @url_helpers ||= Rails.application.routes.url_helpers @url_helpers.upload_short_path(base62: self.base62_sha1(sha1), extension: extension) end |
.signed_url_from_secure_uploads_url(url) ⇒ Object
232 233 234 235 236 237 |
# File 'app/models/upload.rb', line 232 def self.signed_url_from_secure_uploads_url(url) route = UrlHelper.rails_route_from_url(url) url = Rails.application.routes.url_for(route.merge(only_path: true)) secure_upload_s3_path = url[url.index(route[:path])..-1] Discourse.store.signed_url_for_path(secure_upload_s3_path) end |
.unused_callbacks ⇒ Object
68 69 70 |
# File 'app/models/upload.rb', line 68 def self.unused_callbacks @unused_callbacks end |
.verification_statuses ⇒ Object
60 61 62 |
# File 'app/models/upload.rb', line 60 def self.verification_statuses @verification_statuses ||= Enum.new(unchecked: 1, verified: 2, invalid_etag: 3) end |
.with_no_non_post_relations ⇒ Object
88 89 90 91 92 |
# File 'app/models/upload.rb', line 88 def self.with_no_non_post_relations self.joins( "LEFT JOIN upload_references ur ON ur.upload_id = uploads.id AND ur.target_type != 'Post'", ).where("ur.upload_id IS NULL") end |
Instance Method Details
#access_control_post ⇒ Object
when we access this post we don’t care if the post is deleted
20 21 22 |
# File 'app/models/upload.rb', line 20 def access_control_post Post.unscoped { super } end |
#base62_sha1 ⇒ Object
260 261 262 |
# File 'app/models/upload.rb', line 260 def base62_sha1 Upload.base62_sha1(self.sha1) end |
#calculate_dominant_color!(local_path = nil) ⇒ Object
349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 |
# File 'app/models/upload.rb', line 349 def calculate_dominant_color!(local_path = nil) color = nil color = "" if !FileHelper.is_supported_image?("image.#{extension}") || extension == "svg" if color.nil? local_path ||= if local? Discourse.store.path_for(self) else Discourse.store.download_safe(self)&.path end if local_path.nil? # Download failed. Could be too large to download, or file could be missing in s3 color = "" end color ||= begin data = Discourse::Utils.execute_command( "nice", "-n", "10", "convert", local_path, "-depth", "8", "-resize", "1x1", "-define", "histogram:unique-colors=true", "-format", "%c", "histogram:info:", timeout: DOMINANT_COLOR_COMMAND_TIMEOUT_SECONDS, ) # Output format: # 1: (110.873,116.226,93.8821) #6F745E srgb(43.4798%,45.5789%,36.8165%) color = data[/#([0-9A-F]{6})/, 1] raise "Calculated dominant color but unable to parse output:\n#{data}" if color.nil? color rescue Discourse::Utils::CommandError => e # Timeout or unable to parse image # This can happen due to bad user input - ignore and save # an empty string to prevent re-evaluation "" end end if persisted? self.update_column(:dominant_color, color) else self.dominant_color = color end end |
#content ⇒ Object
142 143 144 145 146 147 148 149 150 151 152 153 154 |
# File 'app/models/upload.rb', line 142 def content original_path = Discourse.store.path_for(self) external_copy = nil if original_path.blank? external_copy = Discourse.store.download!(self) original_path = external_copy.path end File.read(original_path) ensure File.unlink(external_copy.path) if external_copy end |
#copied_from_other_post?(post) ⇒ Boolean
204 205 206 207 |
# File 'app/models/upload.rb', line 204 def copied_from_other_post?(post) return false if access_control_post_id.blank? !matching_access_control_post?(post) end |
#create_thumbnail!(width, height, opts = nil) ⇒ Object
115 116 117 118 119 120 |
# File 'app/models/upload.rb', line 115 def create_thumbnail!(width, height, opts = nil) return unless SiteSetting.create_thumbnails? opts ||= {} save(validate: false) if get_optimized_image(width, height, opts) end |
#destroy ⇒ Object
185 186 187 188 189 190 |
# File 'app/models/upload.rb', line 185 def destroy Upload.transaction do Discourse.store.remove_upload(self) super end end |
#dominant_color(calculate_if_missing: false) ⇒ Object
339 340 341 342 343 344 345 346 347 |
# File 'app/models/upload.rb', line 339 def dominant_color(calculate_if_missing: false) val = read_attribute(:dominant_color) if val.nil? && calculate_if_missing calculate_dominant_color! read_attribute(:dominant_color) else val end end |
#fix_dimensions! ⇒ Object
268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 |
# File 'app/models/upload.rb', line 268 def fix_dimensions! return if !FileHelper.is_supported_image?("image.#{extension}") begin path = if local? Discourse.store.path_for(self) else Discourse.store.download!(self).path end if extension == "svg" w, h = begin Discourse::Utils.execute_command( "identify", "-format", "%w %h", path, timeout: MAX_IDENTIFY_SECONDS, ).split(" ") rescue StandardError [0, 0] end else w, h = FastImage.new(path, raise_on_failure: true).size end self.width = w || 0 self.height = h || 0 self.thumbnail_width, self.thumbnail_height = ImageSizer.resize(w, h) self.update_columns( width: width, height: height, thumbnail_width: thumbnail_width, thumbnail_height: thumbnail_height, ) rescue => e Discourse.warn_exception(e, message: "Error getting image dimensions") end nil end |
#fix_image_extension ⇒ Object
156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 |
# File 'app/models/upload.rb', line 156 def fix_image_extension return false if extension == "unknown" begin # this is relatively cheap once cached original_path = Discourse.store.path_for(self) if original_path.blank? external_copy = Discourse.store.download_safe(self) original_path = external_copy&.path end image_info = begin FastImage.new(original_path) rescue StandardError nil end new_extension = image_info&.type&.to_s || "unknown" if new_extension != self.extension self.update_columns(extension: new_extension) true end rescue StandardError self.update_columns(extension: "unknown") true end end |
#get_dimension(key) ⇒ Object
on demand image size calculation, this allows us to null out image sizes and still handle as needed
315 316 317 318 319 320 321 |
# File 'app/models/upload.rb', line 315 def get_dimension(key) if v = read_attribute(key) return v end fix_dimensions! read_attribute(key) end |
#get_optimized_image(width, height, opts = nil) ⇒ Object
this method attempts to correct old incorrect extensions
123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 |
# File 'app/models/upload.rb', line 123 def get_optimized_image(width, height, opts = nil) opts ||= {} fix_image_extension if (!extension || extension.length == 0) opts = opts.merge(raise_on_error: true) begin OptimizedImage.create_for(self, width, height, opts) rescue => ex Rails.logger.info ex if Rails.env.development? opts = opts.merge(raise_on_error: false) if fix_image_extension OptimizedImage.create_for(self, width, height, opts) else nil end end end |
#has_thumbnail?(width, height) ⇒ Boolean
111 112 113 |
# File 'app/models/upload.rb', line 111 def has_thumbnail?(width, height) thumbnail(width, height).present? end |
#height ⇒ Object
327 328 329 |
# File 'app/models/upload.rb', line 327 def height get_dimension(:height) end |
#human_filesize ⇒ Object
450 451 452 |
# File 'app/models/upload.rb', line 450 def human_filesize number_to_human_size(self.filesize) end |
#local? ⇒ Boolean
264 265 266 |
# File 'app/models/upload.rb', line 264 def local? !(url =~ %r{\A(https?:)?//}) end |
#matching_access_control_post?(post) ⇒ Boolean
200 201 202 |
# File 'app/models/upload.rb', line 200 def matching_access_control_post?(post) access_control_post_id == post.id end |
#rebake_posts_on_old_scheme ⇒ Object
454 455 456 |
# File 'app/models/upload.rb', line 454 def rebake_posts_on_old_scheme self.posts.where("cooked LIKE '%/_optimized/%'").find_each(&:rebake!) end |
#secure_params(secure, reason, source = "unknown") ⇒ Object
483 484 485 486 487 488 489 |
# File 'app/models/upload.rb', line 483 def secure_params(secure, reason, source = "unknown") { secure: secure, security_last_changed_reason: reason + " | source: #{source}", security_last_changed_at: Time.zone.now, } end |
#short_path ⇒ Object
209 210 211 |
# File 'app/models/upload.rb', line 209 def short_path self.class.short_path(sha1: self.sha1, extension: self.extension) end |
#short_url ⇒ Object
192 193 194 |
# File 'app/models/upload.rb', line 192 def short_url "upload://#{short_url_basename}" end |
#target_image_quality(local_path, test_quality) ⇒ Object
411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 |
# File 'app/models/upload.rb', line 411 def target_image_quality(local_path, test_quality) @file_quality ||= begin Discourse::Utils.execute_command( "identify", "-format", "%Q", local_path, timeout: MAX_IDENTIFY_SECONDS, ).to_i rescue StandardError 0 end test_quality if @file_quality == 0 || @file_quality > test_quality end |
#thumbnail(width = self.thumbnail_width, height = self.thumbnail_height) ⇒ Object
107 108 109 |
# File 'app/models/upload.rb', line 107 def thumbnail(width = self.thumbnail_width, height = self.thumbnail_height) optimized_images.find_by(width: width, height: height) end |
#thumbnail_height ⇒ Object
335 336 337 |
# File 'app/models/upload.rb', line 335 def thumbnail_height get_dimension(:thumbnail_height) end |
#thumbnail_width ⇒ Object
331 332 333 |
# File 'app/models/upload.rb', line 331 def thumbnail_width get_dimension(:thumbnail_width) end |
#to_markdown ⇒ Object
103 104 105 |
# File 'app/models/upload.rb', line 103 def to_markdown UploadMarkdown.new(self).to_markdown end |
#to_s ⇒ Object
99 100 101 |
# File 'app/models/upload.rb', line 99 def to_s self.url end |
#update_secure_status(source: "unknown", override: nil) ⇒ Object
458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 |
# File 'app/models/upload.rb', line 458 def update_secure_status(source: "unknown", override: nil) if override.nil? mark_secure, reason = UploadSecurity.new(self).should_be_secure_with_reason else mark_secure = override reason = "manually overridden" end secure_status_did_change = self.secure? != mark_secure self.update(secure_params(mark_secure, reason, source)) if Discourse.store.external? begin Discourse.store.update_upload_ACL(self) rescue Aws::S3::Errors::NotImplemented => err Discourse.warn_exception( err, message: "The file store object storage provider does not support setting ACLs", ) end end secure_status_did_change end |
#uploaded_before_secure_uploads_enabled? ⇒ Boolean
196 197 198 |
# File 'app/models/upload.rb', line 196 def uploaded_before_secure_uploads_enabled? original_sha1.blank? end |
#width ⇒ Object
323 324 325 |
# File 'app/models/upload.rb', line 323 def width get_dimension(:width) end |