diff --git a/app/jobs/processing/abstract_unstick_attachments_job.rb b/app/jobs/processing/abstract_unstick_attachments_job.rb new file mode 100644 index 00000000..ce781a3f --- /dev/null +++ b/app/jobs/processing/abstract_unstick_attachments_job.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Processing + # @abstract + class AbstractUnstickAttachmentsJob < ApplicationJob + extend Dry::Core::ClassAttributes + + include JobIteration::Iteration + + defines :model_klass, type: Support::Types::ModelClass + + model_klass ApplicationRecord + + queue_as :maintenance + + # @param [String] cursor + # @return [void] + def build_enumerator(cursor:) + enumerator_builder.active_record_on_records( + model_klass.with_any_stuck_shrine_attachments, + cursor: + ) + end + + # @param [ApplicationRecord] model + # @return [void] + def each_iteration(model) + model.unstick_shrine_attachments! + end + + private + + def model_klass = self.class.model_klass + end +end diff --git a/app/jobs/processing/unstick_collection_attachments_job.rb b/app/jobs/processing/unstick_collection_attachments_job.rb new file mode 100644 index 00000000..270c2d69 --- /dev/null +++ b/app/jobs/processing/unstick_collection_attachments_job.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +module Processing + # @see Collection + class UnstickCollectionAttachmentsJob < AbstractUnstickAttachmentsJob + model_klass Collection + end +end diff --git a/app/jobs/processing/unstick_community_attachments_job.rb b/app/jobs/processing/unstick_community_attachments_job.rb new file mode 100644 index 00000000..a4970986 --- /dev/null +++ b/app/jobs/processing/unstick_community_attachments_job.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +module Processing + # @see Community + class UnstickCommunityAttachmentsJob < AbstractUnstickAttachmentsJob + model_klass Community + end +end diff --git a/app/jobs/processing/unstick_item_attachments_job.rb b/app/jobs/processing/unstick_item_attachments_job.rb new file mode 100644 index 00000000..cb0f6b34 --- /dev/null +++ b/app/jobs/processing/unstick_item_attachments_job.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +module Processing + # @see Item + class UnstickItemAttachmentsJob < AbstractUnstickAttachmentsJob + model_klass Item + end +end diff --git a/app/services/processing/model_integration.rb b/app/services/processing/model_integration.rb index eb42f3a3..1ccc4f77 100644 --- a/app/services/processing/model_integration.rb +++ b/app/services/processing/model_integration.rb @@ -8,6 +8,8 @@ module ModelIntegration include Support::ClassyList::DSL + STALE_TIME = 15.minutes + included do has_simple_symbol_list! :shrine_attachments end @@ -57,10 +59,28 @@ def shrine_attacher_for(name) = __send__("#{name}_attacher") # @return [] def shrine_attachment_names = self.class.shrine_attachment_names + # @return [{ Symbol => Shrine::Attacher }] + def shrine_attachers = shrine_attachment_names.index_with { |name| shrine_attacher_for(name) } + # @raise [Shrine::Error] if there is no file data # @return [Hash] def shrine_file_data_for(name) = shrine_attacher_for(name).file_data + # @!group Stuck Attachment Handling + + # @return [{ Symbol => Boolean }] + def unstick_shrine_attachments! + shrine_attachers.each_with_object({}) do |(name, attacher), result| + stuck = attacher.stuck? + + retry_promoting_attachment!(name) if stuck + + result[name] = stuck + end + end + + # @!endgroup Stuck Attachment Handling + module ClassMethods # @param [Symbol] name # @return [void] @@ -156,6 +176,49 @@ def arel_shrine_with_derivatives_condition(attr, with_derivatives) # simplecov:enable end end + + # @!group Stuck Attachment Handling + + def with_any_stuck_shrine_attachments = with_stuck_shrine_attachments(shrine_attachment_names) + + def with_stuck_shrine_attachments(*names) + names.flatten! + + return none if names.empty? + + conditions = names.map do |name| + arel_attachment_stuck(name) + end + + expr = arel_grouping(arel_or_expressions(conditions)) + + where(expr) + end + + def arel_attachment_stuck(name, column: "#{name}_data") + cached = arel_shrine_storage_condition(arel_table[column], "cache") + stale = arel_attachment_stale(name, column:) + + arel_grouped cached.and(stale) + end + + def arel_attachment_generated_at(name, column: "#{name}_data") + attr = arel_table[column] + + arel_cast(arel_json_get_path_as_text(attr, "metadata", "generated_at"), "timestamptz") + end + + def arel_attachment_stale(name, column: "#{name}_data") + generated_at = arel_attachment_generated_at(name, column:) + + is_stale = generated_at.lt(STALE_TIME.ago) + + unset = generated_at.eq(nil) + + arel_grouped is_stale.or(unset) + end + + # @!endgroup Stuck Attachment Handling end end end diff --git a/app/services/processing/types.rb b/app/services/processing/types.rb index ee292361..b7794e7d 100644 --- a/app/services/processing/types.rb +++ b/app/services/processing/types.rb @@ -19,7 +19,12 @@ module Types AttachmentName = Coercible::Symbol - AttachmentData = Hash | String + FileData = Hash.schema( + id?: Coercible::String, + storage?: Coercible::String, + ).with_key_transform(&:to_sym) + + AttachmentData = FileData | String Model = ::Support::Models::Types::Model end diff --git a/app/uploaders/cached_asset_uploader.rb b/app/uploaders/cached_asset_uploader.rb index 7e109757..5d0631db 100644 --- a/app/uploaders/cached_asset_uploader.rb +++ b/app/uploaders/cached_asset_uploader.rb @@ -8,23 +8,12 @@ # # @see HarvestCachedAsset class CachedAssetUploader < Shrine - plugin :add_metadata - plugin :refresh_metadata plugin :infer_extension, force: true plugin :remote_url, max_size: 5.gigabytes, downloader: ::Support::Networking::SHRINE_REMOTE_URL_DOWNLOADER - plugin :signature plugin :validation_helpers plugin :restore_cached_data plugin :metadata_attributes, filename: "file_name", size: "file_size", mime_type: "content_type", sha256: "signature" - add_metadata :generated_at do |io, **| - Time.current.iso8601 - end - - add_metadata :sha256, skip_nil: true do |io, store: nil, **options| - calculate_signature(io, :sha256, format: :base64) unless store == :cache - end - add_metadata skip_nil: true do |io, metadata:, **| # Deal with invalid unicode encoding in filenames diff --git a/app/uploaders/generic_uploader.rb b/app/uploaders/generic_uploader.rb index 04afcb13..21ad7481 100644 --- a/app/uploaders/generic_uploader.rb +++ b/app/uploaders/generic_uploader.rb @@ -5,23 +5,12 @@ # # It stores some metadata about the kind of asset it detects, see {Assets::ParseKind}. class GenericUploader < Shrine - plugin :add_metadata - plugin :refresh_metadata plugin :infer_extension, force: true plugin :remote_url, max_size: 3.gigabytes, downloader: ::Support::Networking::SHRINE_REMOTE_URL_DOWNLOADER - plugin :signature plugin :validation_helpers plugin :restore_cached_data plugin :metadata_attributes, kind: "kind", filename: "file_name", size: "file_size", mime_type: "content_type" - add_metadata :generated_at do |io, **| - Time.current.iso8601 - end - - add_metadata :sha256, skip_nil: true do |io, store: nil, **options| - calculate_signature(io, :sha256, format: :base64) unless store == :cache - end - add_metadata :kind do |io, **options| MeruAPI::Container["assets.parse_kind"].call(io).value_or("unknown") end diff --git a/app/uploaders/image_uploader.rb b/app/uploaders/image_uploader.rb index cd929cfe..17a12a38 100644 --- a/app/uploaders/image_uploader.rb +++ b/app/uploaders/image_uploader.rb @@ -4,11 +4,8 @@ # # @see ImageAttachments::ImageWrapper class ImageUploader < Shrine - plugin :add_metadata - plugin :refresh_metadata plugin :remote_url, max_size: 100.megabytes, downloader: ::Support::Networking::SHRINE_REMOTE_URL_DOWNLOADER plugin :store_dimensions, analyzer: :ruby_vips - plugin :signature plugin :validation_helpers plugin :restore_cached_data @@ -18,14 +15,6 @@ class ImageUploader < Shrine metadata_method :alt - add_metadata :generated_at do |io, **| - Time.current.iso8601 - end - - add_metadata :sha256 do |io, derivative: nil, **| - calculate_signature(io, :sha256, format: :base64) unless derivative - end - Attacher.validate do validate_mime_type %w[image/jpg image/jpeg image/png image/tiff image/webp image/heic image/heif image/gif image/svg+xml] end diff --git a/app/uploaders/site_logo_uploader.rb b/app/uploaders/site_logo_uploader.rb index ee8ff15a..bf6a0aca 100644 --- a/app/uploaders/site_logo_uploader.rb +++ b/app/uploaders/site_logo_uploader.rb @@ -4,11 +4,8 @@ # # @see ImageAttachments::SiteLogoWrapper class SiteLogoUploader < Shrine - plugin :add_metadata - plugin :refresh_metadata plugin :remote_url, max_size: 100.megabytes, downloader: ::Support::Networking::SHRINE_REMOTE_URL_DOWNLOADER plugin :store_dimensions, analyzer: :ruby_vips - plugin :signature plugin :validation_helpers plugin :restore_cached_data @@ -18,14 +15,6 @@ class SiteLogoUploader < Shrine metadata_method :alt - add_metadata :generated_at do |io, **| - Time.current.iso8601 - end - - add_metadata :sha256 do |io, derivative: nil, **| - calculate_signature(io, :sha256, format: :base64) unless derivative - end - Attacher.validate do validate_mime_type %w[image/jpg image/jpeg image/png image/tiff image/webp image/heic image/heif image/gif image/svg+xml] end diff --git a/config/initializers/900_good_job.rb b/config/initializers/900_good_job.rb index 8cd629b0..5f172f64 100644 --- a/config/initializers/900_good_job.rb +++ b/config/initializers/900_good_job.rb @@ -116,6 +116,21 @@ description: "Process stale orderings", args: -> { [Time.current.iso8601] }, }, + "processing.unstick_collection_attachments": { + cron: "0,20,40 * * * *", + class: "Processing::UnstickCollectionAttachmentsJob", + description: "Unstick collection attachments", + }, + "processing.unstick_community_attachments": { + cron: "0,20,40 * * * *", + class: "Processing::UnstickCommunityAttachmentsJob", + description: "Unstick community attachments", + }, + "processing.unstick_item_attachments": { + cron: "0,20,40 * * * *", + class: "Processing::UnstickItemAttachmentsJob", + description: "Unstick item attachments", + }, "rendering.process_stale_entities": { cron: "*/5 * * * *", class: "Rendering::ProcessStaleEntitiesJob", diff --git a/lib/support/boot/501_shrine_standard_metadata.rb b/lib/support/boot/501_shrine_standard_metadata.rb new file mode 100644 index 00000000..fe2fa49b --- /dev/null +++ b/lib/support/boot/501_shrine_standard_metadata.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +class Shrine + module Plugins + module StandardMetadata + STALE_TIME = 15.minutes + + SafeTime = Dry::Types["params.time"].optional.fallback(nil) + + class << self + def configure(uploader, **options) + uploader.opts[:standard_metadata] ||= {} + uploader.opts[:standard_metadata].merge!(options) + uploader.opts[:standard_metadata][:stale_time] ||= STALE_TIME + + uploader.add_metadata :generated_at, skip_nil: true do |io, store: nil, **| + Time.current.iso8601 unless store.nil? + end + + uploader.add_metadata :sha256, skip_nil: true do |io, store: nil, **options| + calculate_signature(io, :sha256, format: :base64) unless store.nil? + end + end + + # @return [void] + def load_dependencies(uploader, **options) + uploader.plugin :self_registering + uploader.plugin :signature + uploader.plugin :add_metadata + uploader.plugin :refresh_metadata + end + end + + module InstanceMethods + def stale_at = stale_time.ago + + def stale_time = opts.dig(:standard_metadata, :stale_time) || STALE_TIME + end + + module AttacherMethods + def stale? = file.stale? + + def stuck? = cached? && stale? + end + + module FileMethods + def generated_at = metadata["generated_at"].then { Shrine::Plugins::StandardMetadata::SafeTime.(_1) if _1.present? } + + def stale_at = uploader.stale_at + + def stale? = generated_at.then { _1.nil? || _1 < stale_at } + end + end + + register_plugin(:standard_metadata, StandardMetadata) + end +end + +# Enable the plugin automatically +Shrine.plugin :standard_metadata diff --git a/spec/models/collection_spec.rb b/spec/models/collection_spec.rb index 7b5b1cd1..531fcf1e 100644 --- a/spec/models/collection_spec.rb +++ b/spec/models/collection_spec.rb @@ -72,4 +72,15 @@ end end end + + context "when attaching a thumbnail image" do + let(:thumbnail_path) { Rails.root.join("spec", "data", "lorempixel.jpg") } + + it "attaches the thumbnail for processing" do + expect do + collection.thumbnail = thumbnail_path.open("rb+") + collection.save! + end.to have_enqueued_job(Processing::PromoteAttachmentJob) + end + end end diff --git a/spec/requests/graphql/mutations/update_collection_spec.rb b/spec/requests/graphql/mutations/update_collection_spec.rb index c11a373d..53ec0695 100644 --- a/spec/requests/graphql/mutations/update_collection_spec.rb +++ b/spec/requests/graphql/mutations/update_collection_spec.rb @@ -182,6 +182,29 @@ end end + context "when updating a thumbnail" do + include_context "with stubbed revalidation" + + let(:clear_thumbnail) { false } + + let(:new_thumbnail) do + graphql_upload_from "spec", "data", "lorempixel.jpg" + end + + it "updates the thumbnail" do + expect_request! do |req| + req.effect! change { collection.reload.thumbnail.id }.to(be_present) + req.effect! have_enqueued_job(Processing::PromoteAttachmentJob).once + + req.data! expected_shape + end + + expect do + perform_enqueued_jobs + end.to change { collection.reload.thumbnail_data["storage"] }.from("cache").to("store") + end + end + context "when clearing a thumbnail" do let!(:clear_thumbnail) { true } diff --git a/spec/support/contexts/stubbed_revalidation.rb b/spec/support/contexts/stubbed_revalidation.rb new file mode 100644 index 00000000..63b5b6af --- /dev/null +++ b/spec/support/contexts/stubbed_revalidation.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +RSpec.shared_context "with stubbed entity revalidation" do + before do + endpoint = Frontend::Cache::EntityRevalidator.endpoint + + now = Time.current.to_i * 1000 + + body = { revalidated: true, now:, }.to_json + + headers = { "Content-Type" => "application/json" } + + stub_request(:delete, endpoint) + .to_return(status: 200, body:, headers:) + end +end + +RSpec.shared_context "with stubbed instance revalidation" do + include_context "with stubbed entity revalidation" + + before do + endpoint = Frontend::Cache::InstanceRevalidator.endpoint + + now = Time.current.to_i * 1000 + + body = { revalidated: true, now:, }.to_json + + headers = { "Content-Type" => "application/json" } + + stub_request(:delete, endpoint) + .to_return(status: 200, body:, headers:) + end +end + +RSpec.shared_context "with stubbed revalidation" do + include_context "with stubbed entity revalidation" + include_context "with stubbed instance revalidation" +end