diff --git a/app/models/required_operator_document_fmu.rb b/app/models/required_operator_document_fmu.rb index 614bfd607..637031867 100644 --- a/app/models/required_operator_document_fmu.rb +++ b/app/models/required_operator_document_fmu.rb @@ -49,8 +49,9 @@ def create_operator_document_fmus end end + # forest_types are strings (ForestType::TYPES is a HashWithIndifferentAccess), fmu.forest_type is an enum string def applies_to_forest_type?(forest_type) - forest_types.blank? || forest_types.include?(forest_type.to_sym) + forest_types.blank? || forest_types.map(&:to_s).include?(forest_type.to_s) end def fmus diff --git a/lib/tasks/check_data.rake b/lib/tasks/check_data.rake index e12a95ca4..bda35ae11 100644 --- a/lib/tasks/check_data.rake +++ b/lib/tasks/check_data.rake @@ -41,7 +41,7 @@ namespace :check do fmu = od.fmu rod = od.required_operator_document - unless rod.forest_types.include?(fmu.forest_type.to_sym) + unless rod.applies_to_forest_type?(fmu.forest_type) mismatch_count += 1 expected_forest_types = rod.forest_types.join(", ") @@ -49,8 +49,8 @@ namespace :check do puts "Document id: #{od.id} - status: #{od.status}, last updated at: #{od.updated_at} versions: #{od.versions.count}, operator: #{od.operator.name} (id: #{od.operator.id}), Country: #{od.operator.country.name}, FMU forest type: #{fmu.forest_type} but this document is for forest types: #{expected_forest_types}" if ENV["VERBOSE"] - fmu_any_version_with_forest_type = rod.forest_types.any? { |ftype| fmu.versions.where_object(forest_type: ftype).exists? } - rod_any_version_with_forest_type = rod.versions.any? { |v| v.reify.forest_types.include?(fmu.forest_type.to_sym) } + fmu_any_version_with_forest_type = rod.forest_types.any? { |ftype| fmu.versions.where_object(forest_type: Fmu.forest_types[ftype]).exists? } + rod_any_version_with_forest_type = rod.versions.any? { |v| v.reify.applies_to_forest_type?(fmu.forest_type) } puts "======> FMU: #{od.fmu.id}, versions: #{fmu.versions.count}, any version with any of #{expected_forest_types} type: #{fmu_any_version_with_forest_type}" puts "======> Required Document: #{rod.id}, versions: #{rod.versions.count}, any with #{fmu.forest_type} type: #{rod_any_version_with_forest_type}" end diff --git a/lib/tasks/fix.rake b/lib/tasks/fix.rake index d5a9c93da..e0558f04f 100644 --- a/lib/tasks/fix.rake +++ b/lib/tasks/fix.rake @@ -2,6 +2,117 @@ require "benchmark" require "csv" namespace :fix do + # One time repair of the two operator documents deleted on 2026-08-21 by the forest type + # string/symbol comparison bug in RequiredOperatorDocumentFmu#applies_to_forest_type?. + # The documents were restored by hand, this removes everything else the deletion left behind. + # SKIP_STATISTICS=true leaves the statistics alone, for when a backfill regenerates them anyway. + task erroneous_document_deletion: :environment do + for_real = ENV["FOR_REAL"] == "true" + skip_statistics = ENV["SKIP_STATISTICS"] == "true" + puts "DRY RUN, pass FOR_REAL=true to apply" unless for_real + puts "Skipping statistics regeneration" if skip_statistics + + document_ids = [19045, 19349] + statistic_country_ids = [45, nil] + + ActiveRecord::Base.transaction do + documents = OperatorDocument.where(id: document_ids) + raise "expected #{document_ids.size} live documents, found #{documents.count}" unless documents.count == document_ids.size + + operator_ids = documents.distinct.pluck(:operator_id) + raise "expected a single operator, found #{operator_ids.inspect}" unless operator_ids.size == 1 + + operator_id = operator_ids.first + + versions = PaperTrail::Version.where( + item_type: %w[OperatorDocument OperatorDocumentFmu OperatorDocumentCountry], + item_id: document_ids, event: "destroy" + ) + raise "expected #{document_ids.size} destroy versions, found #{versions.count}" unless versions.count == document_ids.size + + deletion_time = versions.minimum(:created_at) + puts "Deletion at #{deletion_time}, operator #{operator_id}" + + histories = OperatorDocumentHistory.only_deleted.where(operator_document_id: document_ids) + raise "expected #{document_ids.size} deleted history rows, found #{histories.count}" unless histories.count == document_ids.size + + histories.each do |history| + raise "history #{history.id} was not born deleted" unless (history.deleted_at - history.created_at).abs < 1 + raise "history #{history.id} has annexes" if history.annex_documents.any? + + puts "Purging history #{history.id} (operator document #{history.operator_document_id})" + history.really_destroy! + end + + puts "Deleting #{versions.count} destroy versions: #{versions.pluck(:id).inspect}" + versions.delete_all + + # the restore moved updated_at forward, put it back on the last real change so it lines up with history + documents.each do |document| + last_history = OperatorDocumentHistory.where(operator_document_id: document.id).order(:operator_document_updated_at).last + raise "no history left for document #{document.id}" if last_history.nil? + + puts "Document #{document.id} updated_at #{document.updated_at} -> #{last_history.operator_document_updated_at}" + document.update_columns(updated_at: last_history.operator_document_updated_at) + end + + # the deletion recalculated the score, which added a score for that day and superseded the then + # current one. Only that score is an artifact, later ones would have been created anyway because + # their summaries differ from the superseded score. + superseded_score = ScoreOperatorDocument.where(operator_id: operator_id) + .where("created_at < ?", deletion_time).order(:created_at, :id).last + raise "no score from before the deletion for operator #{operator_id}" if superseded_score.nil? + + bug_scores = ScoreOperatorDocument.where(operator_id: operator_id, date: deletion_time.to_date).to_a + raise "expected one score dated #{deletion_time.to_date}, found #{bug_scores.map(&:id).inspect}" unless bug_scores.size == 1 + + bug_score = bug_scores.first + raise "score #{bug_score.id} is current, refusing to delete it" if bug_score.current? + + expected_total = superseded_score.total - document_ids.size + raise "score #{bug_score.id} has total #{bug_score.total}, expected #{expected_total}" unless bug_score.total == expected_total + + successor_score = ScoreOperatorDocument.where(operator_id: operator_id) + .where("created_at > ?", bug_score.created_at).order(:created_at, :id).first + raise "nothing superseded score #{bug_score.id}" if successor_score.nil? + + puts "Deleting score #{bug_score.id} (date #{bug_score.date}, total #{bug_score.total})" + ScoreOperatorDocument.where(id: bug_score.id).delete_all + + puts "Score #{superseded_score.id} updated_at #{superseded_score.updated_at} -> #{successor_score.created_at}" + superseded_score.update_columns(updated_at: successor_score.created_at) + + unless skip_statistics + # every statistic from the deletion day onwards was generated while the documents were hidden + statistic_days = OperatorDocumentStatistic + .where(country_id: statistic_country_ids) + .where("date >= ?", deletion_time.to_date) + .distinct.order(:date).pluck(:date) + + statistic_days.each do |day| + statistic_country_ids.each do |country_id| + puts "Regenerating document statistics for country #{country_id.inspect} on #{day}" + OperatorDocumentStatistic.generate_for_country_and_day(country_id, day, true) + end + end + end + + puts + puts "Verification:" + [deletion_time.to_date, Date.current].uniq.each do |day| + visible = OperatorDocumentHistory.at_date(day).where(operator_document_id: document_ids).count + puts " #{day}: #{visible} of #{document_ids.size} documents visible in history" + end + ScoreOperatorDocument.where(operator_id: operator_id).order(:date, :id).last(3).each do |score| + puts " score #{score.id} date=#{score.date} current=#{score.current} all=#{score.all} total=#{score.total}" + end + + raise ActiveRecord::Rollback unless for_real + end + + puts for_real ? "Applied." : "Rolled back." + end + task annexes: :environment do ActiveRecord::Base.transaction do for_real = ENV["FOR_REAL"] == "true" diff --git a/spec/models/operator_document_spec.rb b/spec/models/operator_document_spec.rb index d8d04c672..416552f7e 100644 --- a/spec/models/operator_document_spec.rb +++ b/spec/models/operator_document_spec.rb @@ -321,6 +321,45 @@ expect(history_pending.annex_documents.count).to eql 1 expect(history_not_provided.annex_documents.count).to eql 0 end + + context "with an fmu document" do + let(:forest_types) { [ForestType::TYPES[:ufa][:index]] } # @fmu forest type + let(:required_operator_document_fmu) { + create( + :required_operator_document_fmu, + country: @country, + forest_types: forest_types, + disable_document_creation: true + ) + } + let!(:operator_document) { + create( + :operator_document_fmu, + operator: @operator, + fmu: @fmu, + required_operator_document_fmu: required_operator_document_fmu + ) + } + + context "when the required document still applies to the fmu forest type" do + it "regenerates document state to not provided" do + operator_document.destroy + + expect(operator_document.reload.deleted?).to be(false) + expect(operator_document.status).to eq("doc_not_provided") + end + end + + context "when the required document no longer applies to the fmu forest type" do + let(:forest_types) { [ForestType::TYPES[:vdc][:index]] } + + it "deletes the document" do + operator_document.destroy + + expect(operator_document.reload.deleted?).to be(true) + end + end + end end end diff --git a/spec/models/required_operator_document_fmu_spec.rb b/spec/models/required_operator_document_fmu_spec.rb index ec343d7e7..b718a0763 100644 --- a/spec/models/required_operator_document_fmu_spec.rb +++ b/spec/models/required_operator_document_fmu_spec.rb @@ -31,6 +31,33 @@ it { is_expected.to validate_absence_of(:contract_signature) } end + describe "#applies_to_forest_type?" do + subject { rod.applies_to_forest_type?(forest_type) } + + let(:rod) { build :required_operator_document_fmu, forest_types: forest_types } + + context "when the required document has no forest types" do + let(:forest_types) { [] } + let(:forest_type) { "ufa" } + + it { is_expected.to be true } + end + + context "when the forest type is one of the required document forest types" do + let(:forest_types) { [ForestType::TYPES[:ufa][:index], ForestType::TYPES[:cf][:index]] } + let(:forest_type) { "ufa" } + + it { is_expected.to be true } + end + + context "when the forest type is not one of the required document forest types" do + let(:forest_types) { [ForestType::TYPES[:ufa][:index]] } + let(:forest_type) { "vdc" } + + it { is_expected.to be false } + end + end + describe "Hooks" do describe "#create_operator_document_fmus" do let(:operator_country) { create :country }