SWAMPy

✎ CARVE FLESH

EDITING: backfill_nulls.rb

# frozen_string_literal: true

#
# Copyright (C) 2017 - present Instructure, Inc.
#
# This file is part of Canvas.
#
# Canvas is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation, version 3 of the License.
#
# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.

module DataFixup
  module BackfillNulls
    # if there are multiple fields, and existing non-NULL values might be toasted
    # (large values), you should call this fixup separately for each column.
    # if you were to combine it with other columns, unchanging non-NULL columns
    # would be copied for every row that another column _is_ changing
    #
    # you probably only want to use sleep if you're calling this manually to
    # get the bulk of work done ahead of deploying the actual migration that
    # calls this
    #
    # @param fields [Hash<Symbol => Object>, Array<Symbol>, Symbol]
    #   If a hash, it's column to default value. If an array or a single symbol, it's just
    #   the column(s), and the default_value is passed separately, and applied to all of
    #   the columns.
    def self.run(klass, fields, default_value: false, batch_size: 1000)
      case fields
      when Array
        fields = fields.index_with { default_value }
      when Hash
        # already correct type
      else
        fields = { fields => default_value }
      end
      scope = klass.where(fields.keys.map { |f| "#{f} IS NULL" }.join(" OR "))

if fields.length == 1
        # don't bother with a joined coalesce if there's only one column; it will obviously always be NULL
        updates = { fields.first.first => fields.first.last } if fields.length == 1
      else
        # update all fields in a single query, by assigning the existing non-NULL values
        # over themselves, or the default value if it is NULL
        updates = fields.map { |(f, v)| "#{f}=COALESCE(#{f},#{klass.connection.quote(v)})" }.join(", ")
      end

klass.find_ids_in_ranges(batch_size:) do |start_id, end_id|
        update_count = scope.where(klass.primary_key => start_id..end_id).update_all(updates)
        sleep_interval_per_batch = Setting.get("sleep_interval_per_backfill_nulls_batch", nil).presence&.to_f
        sleep(sleep_interval_per_batch) if update_count > 0 && sleep_interval_per_batch
      end
    end
  end
end

CANCEL

Name	Type	Size	Modified	Actions
↩ ..	DIR	—	—
📁 granular_permissions/	DIR	—	2023-09-24 03:36
📁 lti/	DIR	—	2023-09-24 03:39
📄 add_beta_domain_to_outcomes_service.rb	RB	1.1 KB	2023-09-24 03:36	EDIT
📄 add_external_tool_id_to_quiz_lti_tools.rb	RB	1.2 KB	2023-09-24 03:36	EDIT
📄 add_heap_fields_to_commons.rb	RB	1.4 KB	2023-09-24 03:36	EDIT
📄 add_lti_id_to_users.rb	RB	1.4 KB	2023-09-24 03:36	EDIT
📄 add_manage_account_banks_permission_to_quiz_lti_tools.rb	RB	1.1 KB	2023-09-24 03:36	EDIT
📄 add_new_default_report.rb	RB	937 B	2023-09-24 03:36	EDIT
📄 add_post_policies_to_assignments.rb	RB	2.9 KB	2023-09-24 03:36	EDIT
📄 add_rcs_app_host_to_quiz_lti_tools.rb	RB	1.2 KB	2023-09-24 03:36	EDIT
📄 add_rcs_service_jwt_to_quiz_lti_tools.rb	RB	1.2 KB	2023-09-24 03:36	EDIT
📄 add_role_overrides_for_new_permission.rb	RB	2.2 KB	2023-09-24 03:36	EDIT
📄 add_tool_proxy_to_message_handler.rb	RB	916 B	2023-09-24 03:36	EDIT
📄 add_user_uuid_to_learning_outcome_results.rb	RB	1.3 KB	2023-09-24 03:36	EDIT
📄 back_fill_permanent_expires_at.rb	RB	1.2 KB	2023-09-24 03:36	EDIT
📄 backfill_attachment_ids_on_media_tracks.rb	RB	1.6 KB	2023-09-24 03:36	EDIT
📄 backfill_dev_key_account_bindings.rb	RB	1.2 KB	2023-09-24 03:36	EDIT
📄 backfill_dev_key_account_bindings_for_deleted_keys.rb	RB	1.2 KB	2023-09-24 03:36	EDIT
📄 backfill_dummy_root_account_id_on_folders.rb	RB	908 B	2023-09-24 03:36	EDIT
📄 backfill_moderation_graders.rb	RB	3.3 KB	2023-09-24 03:36	EDIT
📄 backfill_new_default_help_link.rb	RB	1.4 KB	2023-09-24 03:36	EDIT
📄 backfill_nulls.rb	RB	2.7 KB	2023-09-24 03:36	EDIT
📄 backfill_posted_at_on_submissions.rb	RB	1007 B	2023-09-24 03:36	EDIT
📄 change_immersive_reader_allowed_on_to_on.rb	RB	1.4 KB	2023-09-24 03:36	EDIT
📄 cleanup_cross_shard_developer_keys.rb	RB	2.4 KB	2023-09-24 03:36	EDIT
📄 cleanup_invalid_assignment_group_rules.rb	RB	913 B	2023-09-24 03:36	EDIT
📄 clear_account_settings.rb	RB	1.1 KB	2023-09-24 03:36	EDIT
📄 clear_feature_flags.rb	RB	1.1 KB	2023-09-24 03:36	EDIT
📄 clear_old_communication_channel_root_account_ids.rb	RB	1.2 KB	2023-09-24 03:36	EDIT
📄 clear_old_user_root_account_ids.rb	RB	1.1 KB	2023-09-24 03:36	EDIT
📄 copy_big_blue_button_settings.rb	RB	1.2 KB	2023-09-24 03:36	EDIT
📄 copy_built_in_roles_by_root_account.rb	RB	3.9 KB	2023-09-24 03:36	EDIT
📄 copy_role_overrides.rb	RB	1.6 KB	2023-09-24 03:36	EDIT
📄 create_lti_links_for_legacy_lti_tool_settings.rb	RB	1.7 KB	2023-09-24 03:36	EDIT
📄 create_quiz_lti_navigation_placements.rb	RB	1.3 KB	2023-09-24 03:36	EDIT
📄 create_subscriptions_for_plagiarism_tools.rb	RB	2.9 KB	2023-09-24 03:36	EDIT
📄 delete_duplicate_rows.rb	RB	1.4 KB	2023-09-24 03:36	EDIT
📄 delete_empty_adhoc_assignment_overrides.rb	RB	1.2 KB	2023-09-24 03:36	EDIT
📄 delete_empty_progressions.rb	RB	873 B	2023-09-24 03:36	EDIT
📄 delete_extra_placeholder_submissions.rb	RB	2.1 KB	2023-09-24 03:36	EDIT
📄 delete_invalid_communication_channels.rb	RB	1.9 KB	2023-09-24 03:36	EDIT
📄 delete_scores_for_assignment_groups.rb	RB	856 B	2023-09-24 03:36	EDIT
📄 fill_root_account_ids_on_conversation_message_participant.rb	RB	1 KB	2023-09-24 03:36	EDIT
📄 fix_assignment_peer_review_jobs.rb	RB	1.4 KB	2023-09-24 03:36	EDIT
📄 fix_discussion_topic_materialized_views.rb	RB	1.9 KB	2023-09-24 03:36	EDIT
📄 fix_grader_visibility_data.rb	RB	1 KB	2023-09-24 03:36	EDIT
📄 fix_nan_group_weights.rb	RB	900 B	2023-09-24 03:36	EDIT
📄 fix_overwritten_file_module_items.rb	RB	1.3 KB	2023-09-24 03:36	EDIT
📄 fix_planner_overrides_marked_complete_data.rb	RB	884 B	2023-09-24 03:36	EDIT
📄 fix_points_possible_sums_in_quizzes.rb	RB	1.3 KB	2023-09-24 03:36	EDIT
📄 fix_reinserted_verifier_question_yaml.rb	RB	3 KB	2023-09-24 03:36	EDIT
📄 fix_resource_links_from_copied_courses.rb	RB	2.8 KB	2023-09-24 03:36	EDIT
📄 fix_resource_links_from_copied_courses_null_urls.rb	RB	2.3 KB	2023-09-24 03:36	EDIT
📄 fix_select_final_grade_for_existing_accounts.rb	RB	4.7 KB	2023-09-24 03:36	EDIT
📄 fixup_group_originality_reports.rb	RB	952 B	2023-09-24 03:36	EDIT
📄 flag_eportfolios_on_empty_users.rb	RB	1.4 KB	2023-09-24 03:36	EDIT
📄 import_instfs_attachments.rb	RB	4.1 KB	2023-09-24 03:36	EDIT
📄 initialize_submission_cached_due_date.rb	RB	979 B	2023-09-24 03:39	EDIT
📄 invalidate_built_in_role_caches.rb	RB	1.1 KB	2023-09-24 03:36	EDIT
📄 migrate_homeroom_settings_to_columns.rb	RB	1.4 KB	2023-09-24 03:36	EDIT
📄 migrate_messages_to_partitions.rb	RB	1.8 KB	2023-09-24 03:36	EDIT
📄 move_feature_flags_to_settings.rb	RB	3.8 KB	2023-09-24 03:36	EDIT
📄 move_master_import_results.rb	RB	1.5 KB	2023-09-24 03:36	EDIT
📄 move_rce_favorites_to_account_settings.rb	RB	1.2 KB	2023-09-24 03:36	EDIT
📄 move_sub_account_grading_periods_to_courses.rb	RB	2.7 KB	2023-09-24 03:36	EDIT
📄 populate_account_report_date_times.rb	RB	932 B	2023-09-24 03:36	EDIT
📄 populate_context_on_outcome_proficiencies.rb	RB	992 B	2023-09-24 03:36	EDIT
📄 populate_context_on_wiki_pages.rb	RB	1.1 KB	2023-09-24 03:36	EDIT
📄 populate_conversation_participant_private_hash.rb	RB	1.1 KB	2023-09-24 03:36	EDIT
📄 populate_course_id_on_submissions.rb	RB	331 B	2023-09-24 03:36	EDIT
📄 populate_field_on_model_from_association.rb	RB	1.7 KB	2023-09-24 03:36	EDIT
📄 populate_identity_hash_on_context_external_tools.rb	RB	2.2 KB	2023-09-24 03:36	EDIT
📄 populate_last_comment_at_on_submissions.rb	RB	641 B	2023-09-24 03:36	EDIT
📄 populate_media_tracks_web_vtt_content.rb	RB	886 B	2023-09-24 03:36	EDIT
📄 populate_missing_conversation_and_conversation_participant_root_account_ids.rb	RB	1.8 KB	2023-09-24 03:36	EDIT
📄 populate_missing_root_account_ids_if_single_root_account_install.rb	RB	4.2 KB	2023-09-24 03:36	EDIT
📄 populate_points_for_all_scores.rb	RB	2.5 KB	2023-09-24 03:36	EDIT
📄 populate_quoted_entry.rb	RB	999 B	2023-09-24 03:39	EDIT
📄 populate_root_account_id_for_assignments.rb	RB	1020 B	2023-09-24 03:36	EDIT
📄 populate_root_account_id_for_enrollment_dates_overrides.rb	RB	1.4 KB	2023-09-24 03:36	EDIT
📄 populate_root_account_id_for_group_categories.rb	RB	1.3 KB	2023-09-24 03:36	EDIT
📄 populate_root_account_id_on_asset_user_accesses.rb	RB	2.3 KB	2023-09-24 03:36	EDIT
📄 populate_root_account_id_on_attachments.rb	RB	3.7 KB	2023-09-24 03:36	EDIT
📄 populate_root_account_id_on_calendar_events.rb	RB	2.1 KB	2023-09-24 03:36	EDIT
📄 populate_root_account_id_on_models.rb	RB	26.9 KB	2023-09-24 03:36	EDIT
📄 populate_root_account_id_on_user_observers.rb	RB	1.8 KB	2023-09-24 03:36	EDIT
📄 populate_root_account_ids_for_communication_channels.rb	RB	1.6 KB	2023-09-24 03:36	EDIT
📄 populate_root_account_ids_on_communication_channels.rb	RB	1.5 KB	2023-09-24 03:36	EDIT
📄 populate_root_account_ids_on_conversation_message_participant.rb	RB	963 B	2023-09-24 03:36	EDIT
📄 populate_root_account_ids_on_conversations_tables.rb	RB	1.6 KB	2023-09-24 03:36	EDIT
📄 populate_root_account_ids_on_learning_outcomes.rb	RB	2.8 KB	2023-09-24 03:36	EDIT
📄 populate_root_account_ids_on_student_view_users.rb	RB	1.2 KB	2023-09-24 03:36	EDIT
📄 populate_root_account_ids_on_users.rb	RB	3.8 KB	2023-09-24 03:36	EDIT
📄 populate_scores_and_metadata_for_assignment_groups_and_teacher_view.rb	RB	3.5 KB	2023-09-24 03:36	EDIT
📄 populate_scores_course_score.rb	RB	885 B	2023-09-24 03:36	EDIT
📄 populate_submission_anonymous_ids.rb	RB	966 B	2023-09-24 03:39	EDIT
📄 reassociate_grading_period_groups.rb	RB	1.3 KB	2023-09-24 03:36	EDIT
📄 rebuild_quiz_submissions_from_quiz_submission_events.rb	RB	10.4 KB	2023-09-24 03:36	EDIT
📄 rebuild_quiz_submissions_from_quiz_submission_versions.rb	RB	5.1 KB	2023-09-24 03:36	EDIT
📄 recalculate_section_override_dates.rb	RB	3.5 KB	2023-09-24 03:39	EDIT
📄 reclaim_instfs_attachments.rb	RB	4.5 KB	2023-09-24 03:36	EDIT
📄 reinsert_assessment_question_file_verifiers.rb	RB	3.3 KB	2023-09-24 03:36	EDIT
📄 remove_delicious.rb	RB	1.1 KB	2023-09-24 03:36	EDIT
📄 remove_invalid_assignment_overrides.rb	RB	1.2 KB	2023-09-24 03:36	EDIT
📄 remove_invalid_course_pace_module_items.rb	RB	1.5 KB	2023-09-24 03:36	EDIT
📄 remove_invalid_locales.rb	RB	1.3 KB	2023-09-24 03:36	EDIT
📄 remove_orphaned_assignment_override_students.rb	RB	1.1 KB	2023-09-24 03:36	EDIT
📄 resend_plagiarism_events.rb	RB	7 KB	2023-09-24 03:36	EDIT
📄 run_due_date_cacher_for_quiz_lti.rb	RB	1.5 KB	2023-09-24 03:39	EDIT
📄 run_update_course_score_statistic.rb	RB	1.2 KB	2023-09-24 03:36	EDIT
📄 run_update_score_statistics.rb	RB	1.2 KB	2023-09-24 03:36	EDIT
📄 set_account_setting_enable_turnitin.rb	RB	987 B	2023-09-24 03:36	EDIT
📄 set_actl_context_type_for_course_level_tool_proxies.rb	RB	3.1 KB	2023-09-24 03:36	EDIT
📄 set_existing_binding_state.rb	RB	1.2 KB	2023-09-24 03:36	EDIT
📄 set_lti_line_items_coupled.rb	RB	3.4 KB	2023-09-24 03:36	EDIT
📄 set_navigation_placement_settings_for_quiz_lti_tools.rb	RB	1.5 KB	2023-09-24 03:36	EDIT
📄 set_posting_notification_frequency.rb	RB	4.5 KB	2023-09-24 03:36	EDIT
📄 split_up_user_preferences.rb	RB	1.2 KB	2023-09-24 03:36	EDIT
📄 turn_off_anonymous_grading_for_discussion_topics_and_quizzes.rb	RB	1.1 KB	2023-09-24 03:36	EDIT
📄 update_anonymous_grading_settings.rb	RB	3.2 KB	2023-09-24 03:36	EDIT
📄 update_developer_key_access_token_counts.rb	RB	882 B	2023-09-24 03:39	EDIT
📄 update_developer_key_scopes.rb	RB	4.5 KB	2023-09-24 03:39	EDIT
📄 update_grading_standards_to_full_range.rb	RB	1.6 KB	2023-09-24 03:36	EDIT
📄 update_mastery_connect_tool_config.rb	RB	1.2 KB	2023-09-24 03:36	EDIT
📄 validate_assignment_overrides.rb	RB	963 B	2023-09-24 03:36	EDIT