Merge branch '214601-backfill-traversal_ids-for-projects-in-advanced-search' into 'master'

Advanced Search migration to backfill traversal_ids on projects See merge request https://gitlab.com/gitlab-org/gitlab/-/merge_requests/110820 Merged-by: Dmitry Gruzd <dgruzd@gitlab.com> Approved-by: Dmitry Gruzd <dgruzd@gitlab.com> Reviewed-by: Dmitry Gruzd <dgruzd@gitlab.com> Reviewed-by: Terri Chu <tchu@gitlab.com> Co-authored-by: Terri Chu <tchu@gitlab.com>

Merge branch '214601-backfill-traversal_ids-for-projects-in-advanced-search' into 'master'
Advanced Search migration to backfill traversal_ids on projects See merge request https://gitlab.com/gitlab-org/gitlab/-/merge_requests/110820 Merged-by: Dmitry Gruzd <dgruzd@gitlab.com> Approved-by: Dmitry Gruzd <dgruzd@gitlab.com> Reviewed-by: Dmitry Gruzd <dgruzd@gitlab.com> Reviewed-by: Terri Chu <tchu@gitlab.com> Co-authored-by: Terri Chu <tchu@gitlab.com>
10684ee0 · Dmitry Gruzd · 9cbaeb9e · 34d98286 · 10684ee0 · 10684ee0
--- a/ee/app/workers/concerns/elastic/migration_backfill_helper.rb
+++ b/ee/app/workers/concerns/elastic/migration_backfill_helper.rb
@@ -56,6 +56,13 @@ def missing_field_filter
            exists: {
              field: field_name
            }
+          },
+          must: {
+            term: {
+              type: {
+                value: self.class::DOCUMENT_TYPE.es_type
+              }
+            }
          }
        }
      }
@@ -77,7 +84,9 @@ def process_batch!
      document_references = hits.map! do |hit|
        id = hit.dig('_source', 'id')
        es_id = hit['_id']
-        es_parent = "project_#{hit.dig('_source', 'project_id')}"
+
+        # es_parent attribute is used for routing but is nil for some records, e.g., projects, users
+        es_parent = hit['_routing']

        Gitlab::Elastic::DocumentReference.new(self.class::DOCUMENT_TYPE, id, es_id, es_parent)
      end

--- a/ee/elastic/migrate/20230131184300_backfill_traversal_ids_for_projects.rb
+++ b/ee/elastic/migrate/20230131184300_backfill_traversal_ids_for_projects.rb
+# frozen_string_literal: true
+
+class BackfillTraversalIdsForProjects < Elastic::Migration
+  include Elastic::MigrationBackfillHelper
+
+  batched!
+  batch_size 10_000
+  throttle_delay 3.minutes
+
+  DOCUMENT_TYPE = Project
+  UPDATE_BATCH_SIZE = 500
+
+  private
+
+  def index_name
+    DOCUMENT_TYPE.__elasticsearch__.index_name
+  end
+
+  def field_name
+    :traversal_ids
+  end
+end
--- a/ee/lib/elastic/latest/project_class_proxy.rb
+++ b/ee/lib/elastic/latest/project_class_proxy.rb
@@ -53,7 +53,7 @@ def elastic_search(query, options: {})

      # rubocop: disable CodeReuse/ActiveRecord
      def preload_indexing_data(relation)
-        relation.includes(:project_feature, :route)
+        relation.includes(:project_feature, :route, :namespace)
      end
      # rubocop: enable CodeReuse/ActiveRecord
    end

--- a/ee/lib/elastic/latest/project_instance_proxy.rb
+++ b/ee/lib/elastic/latest/project_instance_proxy.rb
@@ -45,6 +45,10 @@ def as_indexed_json(options = {})
          data['schema_version'] = 23_01
        end

+        if ::Elastic::DataMigrationService.migration_has_finished?(:add_traversal_ids_to_original_index_mapping)
+          data['traversal_ids'] = target.elastic_namespace_ancestry
+        end
+
        # Somehow projects are being created and sent for indexing without an associated project_feature
        # https://gitlab.com/gitlab-org/gitlab/-/issues/232654
        # When this happens, log the errors to help with debugging, and raise the error to prevent indexing bad data

--- a/ee/spec/elastic/migrate/2023013001154000_add_schema_version_to_main_index_mapping_spec.rb
+++ b/ee/spec/elastic/migrate/2023013001154000_add_schema_version_to_main_index_mapping_spec.rb
--- a/ee/spec/elastic/migrate/20230131184300_backfill_traversal_ids_for_projects_spec.rb
+++ b/ee/spec/elastic/migrate/20230131184300_backfill_traversal_ids_for_projects_spec.rb
+# frozen_string_literal: true
+
+require 'spec_helper'
+require_relative 'migration_shared_examples'
+require File.expand_path('ee/elastic/migrate/20230131184300_backfill_traversal_ids_for_projects.rb')
+
+RSpec.describe BackfillTraversalIdsForProjects, :elastic_clean, :sidekiq_inline, feature_category: :global_search do
+  let(:version) { 20230131184300 }
+
+  include_examples 'migration backfills fields' do
+    let(:group) { create(:group) }
+    let(:objects) { create_list(:project, 3, :repository, namespace: group) }
+
+    let(:expected_fields) { { traversal_ids: "#{group.id}-" } }
+    let(:expected_throttle_delay) { 3.minutes }
+    let(:expected_batch_size) { 10_000 }
+
+    before do
+      create_list(:snippet, 3, :public)
+    end
+  end
+end
--- a/ee/spec/models/concerns/elastic/project_spec.rb
+++ b/ee/spec/models/concerns/elastic/project_spec.rb
@@ -217,43 +217,55 @@
  end

  describe '.as_indexed_json' do
-    [true, false].each do |migration_finished|
-      context "when add_schema_version_to_main_index_mapping is #{migration_finished ? '' : 'not '}finished" do
-        before do
-          set_elasticsearch_migration_to :add_schema_version_to_main_index_mapping, including: migration_finished
-        end
+    let_it_be(:project) { create(:project) }
+
+    it 'returns json with all needed elements' do
+      expected_hash = project.attributes.extract!(
+        'id',
+        'name',
+        'path',
+        'description',
+        'namespace_id',
+        'created_at',
+        'archived',
+        'updated_at',
+        'visibility_level',
+        'last_activity_at'
+      ).merge({
+        'join_field' => project.es_type,
+        'type' => project.es_type,
+        'schema_version' => 2301,
+        'traversal_ids' => project.elastic_namespace_ancestry,
+        'name_with_namespace' => project.full_name,
+        'path_with_namespace' => project.full_path
+      })
+
+      expected_hash.merge!(
+        project.project_feature.attributes.extract!(
+          'issues_access_level',
+          'merge_requests_access_level',
+          'snippets_access_level',
+          'wiki_access_level',
+          'repository_access_level'
+        )
+      )
+
+      expect(project.__elasticsearch__.as_indexed_json).to eq(expected_hash)
+    end

-        it 'returns json with all needed elements' do
-          project = create :project
-
-          expected_hash = project.attributes.extract!(
-            'id',
-            'name',
-            'path',
-            'description',
-            'namespace_id',
-            'created_at',
-            'archived',
-            'updated_at',
-            'visibility_level',
-            'last_activity_at'
-          ).merge({ 'join_field' => project.es_type, 'type' => project.es_type })
-
-          expected_hash.merge!(
-            project.project_feature.attributes.extract!(
-              'issues_access_level',
-              'merge_requests_access_level',
-              'snippets_access_level',
-              'wiki_access_level',
-              'repository_access_level'
-            )
-          )
-
-          expected_hash['name_with_namespace'] = project.full_name
-          expected_hash['path_with_namespace'] = project.full_path
-          expected_hash['schema_version'] = 2301 if migration_finished
-          expect(project.__elasticsearch__.as_indexed_json).to eq(expected_hash)
-        end
+    context 'when add_schema_version_to_main_index_mapping migration is not finished' do
+      it 'does not include schema_version' do
+        set_elasticsearch_migration_to :add_schema_version_to_main_index_mapping, including: false
+
+        expect(project.__elasticsearch__.as_indexed_json).not_to include(:schema_version)
+      end
+    end
+
+    context 'when backfill_traversal_ids_for_projects migration is not finished' do
+      it 'does not include traversal_ids' do
+        set_elasticsearch_migration_to :backfill_traversal_ids_for_projects, including: false
+
+        expect(project.__elasticsearch__.as_indexed_json).not_to include(:traversal_ids)
      end
    end
  end