Class: Core::ConnectorSettings

Inherits:
Object
  • Object
show all
Defined in:
lib/core/connector_settings.rb

Constant Summary collapse

DEFAULT_REQUEST_PIPELINE = 
'ent-search-generic-ingestion'
DEFAULT_EXTRACT_BINARY_CONTENT = 
true
DEFAULT_REDUCE_WHITESPACE = 
true
DEFAULT_RUN_ML_INFERENCE = 
true
DEFAULT_FILTERING = 
{}
DEFAULT_PAGE_SIZE = 
100

Class Method Summary collapse

Instance Method Summary collapse

Class Method Details

.fetch_all_connectors(page_size = DEFAULT_PAGE_SIZE) ⇒ Object 



52
53
54
55
 
# File 'lib/core/connector_settings.rb', line 52

def self.fetch_all_connectors(page_size = DEFAULT_PAGE_SIZE)
  query = { match_all: {} }
  fetch_connectors_by_query(query, page_size)
end

.fetch_by_id(connector_id) ⇒ Object 



26
27
28
29
30
31
32
 
# File 'lib/core/connector_settings.rb', line 26

def self.fetch_by_id(connector_id)
  es_response = ElasticConnectorActions.get_connector(connector_id)
  return nil unless es_response[:found]

  connectors_meta = ElasticConnectorActions.connectors_meta
  new(es_response, connectors_meta)
end

.fetch_crawler_connectors(page_size = DEFAULT_PAGE_SIZE) ⇒ Object 



47
48
49
50
 
# File 'lib/core/connector_settings.rb', line 47

def self.fetch_crawler_connectors(page_size = DEFAULT_PAGE_SIZE)
  query = { term: { service_type: Utility::Constants::CRAWLER_SERVICE_TYPE } }
  fetch_connectors_by_query(query, page_size)
end

.fetch_native_connectors(page_size = DEFAULT_PAGE_SIZE) ⇒ Object 



34
35
36
37
38
39
40
41
42
43
44
45
 
# File 'lib/core/connector_settings.rb', line 34

def self.fetch_native_connectors(page_size = DEFAULT_PAGE_SIZE)
  require 'connectors/registry' unless defined?(Connectors::REGISTRY)
  query = {
    bool: {
      filter: [
        { term: { is_native: true } },
        { terms: { service_type: Connectors::REGISTRY.registered_connectors } }
      ]
    }
  }
  fetch_connectors_by_query(query, page_size)
end

Instance Method Details

#[](property_name) ⇒ Object 



61
62
63
64
 
# File 'lib/core/connector_settings.rb', line 61

def [](property_name)
  # TODO: handle not found
  @elasticsearch_response[:_source][property_name]
end

#configuration ⇒ Object 



82
83
84
 
# File 'lib/core/connector_settings.rb', line 82

def configuration
  self[:configuration]
end

#connector_status ⇒ Object 



70
71
72
 
# File 'lib/core/connector_settings.rb', line 70

def connector_status
  self[:status]
end

#connector_status_allows_sync? ⇒ Boolean

Returns:

  • (Boolean) 


74
75
76
 
# File 'lib/core/connector_settings.rb', line 74

def connector_status_allows_sync?
  Connectors::ConnectorStatus::STATUSES_ALLOWING_SYNC.include?(connector_status)
end

#extract_binary_content? ⇒ Boolean

Returns:

  • (Boolean) 


101
102
103
 
# File 'lib/core/connector_settings.rb', line 101

def extract_binary_content?
  Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :extract_binary_content), @connectors_meta.dig(:pipeline, :default_extract_binary_content), DEFAULT_EXTRACT_BINARY_CONTENT)
end

#filtering ⇒ Object 



90
91
92
93
94
95
 
# File 'lib/core/connector_settings.rb', line 90

def filtering
  # assume for now, that first object in filtering array or a filter object itself is the only filtering object
  filtering = @elasticsearch_response.dig(:_source, :filtering)

  Utility::Filtering.extract_filter(filtering)
end

#formatted ⇒ Object 



113
114
115
116
117
 
# File 'lib/core/connector_settings.rb', line 113

def formatted
  properties = ["ID: #{id}"]
  properties << "Service type: #{service_type}" if service_type
  "connector (#{properties.join(', ')})"
end

#id ⇒ Object 



57
58
59
 
# File 'lib/core/connector_settings.rb', line 57

def id
  @elasticsearch_response[:_id]
end

#index_name ⇒ Object 



66
67
68
 
# File 'lib/core/connector_settings.rb', line 66

def index_name
  self[:index_name]
end

#needs_service_type? ⇒ Boolean

Returns:

  • (Boolean) 


119
120
121
 
# File 'lib/core/connector_settings.rb', line 119

def needs_service_type?
  service_type.to_s.strip.empty?
end

#ready_for_sync? ⇒ Boolean

Returns:

  • (Boolean) 


127
128
129
130
131
 
# File 'lib/core/connector_settings.rb', line 127

def ready_for_sync?
  Connectors::REGISTRY.registered?(service_type) &&
    valid_index_name? &&
    connector_status_allows_sync?
end

#reduce_whitespace? ⇒ Boolean

Returns:

  • (Boolean) 


105
106
107
 
# File 'lib/core/connector_settings.rb', line 105

def reduce_whitespace?
  Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :reduce_whitespace), @connectors_meta.dig(:pipeline, :default_reduce_whitespace), DEFAULT_REDUCE_WHITESPACE)
end

#request_pipeline ⇒ Object 



97
98
99
 
# File 'lib/core/connector_settings.rb', line 97

def request_pipeline
  Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :name), @connectors_meta.dig(:pipeline, :default_name), DEFAULT_REQUEST_PIPELINE)
end

#run_ml_inference? ⇒ Boolean

Returns:

  • (Boolean) 


109
110
111
 
# File 'lib/core/connector_settings.rb', line 109

def run_ml_inference?
  Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :run_ml_inference), @connectors_meta.dig(:pipeline, :default_run_ml_inference), DEFAULT_RUN_ML_INFERENCE)
end

#running? ⇒ Boolean

Returns:

  • (Boolean) 


133
134
135
 
# File 'lib/core/connector_settings.rb', line 133

def running?
  @elasticsearch_response[:_source][:last_sync_status] == Connectors::SyncStatus::IN_PROGRESS
end

#scheduling_settings ⇒ Object 



86
87
88
 
# File 'lib/core/connector_settings.rb', line 86

def scheduling_settings
  self[:scheduling]
end

#service_type ⇒ Object 



78
79
80
 
# File 'lib/core/connector_settings.rb', line 78

def service_type
  self[:service_type]
end

#update_last_sync!(job) ⇒ Object 



137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
 
# File 'lib/core/connector_settings.rb', line 137

def update_last_sync!(job)
  # if job is nil, connector still needs to be updated, to avoid it stuck at in_progress
  job_status = job&.status || Connectors::SyncStatus::ERROR
  job_error = job.nil? ? 'Could\'t find the job' : job.error
  job_error ||= 'unknown error' if job_status == Connectors::SyncStatus::ERROR
  doc = {
    :last_sync_status => job_status,
    :last_synced => Time.now,
    :last_sync_error => job_error,
    :error => job_error
  }
  if job&.terminated?
    doc[:last_indexed_document_count] = job[:indexed_document_count]
    doc[:last_deleted_document_count] = job[:deleted_document_count]
  end
  Core::ElasticConnectorActions.update_connector_fields(id, doc)
end

#valid_index_name? ⇒ Boolean

Returns:

  • (Boolean) 


123
124
125
 
# File 'lib/core/connector_settings.rb', line 123

def valid_index_name?
  index_name&.start_with?(Utility::Constants::CONTENT_INDEX_PREFIX)
end