Class: LlmMemoryGmailLoader::GmailLoader

Inherits:
Object
  • Object
show all
Includes:
LlmMemory::Loader
Defined in:
lib/llm_memory_gmail_loader.rb

Instance Method Summary collapse

Constructor Details

#initialize ⇒ GmailLoader

Returns a new instance of GmailLoader.



28
29
30
 
# File 'lib/llm_memory_gmail_loader.rb', line 28

def initialize
  LlmMemoryGmailLoader.configure
end

Instance Method Details

#authorize ⇒ Object 



32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
 
# File 'lib/llm_memory_gmail_loader.rb', line 32

def authorize
  key_content = {
    "type" => "service_account",
    "project_id" => LlmMemoryGmailLoader.configuration.google_project_id,
    "private_key_id" => LlmMemoryGmailLoader.configuration.google_private_key_id,
    "private_key" => LlmMemoryGmailLoader.configuration.google_private_key,
    "client_email" => LlmMemoryGmailLoader.configuration.google_client_email,
    "client_id" => LlmMemoryGmailLoader.configuration.google_client_id,
    "auth_uri" => "https://accounts.google.com/o/oauth2/auth",
    "token_uri" => "https://oauth2.googleapis.com/token",
    "auth_provider_x509_cert_url" => "https://www.googleapis.com/oauth2/v1/certs",
    "client_x509_cert_url" => "https://www.googleapis.com/robot/v1/metadata/x509/#{LlmMemoryGmailLoader.configuration.google_client_email}"
  }

  Google::Auth::ServiceAccountCredentials.make_creds(
    json_key_io: StringIO.new(JSON.dump(key_content)),
    scope: Google::Apis::GmailV1::AUTH_GMAIL_READONLY
  )
  # authorizer.sub = "[email protected]" # Replace with the user's email address
end

#create_documents(emails) ⇒ Object 



139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
 
# File 'lib/llm_memory_gmail_loader.rb', line 139

def create_documents(emails)
  docs = []
  emails.each do |email|
    message = @service.get_user_message("me", email.id)
    subject = get_email_subject(message.payload.headers)
    from = get_email_from(message.payload.headers)
    timestamp = get_email_date(message.payload.headers)
    bodies = extract_email_bodies(message.payload)
    if bodies[:text]
      body = bodies[:text].force_encoding("UTF-8").gsub(/　/, " ").gsub(/\s+/, " ").strip
    elsif bodies[:html]
      body = extract_text_from_html(bodies[:html])
    else
      next
    end

    docs.push({
      content: "#{subject}\n#{body}",
      metadata: {
        subject: subject,
        from: from,
        timestamp: timestamp
      }
    })
  end
  docs
end

#extract_email_bodies(payload) ⇒ Object 



96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
 
# File 'lib/llm_memory_gmail_loader.rb', line 96

def extract_email_bodies(payload)
  if payload.mime_type == "text/plain" || payload.mime_type == "text/html"
    {
      text: (payload.mime_type == "text/plain") ? payload.body.data : nil,
      html: (payload.mime_type == "text/html") ? payload.body.data : nil
    }
  elsif payload.mime_type.start_with?("multipart/")
    text_body = nil
    html_body = nil

    payload.parts.each do |part|
      if part.mime_type == "text/plain"
        text_body = part.body.data
      elsif part.mime_type == "text/html"
        html_body = part.body.data
      end
    end

    {
      text: text_body,
      html: html_body
    }
  else
    {
      text: nil,
      html: nil
    }
  end
end

#extract_text_from_html(html) ⇒ Object 



126
127
128
129
130
131
132
133
134
135
136
137
 
# File 'lib/llm_memory_gmail_loader.rb', line 126

def extract_text_from_html(html)
  utf8_html = html.force_encoding("UTF-8")
  document = Nokogiri::HTML(utf8_html)
  document.css("style").remove
  document.css("br").each { |br| br.replace("\n") }
  document.css("a").each do |a|
    link = a["href"]
    text = a.text
    a.replace("#{text} (#{link})") # Replace the <a> tag with its text content and link
  end
  document.text.gsub(/　/, " ").gsub(/\s+/, " ").gsub(/(\r\n)+|\n+/, "\n").strip
end

#get_email_date(headers) ⇒ Object 



87
88
89
90
91
92
93
94
 
# File 'lib/llm_memory_gmail_loader.rb', line 87

def get_email_date(headers)
  date_header = headers.find { |header| header.name == "Date" }
  if date_header
    Time.parse(date_header.value).strftime("%Y%m%d%H%M%S")
  else
    ""
  end
end

#get_email_from(headers) ⇒ Object 



78
79
80
81
82
83
84
85
 
# File 'lib/llm_memory_gmail_loader.rb', line 78

def get_email_from(headers)
  from_header = headers.find { |header| header.name == "From" }
  if from_header
    from_header.value
  else
    ""
  end
end

#get_email_subject(headers) ⇒ Object 



73
74
75
76
 
# File 'lib/llm_memory_gmail_loader.rb', line 73

def get_email_subject(headers)
  subject_header = headers.find { |header| header.name == "Subject" }
  subject_header&.value
end

#list_emails(email:, limit: nil, query: nil) ⇒ Object 



53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
 
# File 'lib/llm_memory_gmail_loader.rb', line 53

def list_emails(email:, limit: nil, query: nil)
  limit ||= 100
  query ||= "label:sent"
  @service.authorization.sub = email
  next_page_token = nil
  sent_emails = []
  count = 0
  max_results = (limit < 100) ? limit : 100
  loop do
    result = @service.list_user_messages("me", q: query, page_token: next_page_token, max_results: max_results)
    break if result.messages.nil?
    sent_emails.concat(result.messages) if result.messages
    count += result.messages.length
    next_page_token = result.next_page_token
    break if next_page_token.nil?
    break if count > limit
  end
  sent_emails
end

#load(args) ⇒ Object 



167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
 
# File 'lib/llm_memory_gmail_loader.rb', line 167

def load(args)
  emails = args[:emails]
  limit = args[:limit]
  query = args[:query]
  raise "emails is required" if emails.nil? || emails.length == 0

  @service ||= Google::Apis::GmailV1::GmailService.new
  @service.authorization = authorize

  results = []
  emails.each do |email|
    results += list_emails(email: email, limit: limit, query: query)
  end

  create_documents(results)
end