Class: JekyllOpenSdgPlugins::FetchRemoteData

Inherits:
Jekyll::Generator
  • Object
show all
Defined in:
lib/jekyll-open-sdg-plugins/fetch_remote_data.rb

Instance Method Summary collapse

Instance Method Details

#fetch_build(path) ⇒ Object

Get a build from a local folder on disk or a remote URL on the Internet.



34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
 
# File 'lib/jekyll-open-sdg-plugins/fetch_remote_data.rb', line 34

def fetch_build(path)

  is_remote = opensdg_is_path_remote(path)
  build = {}
  get_endpoints().each do |key, value|
    endpoint = is_remote ? path + '/' + value : File.join(path, fix_path(value))

    begin
      json_file = is_remote ? open(endpoint) : File.open(endpoint)
      build[key] = JSON.load(json_file)
    rescue StandardError => e
      # For backwards compatibility, forego the exception in some cases.
      abort_build = true
      if ['translations', 'indicator_downloads', 'disaggregation', 'data_packages'].include? key
        abort_build = false
      elsif endpoint.include? '/untranslated/'
        abort_build = false
      end
      if abort_build
        puts e.message
        abort 'Unable to read data from: ' + endpoint
      end
    end
  end

  return build
end

#fix_path(path) ⇒ Object

Fix a Unix path in case we are on Windows.



13
14
15
16
 
# File 'lib/jekyll-open-sdg-plugins/fetch_remote_data.rb', line 13

def fix_path(path)
  path_parts = path.split('/')
  return path_parts.join(File::SEPARATOR)
end

#generate(site) ⇒ Object 



83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
 
# File 'lib/jekyll-open-sdg-plugins/fetch_remote_data.rb', line 83

def generate(site)

  # For below, make sure there is at least an empty hash at
  # site.data.translations.
  if !site.data.has_key?('translations')
    site.data['translations'] = {}
  end

  remote = site.config['remote_data_prefix']
  local = site.config['local_data_folder']

  if !remote && !local
    abort 'Site config must include "remote_data_prefix".'
  end

  build_location = remote ? remote : local
  is_remote = opensdg_is_path_remote(build_location)

  build_location = is_remote ? build_location : File.join(Dir.pwd, build_location)
  translated_builds = site_uses_translated_builds(build_location)

  if translated_builds
    # For translated builds, we get a build for each language, and
    # place them in "subfolders" (so to speak) of site.data.
    subfolders = site.config['languages'].clone
    subfolders.append('untranslated')
    subfolders.each do |language|
      data_target = site.data[language]
      translated_build = is_remote ? build_location + '/' + language : File.join(build_location, language)
      data_source = fetch_build(translated_build)
      if !data_source.empty?
        if data_target
          data_target.deep_merge(data_source)
        else
          site.data[language] = data_source
        end
      end
    end
    # We move the language-specific translations to the
    # site.data.translations location, where all translations are kept.
    site.config['languages'].each do |language|
      translation_target = site.data['translations'][language]
      translation_source = site.data[language]['translations']
      if translation_target
        translation_target.deep_merge(translation_source)
      else
        site.data['translations'][language] = translation_source
      end
    end
    # And there are some parts of the build that don't need to be translated
    # and should be moved to the top level.
    first_language = site.config['languages'][0]
    site.data['reporting'] = site.data[first_language]['reporting']
    site.data['schema'] = site.data[first_language]['schema']
    site.data['zip'] = site.data[first_language]['zip']
  else
    # For untranslated builds, we download one build only, and place it
    # in the "root" (so to speak) of site.data. Nothing else is needed.
    target = site.data
    source = fetch_build(build_location)
    if !source.empty?
      target.deep_merge(source)
    end
  end

  # Finally support the deprecated 'remote_translations' option.
  # This is deprecated because translations should now be in the
  # data repository, where they will be fetched in fetch_build().
  if site.config['remote_translations']
    key = 'translations'
    target = site.data[key]
    site.config['remote_translations'].each do |endpoint|
      begin
        source = JSON.load(open(endpoint))
        if target
          target.deep_merge(source)
        else
          site.data[key] = source
        end
      rescue StandardError => e
        puts e.message
        abort 'Unable to fetch remote translation from: ' + endpoint
      end
    end
  end
end

#get_endpoints ⇒ Object

Our hardcoded list of pieces of the build that we expect.



19
20
21
22
23
24
25
26
27
28
29
30
31
 
# File 'lib/jekyll-open-sdg-plugins/fetch_remote_data.rb', line 19

def get_endpoints()
  return {
    'meta' => 'meta/all.json',
    'headlines' => 'headline/all.json',
    'schema' => 'meta/schema.json',
    'reporting' => 'stats/reporting.json',
    'disaggregation' => 'stats/disaggregation.json',
    'translations' => 'translations/translations.json',
    'zip' => 'zip/all_indicators.json',
    'indicator_downloads' => 'downloads/indicator-downloads.json',
    'data_packages' => 'data-packages/all.json',
  }
end

#site_uses_translated_builds(path) ⇒ Object

Predict (before data has been fetched) whether the site is using translated builds or not.



64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
 
# File 'lib/jekyll-open-sdg-plugins/fetch_remote_data.rb', line 64

def site_uses_translated_builds(path)

  is_remote = opensdg_is_path_remote(path)
  endpoints = get_endpoints()
  # For a quick test, we just use 'meta'.
  meta = endpoints['meta']
  endpoint = is_remote ? path + '/' + meta : File.join(path, fix_path(meta))

  begin
    json_file = is_remote ? open(endpoint) : File.open(endpoint)
  rescue StandardError => e
    # If we didn't find an untranslated 'meta', we assume translated builds.
    return true
  end

  # Other wise assume untranslated builds.
  return false
end