Class: WWDCDownloader

Inherits:
Object
  • Object
show all
Defined in:
lib/wwdcdownloader.rb

Constant Summary collapse

BASE_URI = 
'https://devimages-cdn.apple.com/wwdc-services/h8a19f8f/049CCC2F-0D8A-4F7D-BAB9-2D8F5BAA7030/contents.json'
WWDC_LIBRARIES = 
[{:base => 'https://developer.apple.com/library/content', :lib => '/navigation/library.json'}]

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(dl_dir, min_date) ⇒ WWDCDownloader

Returns a new instance of WWDCDownloader.



32
33
34
35
36
37
38
39
40
41
42
 
# File 'lib/wwdcdownloader.rb', line 32

def initialize(dl_dir, min_date)
  self.dl_dir = dl_dir
  self.min_date = min_date
  self.downloaded_files = []
  self.proxy_uri = nil

  if ENV['http_proxy'] || ENV['HTTP_PROXY']
    uri = (ENV['http_proxy']) ? ENV['http_proxy'] : ENV['HTTP_PROXY']
    self.proxy_uri = URI.parse(uri)
  end
end

Instance Attribute Details

#dl_dir ⇒ Object

Returns the value of attribute dl_dir.



30
31
32
 
# File 'lib/wwdcdownloader.rb', line 30

def dl_dir
  @dl_dir
end

#downloaded_files ⇒ Object

Returns the value of attribute downloaded_files.



30
31
32
 
# File 'lib/wwdcdownloader.rb', line 30

def downloaded_files
  @downloaded_files
end

#min_date ⇒ Object

Returns the value of attribute min_date.



30
31
32
 
# File 'lib/wwdcdownloader.rb', line 30

def min_date
  @min_date
end

#proxy_uri ⇒ Object

Returns the value of attribute proxy_uri.



30
31
32
 
# File 'lib/wwdcdownloader.rb', line 30

def proxy_uri
  @proxy_uri
end

Class Method Details

.run!(*args) ⇒ Object 



228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
 
# File 'lib/wwdcdownloader.rb', line 228

def self.run!(*args)
  puts "WWDC 2017 Session Material Downloader"
  puts "by Johannes Fahrenkrug, @jfahrenkrug, springenwerk.com"
  puts "See you next year!"
  puts
  puts "Usage: wwdcdownloader [<target-dir>]"
  puts

  dl_dir = if args.size == 1
    args.last
  else
    'wwdc2017-assets'
  end

  w = WWDCDownloader.new(dl_dir, '2017-06-01')
  w.load
  return 0
end

Instance Method Details

#download_file(url, filename, dest_dir, duplicates_ok = true) ⇒ Object 



94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
 
# File 'lib/wwdcdownloader.rb', line 94

def download_file(url, filename, dest_dir, duplicates_ok = true)
  did_download = false
  outfilename = dest_dir + "/" + filename
  if duplicates_ok or (!File.exists?(outfilename) and !self.downloaded_files.include?(url))
    # remember what we downloaded
    self.downloaded_files << url

    puts "  Downloading #{url}"
    begin
      self.read_url(url) do |downloaded_file|
        open(outfilename, 'wb') do |file|
          file.write(downloaded_file)
        end
        did_download = true
      end
    rescue Exception => e
      puts "  Download failed #{e}"
    end
  elsif !duplicates_ok
    puts "  Already downloaded this file, skipping."
  end

  did_download
end

#download_sample_code_from_book_json(book_json_url, code_base_url, dest_dir, duplicates_ok) ⇒ Object 



77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
 
# File 'lib/wwdcdownloader.rb', line 77

def download_sample_code_from_book_json(book_json_url, code_base_url, dest_dir, duplicates_ok)
  did_download = false
  self.read_url(book_json_url) do |book_json|
    if book_json[0,1] == '<'
      puts " Sorry, this samplecode apparently isn't available yet: #{code_base_url}/book.json"
    else
      book_res = JSON.parse(book_json)
      filename = book_res["sampleCode"]
      url = "#{code_base_url}/#{filename}"

      did_download = download_file(url, filename, dest_dir, duplicates_ok)
    end
  end

  did_download
end

#load ⇒ Object 



119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
 
# File 'lib/wwdcdownloader.rb', line 119

def load
  mkdir(dl_dir)

  # get the sessions JSON
  self.read_url(BASE_URI) do |body|
    res = JSON.parse(body)

    sessions = res['contents']
    resources = res['resources']

    if sessions.size > 0

      sessions.each do |session|
        if session['type'] == 'Session' && session['eventId'] == 'wwdc2017'
          title = session['title']
          session_id = session['id'].gsub('wwdc2017-', '')
          puts "Session #{session_id} '#{title}'..."

          if session['related'] && session['related']['resources']
            # Iterate over the resources
            related_resource_ids = session['related']['resources']
            did_download = false

            # get the files
            dirname = "#{dl_dir}/#{session_id}-#{title.gsub(/\/|&|!/, '')}"
            puts "  Creating #{dirname}"
            did_create_dir = mkdir(dirname)

            resources.each do |resource|
              if resource['resource_type'] == 'samplecode' && related_resource_ids.include?(resource['id'])
                puts "  Found related resource #{resource['url']}"

                # Zip file? download right away
                if resource['url'] =~ /(\w+\.zip)$/
                  uri = URI.parse(resource['url'])
                  filename = File.basename(uri.path)

                  if download_file(resource['url'], filename, dirname, true)
                    did_download = true
                  end
                # Sample code landing page?
                else
                  # Sanitize URL
                  url = resource['url'].split('/Introduction/Intro.html')[0]

                  if download_sample_code_from_book_json("#{url}/book.json", url, dirname, false)
                    did_download = true
                  end
                end
              end
            end

            if !did_download and did_create_dir
              Dir.delete(dirname)
            end
          end
        end
      end
    else
      print "No sessions :(.\n"
    end
  end

  # scrape the WWDC libraries...
  puts
  puts "Scraping the WWDC libraries..."
  WWDC_LIBRARIES.each do |lib_hash|
    lib = "#{lib_hash[:base]}#{lib_hash[:lib]}"
    puts
    puts "Library #{lib}"
    self.read_url(lib) do |body|
      body = body.gsub("''", '""')
      # Fix misformatted JSON
      body = body.gsub(/\"platform\"\s*:\s*12,$\s*\}/, "\"platform\": 12\n}")
      res = JSON.parse(body)

      docs = res['documents']

      if docs.size > 0
        docs.each do |doc|
          if doc[2] == 5 and doc[3] >= self.min_date # sample code and newer or equal to min date
            title = doc[0]

            puts "Sample Code '#{title}'..."

            # get the files
            dirname = "#{dl_dir}/#{title.gsub(/\/|&|!|:/, '')}"
            did_create_dir = mkdir(dirname)
            puts "  Created #{dirname}" if did_create_dir

            segments = doc[9].split('/')
            url = "#{lib_hash[:base]}/samplecode/#{segments[2]}/book.json"

            #puts url
            did_download = download_sample_code_from_book_json(url, "#{lib_hash[:base]}/samplecode/#{segments[2]}", dirname, false)
            if !did_download and did_create_dir
              Dir.delete( dirname )
            end
          end
        end
      else
        print "No code samples :(.\n"
      end
    end
  end

  puts "Done."
end

#mkdir(dir) ⇒ Object

Creates the given directory if it doesn’t exist already.



45
46
47
48
49
50
51
52
 
# File 'lib/wwdcdownloader.rb', line 45

def mkdir(dir)
  if File.directory?(dir)
    false
  else
    Dir.mkdir dir
    true
  end
end

#read_url(url) ⇒ Object 



54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
 
# File 'lib/wwdcdownloader.rb', line 54

def read_url(url)
  uri = URI.parse(url)

  http = nil

  if self.proxy_uri
    http = Net::HTTP.new(uri.host, uri.port, self.proxy_uri.host, self.proxy_uri.port, self.proxy_uri.user, self.proxy_uri.password)
  else
    http = Net::HTTP.new(uri.host, uri.port)
  end

  http.use_ssl = true

  http.start do |http|
   request = Net::HTTP::Get.new(uri.request_uri)
   response = http.request(request)

   if response.code == '200'
     yield(response.body)
   end
 end
end