Class: Sync

Inherits:
Object
  • Object
show all
Defined in:
lib/examples/sync.rb

Instance Method Summary collapse

Constructor Details

#initialize ⇒ Sync

Returns a new instance of Sync.



32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
 
# File 'lib/examples/sync.rb', line 32

def initialize
  if $stdout.tty? then
    @log = Logger.new($stdout)
    @log.level = Logger::DEBUG
  else
    @log = Logger.new(File.expand_path(E621::Config.paths["logging"]))
    @log.level = Logger::INFO
  end
  @log.formatter = proc do |sev,dat,prog,msg|
    "#{Time.now.strftime($form)} [#{sev.pad(5)}] #{msg}#$/"
  end
  E621::Config.config = File.expand_path("~/.hexagon/conf.json")
  api = API.new("user") # Used for login
  @log.info("Program #$0 started.")
  @uid = api.get("index",{"name"=>api.user}).first["id"]
  @max_wait = 1
  tags
end

Instance Method Details

#download(post, name, set, id = 0) ⇒ Object 



159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
 
# File 'lib/examples/sync.rb', line 159

def download(post,name,set,id=0)
  artist = post.tags.split(/\s+/).map do |t|
    @tags[@tags.index(t)] if @tags.index(t)
  end
  begin
    artist = artist.compact.first.sub("_(artist)","")
  rescue
    artist = "unknown"
  end
  if @mod == "set" then
    string = [post.created_at.to_i,post.id.pad(8),artist+"_"+name,post.file_ext]
  elsif @mod == "pool" then
    string = [set.id.pad(5),id.pad(4),post.id.pad(8),artist,name.downcase.gsub(/\s/,"_"),post.file_ext]
  end
  string = string.join(".")
  tries = 0
  begin
    post.download("#{name}/#{string}")
  rescue
    raise if tries > 4
    tries += 1
    puts "#{Time.now.strftime($form)}: #$!"
    @log.info("#$!")
    sleep 1
    retry
  end
  File.utime(post.created_at,post.created_at,"#{name}/#{string}")
  @posts["downloads"] << post.id
  jposts = @posts.to_json
  File.open(@set_files+"/#{name}.json","w"){|f|f.print jposts}
  s = "Downloaded post #{" "*(6-post.id.to_s.length)}#{post.id} from \"#{set.name}\"."
  puts "#{Time.now.strftime($form)}: #{s}"
  sleep(rand*@max_wait)
  @log.info(s)
end

#set(set) ⇒ Object 



118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
 
# File 'lib/examples/sync.rb', line 118

def set(set)
  sid, owner, query = set["id"], set["owner"], set["search"]
  if owner == uid then
    Search.new("fav:maxine_red #{query} order:id".split(" ")).each_post do |post|
      next if @posts["downloads"].include?(post.id)
      download(post,name,set)
    end
    set = api.get("show", {"id"=>sid})
    posts = set["posts"].map{|post|Post.new(post).id}
    name = set["shortname"]
    local = Dir["#{name}/*"].reject{|x|x.match(/db$/)}.map{|x|x.split(".")[1].to_i}.sort.uniq
    (posts-local).each do |id|
      post = Post.new({"id"=>id})
      f = post.remove_from_set(sid)
      if f["success"] then
        s = "Removed Post #{" "*(6-post.id.to_s.length)}#{post.id} from \"#{set["name"]}\""
        puts "#{Time.now.strftime($form)}: \e[1;33m#{s}\e[0m"
        @log.info(s)
      else
        s = "#{f["reason"].to_s.gsub(/<.+?>/,"")}."
        puts "#{Time.now.strftime($form)}: \e[1;31m#{s}\e[0m"
        @log.info(s)
      end
    end
    (local-posts).each do |id|
      next if id == 0
      post = Post.new({"id"=>id})
      f = post.add_to_set(sid)
      if f["success"] then
        s = "Added Post #{" "*(6-post.id.to_s.length)}#{post.id} to \"#{set["name"]}\"."
        puts "#{Time.now.strftime($form)}: #{s}"
        @log.info(s)
      else
        s = "#{f["reason"].gsub(/<.+?>/,"")}. Removing local file."
        puts "#{Time.now.strftime($form)}: #{s}"
        @log.info(s)
        File.unlink(Dir["#{set["shortname"]}/*.#{"0"*(8-post.id.to_s.length)}#{post.id}.*"].first)
      end
    end
  end
end

#tags ⇒ Object 



51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
 
# File 'lib/examples/sync.rb', line 51

def tags
  tags = File.expand_path("~/.hexagon/tags.json")
  if !File.exist?(tags) then
    @tags = Array.new
  else
    File.open(tags) do |f|
      @tags = f.read.parse
    end
  end
  @tags = @tags.sort{|k1,k2|k1["id"]<=>k2["id"]}
  api = API.new("tag")
  body,page = [2],1
  until body == Array.new do
    body = api.get("index",{"limit"=>0,"order"=>"count","after_id"=>@tags.last["id"],"page"=>page})
    body = body.map{|x|x["name"] = x["name"].encode("us-ascii", :invalid => :replace, :undef => :replace, :replace => "");x}
    @tags += body
    page += 1
  end
  jtags = @tags.to_json
  File.open(tags,"w"){|f|f.print jtags}
  @tags = @tags.reject{|x|x["type"]!=1}.map{|x|x["name"]}
end

#worker(mod) ⇒ Object 



74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
 
# File 'lib/examples/sync.rb', line 74

def worker(mod)
  @mod = mod
  Dir.chdir(File.expand_path("~/Dropbox/Furry/e621/#{mod}s")) do
    sets = Array.new
    File.open(File.expand_path(E621::Config.paths["j#{mod}s"])) do |f|
      sets = f.read.parse
    end
    @set_files = File.expand_path(E621::Config.paths["sets"])
    api = API.new(mod)
    sets.each do |oset|
      id = mod == "set" ? oset["id"] : oset
      if mod == "set" then
        set = Set.new(api.get("show", {"id"=>id}))
      elsif mod == "pool" then
        set = Pool.new(api.get("show", {"id"=>id}))
      end
      set.name = set.name.gsub("_"," ")
      @log.info("Fetching #{mod} \"#{set.name}\"")
      set.name = "Love_Can_Be_Different" if set.name == "Love_can_be_different"
      set.name = set.name.gsub("_"," ").encode("us-ascii", :invalid => :replace, :undef => :replace, :replace => "")
      set.name = set.name.gsub(/[^0-9, ,_,a-z,\-]/i,"").sub(/\s+$/,"")
      name = mod == "set" ? set.shortname : id.pad(5)
      if !File.exists?(@set_files+"/#{name}.json") then
        File.open(@set_files+"/#{name}.json","w"){|f|f.print "{\"downloads\":[],\"updated_at\":0}"}
      end 
      File.open(@set_files+"/#{name}.json"){|f|@posts = f.read.parse}
      @posts.store("updated_at",0) if !@posts.has_key?("updated_at")
      next if set.updated_at.to_i <= @posts["updated_at"].to_i
      dname = mod == "pool" ? set.name : name
      Dir.mkdir(dname) unless File.exist?(dname)
      i = 1
      set.each_post do |post|
        next if @posts["downloads"].include?(post.id)
        download(post,dname,set,i)
        i += 1
      end
      advanced_set(oset) if mod == "set"
    end
  end
  @posts["updated_at"] = set.updated_at.to_i
  jposts = @posts.to_json
  File.open(@set_files+"/#{name}.json","w"){|f|f.print jposts}
end