Top Level Namespace

Includes:
Ruby_extractor

Defined Under Namespace

Modules: API, Cloner, ExtractRuby, LicenseAuto, Misc Classes: ErlangParser, GolangParser, GradleParser, License_recognition, ManifestParser, ManifestYAML, MavenParser, Message, NpmParser, PackUpdate, PacksSaver, RubyDocInfo, RubyGemsOrg, RubyGemsOrgDB, Website

Constant Summary collapse

GOLANG_STD_PACKAGES =

Golang(v1.4.2 darwin/amd64) system libraries

[
'cmd/addr2line',
'cmd/cgo',
'cmd/fix',
'cmd/go',
'cmd/gofmt',
'cmd/nm',
'cmd/objdump',
'cmd/pack',
'cmd/pprof',
'cmd/yacc',
'archive/tar',
'archive/zip',
'bufio',
'bytes',
'cmd/internal/goobj',
'cmd/internal/objfile',
'cmd/pprof/internal/commands',
'cmd/pprof/internal/driver',
'cmd/pprof/internal/fetch',
'cmd/pprof/internal/plugin',
'cmd/pprof/internal/profile',
'cmd/pprof/internal/report',
'cmd/pprof/internal/svg',
'cmd/pprof/internal/symbolizer',
'cmd/pprof/internal/symbolz',
'cmd/pprof/internal/tempfile',
'compress/bzip2',
'compress/flate',
'compress/gzip',
'compress/lzw',
'compress/zlib',
'container/heap',
'container/list',
'container/ring',
'crypto',
'crypto/aes',
'crypto/cipher',
'crypto/des',
'crypto/dsa',
'crypto/ecdsa',
'crypto/elliptic',
'crypto/hmac',
'crypto/md5',
'crypto/rand',
'crypto/rc4',
'crypto/rsa',
'crypto/sha1',
'crypto/sha256',
'crypto/sha512',
'crypto/subtle',
'crypto/tls',
'crypto/x509',
'crypto/x509/pkix',
'database/sql',
'database/sql/driver',
'debug/dwarf',
'debug/elf',
'debug/gosym',
'debug/macho',
'debug/pe',
'debug/plan9obj',
'encoding',
'encoding/ascii85',
'encoding/asn1',
'encoding/base32',
'encoding/base64',
'encoding/binary',
'encoding/csv',
'encoding/gob',
'encoding/hex',
'encoding/json',
'encoding/pem',
'encoding/xml',
'errors',
'expvar',
'flag',
'fmt',
'go/ast',
'go/build',
'go/doc',
'go/format',
'go/parser',
'go/printer',
'go/scanner',
'go/token',
'hash',
'hash/adler32',
'hash/crc32',
'hash/crc64',
'hash/fnv',
'html',
'html/template',
'image',
'image/color',
'image/color/palette',
'image/draw',
'image/gif',
'image/jpeg',
'image/png',
'index/suffixarray',
'io',
'io/ioutil',
'log',
'log/syslog',
'math',
'math/big',
'math/cmplx',
'math/rand',
'mime',
'mime/multipart',
'net',
'net/http',
'net/http/cgi',
'net/http/cookiejar',
'net/http/fcgi',
'net/http/httptest',
'net/http/httputil',
'net/http/internal',
'net/http/pprof',
'net/mail',
'net/rpc',
'net/rpc/jsonrpc',
'net/smtp',
'net/textproto',
'net/url',
'os',
'os/exec',
'os/signal',
'os/user',
'path',
'path/filepath',
'reflect',
'regexp',
'regexp/syntax',
'runtime',
'runtime/cgo',
'runtime/debug',
'runtime/pprof',
'runtime/race',
'sort',
'strconv',
'strings',
'sync',
'sync/atomic',
'syscall',
'testing',
'testing/iotest',
'testing/quick',
'text/scanner',
'text/tabwriter',
'text/template',
'text/template/parse',
'time',
'unicode',
'unicode/utf16',
'unicode/utf8',
'unsafe']
AUTO_ROOT = 
'/tmp/license_auto_cache'
LAUNCHPAD_SOURCE_DIR = 
"#{AUTO_ROOT}/lp"
MANIFEST_SOURCE_DIR = 
"#{AUTO_ROOT}/manifest"
STD_LICENSE_DIR = 
"./extractor_ruby/Package_license"
HTTPARTY_DOWNLOAD_TIMEOUT = 
480
LICENSE_WEBSITE_URL = 
'http://localhost:3000'
LICENSE_SORTED_FREQUENCY =

Statistic

[
    'MIT',
    'Apache2.0',
    'BSD',
    'GPL2.0',
    'RubyClause-6',
    'LGPL2.1',
    'GPL3.0',
    'MPL 2.0',
    'Python',
    'LGPL3.0',
    'PublicDomain',
    'EPL1.0',
    'LGPL2.0',
    'MPL1.1',
    'CDDL1.0',
    'OpenSSL',
    'GPL1.0',
    'PerlArtistic',
    'CDDL1.1',
    'Artistic2.0',
    'TP-Free',
    'Artistic1.0',
    'AFL2.1',
    'CPL1.0',
    'Apache1.0',
]

Instance Method Summary collapse

Instance Method Details

#add_product(product_name) ⇒ Object 



5
6
7
8
9
10
11
12
13
14
15
 
# File 'lib/db.rb', line 5

def add_product(product_name)
  r = $conn.exec_params("select * from product where name = $1", [product_name])
  if r.ntuples == 1
    return false, r[0]
  else
    product = $conn.exec_params("insert into product (name) select $1 returning *", [product_name])
    if product.ntuples == 1
      return true, product[0]
    end
  end
end

#add_repo(repo_name, source_url, parent_repo_id = nil, priv = -1)) ⇒ Object 



17
18
19
20
21
22
23
24
25
26
27
28
 
# File 'lib/db.rb', line 17

def add_repo(repo_name, source_url, parent_repo_id=nil, priv=-1)
  r = $conn.exec_params("select * from repo where name = $1 and source_url = $2", [repo_name, source_url])
  if r.ntuples == 1
    return false, r[0]
  else
    repo = $conn.exec_params("insert into repo (name, source_url, priv, parent_repo_id) select $1, $2, $3, $4 returning *",
                             [repo_name, source_url, priv, parent_repo_id])
    if repo.ntuples == 1
      return true, repo[0]
    end
  end
end

#api_add_pack(pack_name, pack_version, lang, homepage, source_url, license, status, cmt, project_url = nil) ⇒ Object 



105
106
107
108
109
110
111
112
113
114
115
 
# File 'lib/db.rb', line 105

def api_add_pack(pack_name, pack_version, lang, homepage, source_url, license, status, cmt, project_url=nil)
  # "select * from select add_pack('goose', 'unknown', 'Golang', null, null, null, null, null) as t(pack_id integer, new bool)"
  # $plog.info("status: #{status}")
  r = $conn.exec_params("select * from add_pack($1, $2, $3, $4, $5, $6, $7, $8, $9) as t(pack_id integer, is_newbie bool)",
                        [pack_name, pack_version, lang, homepage, source_url, license, status, cmt, project_url])
  ret = nil
  if r.ntuples == 1
    ret = r[0]
  end
  ret
end

#api_add_product_repo(release_id, parent_repo_id, sub_repo_id) ⇒ Object 



76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
 
# File 'lib/db.rb', line 76

def api_add_product_repo(release_id, parent_repo_id, sub_repo_id)
  all_products = $conn.exec_params("
    select * from product_repo
      where release_id = $1
      and repo_id = $2", [release_id, parent_repo_id])
  all_products.each {|p|
    product_id = p['product_id']
    begin
      pg_result = $conn.exec_params("select * from product_repo where release_id = $1 and product_id = $2 and repo_id = $3",
                          [release_id, product_id, sub_repo_id])
      if pg_result.ntuples == 0
        $plog.debug("release_id: #{release_id}, repo_id: #{sub_repo_id}, parent_repo_id: #{parent_repo_id}")
        r = $conn.exec_params("insert into product_repo (release_id, product_id, repo_id) values ($1, $2, $3)",
                          [release_id, product_id, sub_repo_id])
      end
    rescue Exception => e
      $plog.fatal(e)
    end
  }

end

#api_add_product_repo_pack(repo_id, pack_id, release_id) ⇒ Object 



98
99
100
101
102
103
 
# File 'lib/db.rb', line 98

def api_add_product_repo_pack(repo_id, pack_id, release_id)
  # $plog.debug("api_add_product_repo_pack: (repo_id=#{repo_id}, pack_id=#{pack_id}, release_id=#{release_id})")
  r = $conn.exec_params("select add_product_repo_pack($1, $2, $3)", [repo_id, pack_id, release_id])
  # $plog.debug("SQL: select add_product_repo_pack(#{repo_id}, #{pack_id}, #{release_id})")
  r[0]
end

#api_clear_relations(release_id, repo_id) ⇒ Object 



40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
 
# File 'lib/db.rb', line 40

def api_clear_relations(release_id, repo_id)
  r = $conn.exec_params("
    delete from product_repo_pack
      where id in (
        select r.id from product_repo_pack r
          join pack on pack.id = r.pack_id
          where product_repo_id in (
           select id from product_repo where
              release_id = $1
              and repo_id = $2
          )
          and pack.status < 41
      )", [release_id, repo_id])
  $plog.warn("You are rerunning the repo's deps, the history relation table data of this repo was deleted")
end

#api_get_case_by_id(case_id) ⇒ Object 



56
57
58
59
60
61
62
63
 
# File 'lib/db.rb', line 56

def api_get_case_by_id(case_id)
  repo_id = nil
  r = $conn.exec_params("select product_id, release_id, repo_id from product_repo where id = $1", [case_id])
  if r.ntuples == 1
    repo_id = r[0]
  end
  repo_id
end

#api_get_complete_ratio(release_id, repo_id) ⇒ Object 



117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
 
# File 'lib/db.rb', line 117

def api_get_complete_ratio(release_id, repo_id)
  # TODO: port
  ratio = 0.0
  api_path = '/api/v1/repo/complete_ratio'
  api_url = "#{LICENSE_WEBSITE_URL}#{api_path}"
  response = HTTParty.get(api_url,
                          :query => {
                            :release_id => release_id,
                            :repo_id => repo_id
                          })
  if response.code == 200
    ratio = response.licenses.to_f
    $plog.debug("complete_ratio: #{ratio}")
    if ratio >= 0.95
      release_name = nil
      repo_source_url = nil
      content = "Your release: #{release_name}, repo: #{repo_source_url} has completed, check it please."
      Message.send(content)
    end
  else
    $plog.error("#{response}")
  end
  ratio
end

#api_get_gemdata_by_name(name) ⇒ Object 



186
187
188
189
190
191
192
193
194
 
# File 'lib/db.rb', line 186

def api_get_gemdata_by_name(name)
  r = $gemconn.exec_params("select rubygems.name, versions.number, linksets.home, linksets.code, versions.licenses 
                              from rubygems, versions, linksets 
                              where rubygems.id = versions.rubygem_id 
                              and rubygems.id = linksets.rubygem_id 
                              and rubygems.name = $1 
                              and versions.latest = true 
                              and versions.platform = $2", [name, 'ruby'])
end

#api_get_gemdata_by_name_and_version(name, version) ⇒ Object 



196
197
198
199
200
201
202
203
 
# File 'lib/db.rb', line 196

def api_get_gemdata_by_name_and_version(name, version)
  r = $gemconn.exec_params("select rubygems.name, versions.number, linksets.home, linksets.code, versions.licenses 
                              from rubygems, versions, linksets 
                              where rubygems.id = versions.rubygem_id 
                              and rubygems.id = linksets.rubygem_id 
                              and rubygems.name = $1 
                              and versions.number = $2", [name, version])
end

#api_get_manifest_download_url(pack_id) ⇒ Object 



251
252
253
 
# File 'lib/db.rb', line 251

def api_get_manifest_download_url(pack_id)
  r = $conn.exec_params("select source_url from pack where id = $1", [pack_id])
end

#api_get_pack_by_id(pack_id) ⇒ Object 



142
143
144
145
146
147
148
149
150
 
# File 'lib/db.rb', line 142

def api_get_pack_by_id(pack_id)
  pack = nil
  r = $conn.exec_params("select id, name, version, source_url, lang, homepage, license, license_text, status from pack
                           where id = $1", [pack_id])
  if r.ntuples == 1
    pack = r[0]
  end
  pack
end

#api_get_packs_by_name(name, version, lang) ⇒ Object 



181
182
183
184
 
# File 'lib/db.rb', line 181

def api_get_packs_by_name(name, version, lang)
  r = $conn.exec_params("select * from pack where name = $1 and version = $2 and lang = $3",[name, version, lang])
  return r
end

#api_get_repo_by_url(url) ⇒ Object 



260
261
262
 
# File 'lib/db.rb', line 260

def api_get_repo_by_url(url)
  r = $conn.exec_params("select * from repo where source_url = $1", [url])
end

#api_get_repo_list_by_product(name, release_name, release_version) ⇒ Object 



223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
 
# File 'lib/db.rb', line 223

def api_get_repo_list_by_product(name, release_name, release_version)
  repo_list = nil
  r = $conn.exec_params("select product_repo.id, repo.name 
                            from product_repo 
                            join product on product_repo.product_id = product.id 
                            join repo on product_repo.repo_id = repo.id 
                            join release_tbl on product_repo.release_id = release_tbl.id 
                            where product.name = $1 
                            and release_tbl.name = $2 
                            and release_tbl.version = $3", [name, release_name, release_version])
  if r.ntuples > 0
    repo_list = r
  end
  repo_list
end

#api_get_repo_manifest_file_list(repo_id) ⇒ Object 



72
73
74
 
# File 'lib/db.rb', line 72

def api_get_repo_manifest_file_list(repo_id)
  r = $conn.exec_params("select ymls from repo where id = $1", [repo_id])
end

#api_get_repo_source_url(repo_id) ⇒ Object 



30
31
32
33
34
35
36
37
38
 
# File 'lib/db.rb', line 30

def api_get_repo_source_url(repo_id)
  source_url = nil
  r = $conn.exec_params("select source_url from repo where id = $1", [repo_id])
  if r.ntuples == 1
     source_url = r[0]['source_url']
  end

  source_url
end

#api_get_std_license_name(where = 'where 1 = 1') ⇒ Object 



152
153
154
 
# File 'lib/db.rb', line 152

def api_get_std_license_name(where='where 1 = 1')
  r = $conn.exec("select * from std_license #{where}")
end

#api_get_template_result_by_product(name, release_name, release_version) ⇒ Object 



205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
 
# File 'lib/db.rb', line 205

def api_get_template_result_by_product(name, release_name, release_version)
  list = nil
  r = $conn.exec_params("select product.name, repo.name, pack.name, pack.version, pack.unclear_license, pack.license, pack.license_text, pack.source_url 
                            from product_repo_pack 
                            join pack on product_repo_pack.pack_id = pack.id 
                            join product_repo on product_repo_pack.product_repo_id = product_repo.id 
                            join repo on product_repo.repo_id = repo.id 
                            join product on product_repo.product_id = product.id 
                            join release_tbl on product_repo.release_id = release_tbl.id 
                            where product.name = $1 
                            and release_tbl.name = $2 
                            and release_tbl.version = $3", [name, release_name, release_version])
  if r.ntuples > 0
    list = r
  end
  list
end

#api_get_template_result_by_product_repo_id(id) ⇒ Object 



239
240
241
242
243
244
245
246
247
248
249
 
# File 'lib/db.rb', line 239

def api_get_template_result_by_product_repo_id(id)
  list = nil
  r = $conn.exec_params("select pack.name, pack.version, pack.unclear_license, pack.license, pack.license_text, pack.source_url 
                            from product_repo_pack 
                            join pack on product_repo_pack.pack_id = pack.id 
                            where product_repo_pack.product_repo_id = $1", [id])
  if r.ntuples > 0
    list = r
  end
  list
end

#api_get_whitelist_orgs(org_url) ⇒ Object

org_url: github.com/your_org_or_person_name



256
257
258
 
# File 'lib/db.rb', line 256

def api_get_whitelist_orgs(org_url)
  r = $conn.exec_params("select * from whitelist_orgs where url = $1", [org_url])
end

#api_query_product_repo(release_id, repo_id) ⇒ Object 



65
66
67
68
69
70
 
# File 'lib/db.rb', line 65

def api_query_product_repo(release_id, repo_id)
  r = $conn.exec_params("
        select * from product_repo
          where release_id = $1
          and repo_id = $2", [release_id, repo_id])
end

#api_setup_case_status(repo_id, status, cmt) ⇒ Object 



161
162
163
164
 
# File 'lib/db.rb', line 161

def api_setup_case_status(repo_id, status, cmt)
  $plog.debug(cmt)
  r = $conn.exec_params("update product_repo set status = $1, cmt = $2, update_at = now() where id = $3", [status, cmt[0..79], repo_id])
end

#api_setup_pack_status(pack_id, status, cmt) ⇒ Object 



156
157
158
159
 
# File 'lib/db.rb', line 156

def api_setup_pack_status(pack_id, status, cmt)
  $plog.debug(cmt)
  r = $conn.exec_params("update pack set status = $1, cmt = $2, update_at = now() where id = $3", [status, cmt[0..79], pack_id])
end

#api_update_pack_info(pack_id, pack) ⇒ Object 



167
168
169
170
171
172
173
174
175
176
177
178
179
 
# File 'lib/db.rb', line 167

def api_update_pack_info(pack_id, pack)
  # r = $conn.exec_params("select update_pack($1,$2,$3,$4,$5,$6,$7,$8,$9)",[pack_id,pack['version'],pack['homepage'],pack['source_url'],pack['license_url'],pack['license'],pack['unclear_license'],pack['license_text'],pack['status']])


  r = $conn.exec_params("select update_pack($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)",
                        [pack_id, pack[:version], pack[:homepage], pack[:source_url], pack[:license_url],
                         pack[:license], pack[:unclear_license], pack[:license_text], pack[:status], pack[:project_url]])
  if(r[0] == -1)
    return false
  else
    return true
  end
end

#log_usage_example ⇒ Object 



44
45
46
47
48
49
50
 
# File 'lib/license_auto/config/config.rb', line 44

def log_usage_example()
  $plog.debug "This is a message with level DEBUG"
  $plog.info "This is a message with level INFO"
  $plog.warn "This is a message with level WARN"
  $plog.error "This is a message with level ERROR"
  $plog.fatal "This is a message with level FATAL"
end