Nlpir

A rubygem wrapper of chinese segment tools ICTCLAS2013

Installation

Add this line to your application's Gemfile:

gem 'nlpir'

And then execute:

$ bundle

Or install it yourself as:

$ gem install nlpir

Usage

Nlpir version 0.0.3 just support the *nix OS, and We`ll release it for windows platform in few days.

some DEFINE you may use :

      NLPIR_FALSE = 0
      NLPIR_TRUE = 1
      POS_MAP_NUMBER = 4
      ICT_POS_MAP_FIRST = 1            #计算所一级标注集
      ICT_POS_MAP_SECOND = 0        #计算所二级标注集
      PKU_POS_MAP_SECOND = 2        #北大二级标注集
      PKU_POS_MAP_FIRST = 3        #北大一级标注集
      POS_SIZE = 40

      Result_t = struct ['int start','int length',"char  sPOS[#{POS_SIZE}]",'int iPOS',
                      'int word_ID','int word_type','double weight']

      GBK_CODE = 0                                                    #默认支持GBK编码
      UTF8_CODE = GBK_CODE + 1                          #UTF8编码
      BIG5_CODE = GBK_CODE + 2                          #BIG5编码
      GBK_FANTI_CODE = GBK_CODE + 3             #GBK编码，里面包含繁体字

after you gem install it:

        require 'nlpir'
        include Nlpir

        s = "坚定不移沿着中国特色社会主义道路前进  为全面建成小康社会而奋斗"
        #first of all : Call the NLPIR API NLPIR_Init

        NLPIR_Init(nil, UTF8_CODE , File.expand_path("../", __FILE__))

        #example1:   Process a paragraph, and return the result text with POS or not
         puts NLPIR_ParagraphProcess(s, NLPIR_TRUE)
         puts NLPIR_ParagraphProcess(s, NLPIR_FALSE)

        #example2:   Process a paragraph, and return an array filled elements are POSed words.
        #tips: NLPIR_ParagraphProcessA() return the array, and its memory is malloced by NLPIR, it will be freed by NLPIR_Exit() (memory in server)

        words_list = NLPIR_ParagraphProcessA(s)
        i=1
        words_list.each do |a|
          sWhichDic=""
          case a.word_type
            when 0
              sWhichDic = "核心词典"
            when 1
              sWhichDic = "用户词典"
            when 2
              sWhichDic = "专业词典"
          end
          puts  "No.#{i}:start:#{a.start}, length:#{a.length}, POS_ID:#{a.sPOS},word_ID:#{a.word_ID},word_type:#{a.word_type} , UserDefine:#{sWhichDic}, Word:#{s.byteslice(a.start,a.length)}, Weight:#{a.weight}\n"
          i += 1 
        end

        #example3:   Process a paragraph, and return an array filled elements are POSed words.
        #tips: NLPIR_ParagraphProcessAW() return the array, and its memory is malloced by ruby::fiddle,and be collect by GC (memory in agent)

        words_list = NLPIR_ParagraphProcessAW(s)
        i=1
        words_list.each do |a|
          sWhichDic=""
          case a.word_type
            when 0
              sWhichDic = "核心词典"
            when 1
              sWhichDic = "用户词典"
            when 2
              sWhichDic = "专业词典"
          end
          puts  "No.#{i}:start:#{a.start}, length:#{a.length}, POS_ID:#{a.sPOS},word_ID:#{a.word_ID},word_type:#{a.word_type} , UserDefine:#{sWhichDic}, Word:#{s.byteslice(a.start,a.length)}, Weight:#{a.weight}\n"
          i += 1 
        end

        #example4:   Process a text file, and wirte the result text to file
         puts NLPIR_FileProcess("./test.txt", "./test_result.txt", NULL)


        #example5:   Get ProcessAWordCount, it returns the count of the words
         puts count = NLPIR_GetParagraphProcessAWordCount(s)


        #example6:   Add/Delete a word to the user dictionary (the path of user dictionary is ./data/userdict.dpat)
         puts NLPIR_ParagraphProcess("我们都是爱思客")
            #add a user word
         NLPIR_AddUserWord("都是爱思客 n")
         NLPIR_AddUserWord("思客 n")
         NLPIR_AddUserWord("你是 n")
         NLPIR_AddUserWord("都是客 n")
         NLPIR_AddUserWord("都是爱 n")
         puts NLPIR_ParagraphProcess("我们都是爱思客")
            #save the user word to disk
         NLPIR_SaveTheUsrDic()
         puts NLPIR_ParagraphProcess("我们都是爱思客")
            #delete a user word
        # puts i=NLPIR_DelUsrWord("都是爱思客")
         #NLPIR_DelUsrWord("爱思客")
         NLPIR_SaveTheUsrDic()
         puts NLPIR_ParagraphProcess("我们都是爱思客")


        #example7:   Import user-defined dictionary from a text file. and puts NLPIR result
         puts NLPIR_ParagraphProcess("1989年春夏之交的政治风波1989年政治风波24小时降雪量24小时降雨量863计划ABC防护训练APEC会议BB机BP机C2系统C3I系统C3系统C4ISR系统C4I系统CCITT建议")
         puts NLPIR_ImportUserDict("./userdict.txt")
          NLPIR_AddUserWord("1989年春夏之交的政治风波 n")
            #you can see the example file: ./userdict.txt to know the userdict`s format requirements
         puts NLPIR_ParagraphProcess("1989年春夏之交的政治风波1989年政治风波24小时降雪量24小时降雨量863计划ABC防护训练APEC会议BB机BP机C2系统C3I系统C3系统C4ISR系统C4I系统CCITT建议")
         NLPIR_DelUsrWord("1989年春夏之交的政治风波")
         puts NLPIR_ParagraphProcess("1989年春夏之交的政治风波1989年政治风波24小时降雪量24小时降雨量863计划ABC防护训练APEC会议BB机BP机C2系统C3I系统C3系统C4ISR系统C4I系统CCITT建议")
         puts "\n"

        #example8:   Get keywords of text
            #2nd parameter is the MaxNumber of keywords
            #3rd parameter is a swith to show the WeightOut or not  
         puts NLPIR_GetKeyWords(s, 50,NLPIR_TRUE)


        #example9:   Get keywords from file
         puts NLPIR_GetFileKeyWords("./test.txt",50, NLPIR_TRUE)


        #example10:   Find new words from text
         puts NLPIR_GetNewWords(s, 50, NLPIR_TRUE)


        #example11:   Find new words from file
         puts NLPIR_GetFileNewWords("./test.txt")


        #example12:   Extract a finger print from the paragraph 
         puts NLPIR_FingerPrint(s)


        #example13:   select which pos map will use  
            #ICT_POS_MAP_FIRST             #//计算所一级标注集
            #ICT_POS_MAP_SECOND        #//计算所二级标注集
            #PKU_POS_MAP_SECOND        #//北大二级标注集
            #PKU_POS_MAP_FIRST            #//北大一级标注集
         NLPIR_SetPOSmap(ICT_POS_MAP_FIRST)
         puts NLPIR_ParagraphProcess(s)
         NLPIR_SetPOSmap(PKU_POS_MAP_FIRST)
         puts NLPIR_ParagraphProcess(s)



        # 新词发现批量处理功能
        #以下函数为2013版本专门针对新词发现的过程，一般建议脱机实现，不宜在线处理
        #  新词识别完成后，再自动导入到分词系统中，即可完成

        NLPIR_NWI_Start() #启动新词发现功能
        f=File.new("test.txt", "r")
        text=f.read
        NLPIR_NWI_AddFile(text)#添加新词训练的文件，可反复添加
        NLPIR_NWI_Complete()#添加文件或者训练内容结束
        f.close() 
        puts NLPIR_NWI_GetResult()#输出新词识别结果
        #puts NLPIR_FileProcess("a.txt","b.txt")
        NLPIR_NWI_Result2UserDict()#新词识别结果导入到用户词典


        #at the end call NLPIR_Exit() to free system materials
        NLPIR_Exit()

Contributing

1. Fork it
2. Create your feature branch (git checkout -b my-new-feature)
3. Commit your changes (git commit -am 'Add some feature')
4. Push to the branch (git push origin my-new-feature)
5. Create new Pull Request