Class: HtmlTokenizer::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
ext/better_html_ext/tokenizer.c

Instance Method Summary collapse

Constructor Details

#initialize ⇒ Object 



150
151
152
153
154
155
156
157
158
159
160
161
 
# File 'ext/better_html_ext/tokenizer.c', line 150

static VALUE tokenizer_initialize_method(VALUE self)
{
  struct tokenizer_t *tk = NULL;

  Tokenizer_Get_Struct(self, tk);
  DBG_PRINT("tk=%p initialize", tk);

  tokenizer_init(tk);
  tk->f_callback = tokenizer_yield_tag;

  return Qnil;
}

Instance Method Details

#tokenize(source) ⇒ Object 



687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
 
# File 'ext/better_html_ext/tokenizer.c', line 687

static VALUE tokenizer_tokenize_method(VALUE self, VALUE source)
{
  struct tokenizer_t *tk = NULL;
  char *c_source;

  if(NIL_P(source))
    return Qnil;

  Check_Type(source, T_STRING);
  Tokenizer_Get_Struct(self, tk);

  c_source = StringValueCStr(source);
  tk->scan.cursor = 0;
  tokenizer_set_scan_string(tk, c_source, strlen(c_source));
  tk->scan.enc_index = rb_enc_get_index(source);
  tk->scan.mb_cursor = 0;

  tokenizer_scan_all(tk);

  tokenizer_free_scan_string(tk);

  return Qtrue;
}