Class: HtmlTokenizer::Tokenizer

Inherits:

Object

• Object
• HtmlTokenizer::Tokenizer

Defined in:

ext/html_tokenizer_ext/tokenizer.c

Instance Method Summary

• #initialize ⇒ Object constructor
• #tokenize(source) ⇒ Object

Constructor Details

#initialize ⇒ Object

# File 'ext/html_tokenizer_ext/tokenizer.c', line 144

static VALUE tokenizer_initialize_method(VALUE self)
{
  struct tokenizer_t *tk = NULL;

  Tokenizer_Get_Struct(self, tk);
  DBG_PRINT("tk=%p initialize", tk);

  tokenizer_init(tk);
  tk->f_callback = tokenizer_yield_tag;

  return Qnil;
}

Instance Method Details

#tokenize(source) ⇒ Object

# File 'ext/html_tokenizer_ext/tokenizer.c', line 658

static VALUE tokenizer_tokenize_method(VALUE self, VALUE source)
{
  struct tokenizer_t *tk = NULL;
  char *c_source;
  char *old;

  if(NIL_P(source))
    return Qnil;

  Check_Type(source, T_STRING);
  Tokenizer_Get_Struct(self, tk);

  c_source = StringValueCStr(source);
  tk->scan.cursor = 0;
  tk->scan.length = strlen(c_source);
  tk->scan.enc_index = rb_enc_get_index(source);
  tk->scan.mb_cursor = 0;

  old = tk->scan.string;
  REALLOC_N(tk->scan.string, char, tk->scan.length+1);
  DBG_PRINT("tk=%p realloc(tk->scan.string) %p -> %p length=%lu", tk, old,
    tk->scan.string,  tk->scan.length+1);
  strncpy(tk->scan.string, c_source, tk->scan.length);

  tokenizer_scan_all(tk);

  DBG_PRINT("tk=%p xfree(tk->scan.string) 0x%p", tk, tk->scan.string);
  xfree(tk->scan.string);
  tk->scan.string = NULL;

  return Qtrue;
}