Class: MiniCSS::CSS::Scanner

Inherits:

Object

• Object
• MiniCSS::CSS::Scanner

Defined in:

ext/minicss_scanner/minicss_scanner.c

Instance Method Summary

• #consume_token ⇒ Object
• #eof? ⇒ Boolean
• #initialize(str, allow_unicode_ranges) ⇒ Object constructor

  scanner_initialize - Ruby constructor for MiniCSS::CSS::Scanner.

• #tokens ⇒ Object

Constructor Details

#initialize(str, allow_unicode_ranges) ⇒ Object

scanner_initialize - Ruby constructor for MiniCSS::CSS::Scanner

Ruby signature:

Scanner#initialize(str)

@self: The Ruby scanner object being initialized. @str: A Ruby String containing UTF-8 CSS source text.

Behavior:

- Validates that @str is a Ruby String (raises TypeError otherwise).
- Duplicates @str to ensure scanner owns a stable buffer.
- Initializes scanner_t fields:
  * str   = duplicated Ruby String (kept alive by GC)
  * p     = pointer to start of string bytes
  * end   = pointer to one past last byte
  * idx_cp = 0 (start of code point stream)
  * line   = 1 (1-based for error reporting)
  * col    = 1 (1-based for error reporting)
- Primes the lookahead buffer (look[0..3]) by decoding
the first four UTF-8 code points from the string.

Returns:

The initialized Ruby object (self).

Notes:

- After initialization, the scanner is ready to be used
by tokenizer methods, with look[0] representing the next
code point to consume.
- The lookahead mechanism allows token classification
without repeatedly decoding UTF-8.

# File 'ext/minicss_scanner/minicss_scanner.c', line 412

static VALUE scanner_initialize(const VALUE self, VALUE str, VALUE allow_unicode_ranges) {
    Check_Type(str, T_STRING);
    scanner_t *sc;
    TypedData_Get_Struct(self, scanner_t, &scanner_type, sc);

    str = rb_str_dup(str);
    sc->str = str;
    sc->p = (const uint8_t *) RSTRING_PTR(str);
    sc->end = sc->p + RSTRING_LEN(str);
    sc->idx_cp = 0;
    sc->line = 1;
    sc->col = 1;
    sc->allow_unicode_ranges = RTEST(allow_unicode_ranges);
    sc->tokens = rb_ary_new();

    // prime lookahead
    sc->look[0] = next_utf8_cp(&sc->p, sc->end);
    sc->look[1] = next_utf8_cp(&sc->p, sc->end);
    sc->look[2] = next_utf8_cp(&sc->p, sc->end);
    sc->look[3] = next_utf8_cp(&sc->p, sc->end);

    return self;
}

Instance Method Details

#consume_token ⇒ Object

# File 'ext/minicss_scanner/minicss_scanner.c', line 1128

static VALUE scanner_consume_token(const VALUE self) {
    scanner_t *sc;
    TypedData_Get_Struct(self, scanner_t, &scanner_type, sc);

    scanner_consume_comments(sc);
    scanner_consume_whitespace(sc);

    switch (sc->look[0]) {
        case QUOTATION_MARK:
        case APOSTROPHE: {
            const int p = sc->look[0];
            scanner_consume(sc);
            scanner_consume_string_token_c(sc, p);
            return Qtrue;
        }

        case NUMBER_SIGN: {
            scanner_start_token(sc);
            const char cp1 = (char) sc->look[1], cp2 = (char) sc->look[2];

            if (ident_point(sc->look[1]) || scanner_valid_escape_c(cp1, cp2)) {
                scanner_consume(sc);
                const VALUE flag = ident_start(sc->look[0]) ? rb_intern("id") : rb_intern("unrestricted");
                const VALUE value = scanner_consume_ident_sequence(sc);
                VALUE newValue = rb_str_new_cstr("#");
                newValue = rb_str_append(newValue, value);

                const VALUE push_token_opts = rb_hash_new();
                rb_hash_aset(push_token_opts, sym_flag, flag);
                rb_hash_aset(push_token_opts, sym_literal, newValue);
                scanner_push_token_opts(sc, sym_hash, push_token_opts);
                return Qnil;
            }
            scanner_consume(sc);
            scanner_push_token_simple(sc, sym_delim);
            return Qnil;
        }

        case LEFT_PARENTHESIS: {
            scanner_start_token(sc);
            scanner_consume(sc);
            scanner_push_token_simple(sc, sym_left_parenthesis);
            return Qnil;
        }

        case RIGHT_PARENTHESIS: {
            scanner_start_token(sc);
            scanner_consume(sc);
            scanner_push_token_simple(sc, sym_right_parenthesis);
            return Qnil;
        }

        case PLUS_SIGN: {
            const char p1 = (char) sc->look[1], p2 = (char) sc->look[2];
            if (is_digit(p1) || p1 == FULL_STOP && is_digit(p2)) {
                scanner_consume_numeric_token(sc);
                return Qnil;
            }

            scanner_start_token(sc);
            scanner_consume(sc);
            scanner_push_token_simple(sc, sym_delim);
            return Qnil;
        }

        case COMMA: {
            scanner_start_token(sc);
            scanner_consume(sc);
            scanner_push_token_simple(sc, sym_comma);
            return Qnil;
        }

        case HYPHEN_MINUS: {
            if (is_digit(sc->look[1]) || (sc->look[1] == FULL_STOP && is_digit(sc->look[2]))) {
                scanner_consume_numeric_token(sc);
                return Qnil;
            }

            if (sc->look[1] == HYPHEN_MINUS && sc->look[2] == GREATER_THAN) {
                scanner_start_token(sc);
                scanner_consume(sc); // -
                scanner_consume(sc); // -
                scanner_consume(sc); // >
                scanner_push_token_simple(sc, sym_cdc);
                return Qnil;
            }

            if (scanner_ident_sequence_start(sc)) {
                scanner_consume_ident_like_token(sc);
                return Qnil;
            }

            scanner_start_token(sc);
            scanner_consume(sc);
            scanner_push_token_simple(sc, sym_delim);
            return Qnil;
        }

        case FULL_STOP: {
            if (is_digit(sc->look[1])) {
                scanner_consume_numeric_token(sc);
                return Qnil;
            }

            scanner_start_token(sc);
            scanner_consume(sc);
            scanner_push_token_simple(sc, sym_delim);
            return Qnil;
        }

        case COLON: {
            scanner_start_token(sc);
            scanner_consume(sc);
            scanner_push_token_simple(sc, sym_colon);
            return Qnil;
        }

        case SEMICOLON: {
            scanner_start_token(sc);
            scanner_consume(sc);
            scanner_push_token_simple(sc, sym_semicolon);
            return Qnil;
        }

        case LESS_THAN: {
            scanner_start_token(sc);
            if (sc->look[0] == LESS_THAN &&
                sc->look[1] == EXCLAMATION_MARK &&
                sc->look[2] == HYPHEN_MINUS &&
                sc->look[3] == HYPHEN_MINUS) {
                scanner_consume(sc); // <
                scanner_consume(sc); // !
                scanner_consume(sc); // -
                scanner_consume(sc); // -
                scanner_push_token_simple(sc, sym_cdo);
                return Qnil;
            }

            scanner_start_token(sc);
            scanner_consume(sc);
            scanner_push_token_simple(sc, sym_delim);
            return Qnil;
        }

        case COMMERCIAL_AT: {
            scanner_start_token(sc);
            scanner_consume(sc);

            if (scanner_ident_sequence_start(sc)) {
                const VALUE raw_val = scanner_consume_ident_sequence(sc);
                const VALUE val = rb_str_new_cstr("@");
                rb_str_append(val, raw_val);
                const VALUE push_opts = rb_hash_new();
                rb_hash_aset(push_opts, sym_literal, val);
                scanner_push_token_opts(sc, sym_at_keyword, push_opts);
                return Qnil;
            }

            scanner_push_token_simple(sc, sym_delim);
            return Qnil;
        }

        case LEFT_SQUARE_BRACKET: {
            scanner_start_token(sc);
            scanner_consume(sc);
            scanner_push_token_simple(sc, sym_left_square_bracket);
            return Qnil;
        }

        case REVERSE_SOLIDUS: {
            if (scanner_valid_escape) {
                scanner_consume_ident_like_token(sc);
                return Qnil;
            }

            scanner_start_token(sc);
            scanner_consume(sc);
            scanner_push_token_simple(sc, sym_delim);
            return Qnil;
        }

        case RIGHT_SQUARE_BRACKET: {
            scanner_start_token(sc);
            scanner_consume(sc);
            scanner_push_token_simple(sc, sym_right_square_bracket);
            return Qnil;
        }

        case LEFT_CURLY: {
            scanner_start_token(sc);
            scanner_consume(sc);
            scanner_push_token_simple(sc, sym_left_curly);
            return Qnil;
        }

        case RIGHT_CURLY: {
            scanner_start_token(sc);
            scanner_consume(sc);
            scanner_push_token_simple(sc, sym_right_curly);
            return Qnil;
        }

        case 0x75:
        case 0x55:
            if (sc->allow_unicode_ranges && scanner_unicode_range_start(sc) == Qtrue) {
                scanner_consume_unicode_range(sc);
                return Qnil;
            }

            scanner_consume_ident_like_token(sc);
            return Qnil;

        default: {
            if (sc->look[0] == EOF) {
                return Qnil;
            }

            if (is_ws(sc->look[0])) {
                scanner_consume_whitespace(sc);
                return Qnil;
            }

            if (is_digit(sc->look[0])) {
                scanner_consume_numeric_token(sc);
                return Qnil;
            }

            if (ident_start(sc->look[0])) {
                scanner_consume_ident_like_token(sc);
                return Qnil;
            }

            scanner_start_token(sc);
            scanner_consume(sc);
            scanner_push_token_simple(sc, sym_delim);
            return Qnil;
        }
    }
}

#eof? ⇒ Boolean

Returns:

• (Boolean)

# File 'ext/minicss_scanner/minicss_scanner.c', line 1374

static VALUE scanner_at_eof(const VALUE self) {
    scanner_t *sc;
    TypedData_Get_Struct(self, scanner_t, &scanner_type, sc);
    return sc->look[0] == EOF ? Qtrue : Qfalse;
}

#tokens ⇒ Object

# File 'ext/minicss_scanner/minicss_scanner.c', line 1368

static VALUE scanner_tokens(const VALUE self) {
    scanner_t *sc;
    TypedData_Get_Struct(self, scanner_t, &scanner_type, sc);
    return sc->tokens;
}