Class: Yap::Shell::Parser::Lexer

Inherits:
Object
  • Object
show all
Defined in:
lib/yap/shell/parser/lexer.rb

Defined Under Namespace

Classes: Error, HeredocMissingEndDelimiter, LineContinuationFound, NonterminatedString, Token

Constant Summary collapse

COMMAND_SUBSTITUTION = 
/\A(`|\$\()/
ARG = 
/[^\s;\|\(\)\{\}\[\]\&\!\\\<\>`][^\s;\|\(\)\{\}\[\]\&\>\<`]*/
COMMAND = 
/\A(#{ARG})/
LITERAL_COMMAND = 
/\A\\(#{ARG})/
COMMENT = 
/\A#[^$]+/
WHITESPACE = 
/\A\s+/
LH_ASSIGNMENT = 
/\A(([A-z_][\w]*)=)/
RH_VALUE = 
/\A(\S+)/
STATEMENT_TERMINATOR = 
/\A(;)/
PIPE_TERMINATOR = 
/\A(\|)/
CONDITIONAL_TERMINATOR = 
/\A(&&|\|\|)/
HEREDOC_START = 
/\A<<-?([A-z0-9]+)\s*\n/
INTERNAL_EVAL = 
/\A(?:(\!)|([0-9]+))/
SUBGROUP = 
/\A(\(|\))/
REDIRECTION1 = 
/\A(([12]?>>)\s*(#{ARG}))/
REDIRECTION2 = 
/\A(([12]?>&?[12]?)\s*(?![12][\?]?>)(#{ARG})?)/
REDIRECTION3 = 
/\A((&>|<|&>>)\s*(#{ARG}))/
NUMERIC_RANGE = 
/\A\(((\d+)\.\.(\d+))\)(\.each)?/
NUMERIC_RANGE_W_CALL = 
/\A\(((\d+)\.\.(\d+))\)(\.each)?\s*:\s*/
NUMERIC_RANGE_W_PARAM = 
/\A(\((\d+)\.\.(\d+))\)\s+as\s+([A-z0-9,\s]+)\s*:\s*/
NUMERIC_REPETITION = 
/\A((\d+)(\.times))/
NUMERIC_REPETITION_2 = 
/\A((\d+)(\.times))\s*:\s*/
NUMERIC_REPETITION_W_PARAM = 
/\A((\d+)(\.times))\s+as\s+([A-z0-9,\s]+)\s*:\s*/
BLOCK_BEGIN = 
/\A\s+(\{)\s*(?:\|\s*([A-z0-9,\s]+)\s*\|)?/
BLOCK_END = 
/\A\s+(\})\s*/
LINE_CONTINUATION = 
/.*\\\Z/
SPLIT_BLOCK_PARAMS_RGX = 
/\s*,\s*/

Instance Method Summary collapse

Instance Method Details

#each_command_substitution_for(input, &blk) ⇒ Object

Loop over the given input and yield command substitutions. This yields an object that responds to #str, and #position.

  • The #str will be the contents of the command substitution, e.g. foo in ‘foo` or $(foo)

  • The #position will be range denoting where the command substitution started and stops in the string

This will yield a result for every command substitution found.

Note

This will not yield nested command substitutions. The caller is responsible for that.



85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
 
# File 'lib/yap/shell/parser/lexer.rb', line 85

def each_command_substitution_for(input, &blk)
  return unless input

  i = 0
  loop do
    break if i >= input.length

    @chunk = input[i..-1]
    if md=@chunk.match(COMMAND_SUBSTITUTION)
      start = i
      delimiter = md[1] == "$(" ? ")" : md[1]
      result = process_string @chunk[md[0].length-1..-1], delimiter
      consumed_length_so_far = result.consumed_length + (md[0].length - 1)
      i += consumed_length_so_far
      yield OpenStruct.new(str:result.str, position:(start..i))
    else
      i += 1
    end
  end
end

#tokenize(str) ⇒ Object 



106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
 
# File 'lib/yap/shell/parser/lexer.rb', line 106

def tokenize(str)
  debug_log "##{__callee__} entered with: #{str.inspect}"
  @chunk = str
  @tokens = []
  @lineno = 0
  @looking_for_args = false
  @tokens_to_add_when_done = []

  max = 100
  count = 0
  @current_position = 0
  last_position = -1
  process_next_chunk = -> { @chunk = str.slice(@current_position..-1) ; @chunk != "" }

  strip_line_continutations_before_newlines
  line_continuation_token

  while process_next_chunk.call
    debug_log "##{__callee__} looking for next token in: #{@chunk.inspect}"
    result =
      comment_token ||
      block_token ||
      numerical_range_token ||
      command_substitution_token ||
      subgroup_token ||
      assignment_token ||
      literal_command_token ||
      string_token ||
      command_token ||
      whitespace_token ||
      terminator_token ||
      redirection_token ||
      heredoc_token ||
      argument_token ||
      internal_eval_token

    count += 1
    # raise "Infinite loop detected on #{@chunk.inspect}" if count == max
    raise "Infinite loop detected in #{str.inspect}\non chunk:\n  #{@chunk.inspect}" if @current_position == last_position

    last_position = @current_position
    @current_position += result.to_i
  end

  @tokens_to_add_when_done.each do |args|
    token *args
  end

  token :BlankLine, str if @tokens.empty?

  debug_log "##{__callee__} exit returning: #{@tokens.inspect}"
  @tokens
end