| Class | CodeRay::Scanners::JavaScript |
| In: |
lib/coderay/scanners/java_script.rb
|
| Parent: | Scanner |
| KEYWORDS | = | %w[ break case catch continue default delete do else finally for function if in instanceof new return switch throw try typeof var void while with ] | The actual JavaScript keywords. | |
| PREDEFINED_CONSTANTS | = | %w[ false null true undefined ] | ||
| MAGIC_VARIABLES | = | %w[ this arguments ] | ||
| KEYWORDS_EXPECTING_VALUE | = | WordList.new.add %w[ case delete in instanceof new return throw typeof with ] | ||
| RESERVED_WORDS | = | %w[ abstract boolean byte char class debugger double enum export extends final float goto implements import int interface long native package private protected public short static super synchronized throws transient volatile ] | Reserved for future use. | |
| IDENT_KIND | = | WordList.new(:ident). add(RESERVED_WORDS, :reserved). add(PREDEFINED_CONSTANTS, :pre_constant). add(MAGIC_VARIABLES, :local_variable). add(KEYWORDS, :keyword) | ||
| ESCAPE | = | / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x | ||
| UNICODE_ESCAPE | = | / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x | ||
| REGEXP_ESCAPE | = | / [bBdDsSwW] /x | ||
| STRING_CONTENT_PATTERN | = | { "'" => /[^\\']+/, '"' => /[^\\"]+/, '/' => /[^\\\/]+/, } | ||
| KEY_CHECK_PATTERN | = | { "'" => / (?> [^\\']* (?: \\. [^\\']* )* ) ' \s* : /mx, '"' => / (?> [^\\"]* (?: \\. [^\\"]* )* ) " \s* : /mx, } |
# File lib/coderay/scanners/java_script.rb, line 54
54: def scan_tokens tokens, options
55:
56: state = :initial
57: string_delimiter = nil
58: value_expected = true
59: key_expected = false
60: function_expected = false
61:
62: until eos?
63:
64: kind = nil
65: match = nil
66:
67: case state
68:
69: when :initial
70:
71: if match = scan(/ \s+ | \\\n /x)
72: value_expected = true if !value_expected && match.index(?\n)
73: tokens << [match, :space]
74: next
75:
76: elsif scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
77: value_expected = true
78: kind = :comment
79:
80: elsif check(/\.?\d/)
81: key_expected = value_expected = false
82: if scan(/0[xX][0-9A-Fa-f]+/)
83: kind = :hex
84: elsif scan(/(?>0[0-7]+)(?![89.eEfF])/)
85: kind = :oct
86: elsif scan(/\d+[fF]|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/)
87: kind = :float
88: elsif scan(/\d+/)
89: kind = :integer
90: end
91:
92: elsif value_expected && match = scan(/<([[:alpha:]]\w*) (?: [^\/>]*\/> | .*?<\/\1>)/xim)
93: # FIXME: scan over nested tags
94: xml_scanner.tokenize match
95: value_expected = false
96: next
97:
98: elsif match = scan(/ [-+*=<>?:;,!&^|(\[{~%]+ | \.(?!\d) /x)
99: value_expected = true
100: last_operator = match[-1]
101: key_expected = (last_operator == ?{) || (last_operator == ?,)
102: function_expected = false
103: kind = :operator
104:
105: elsif scan(/ [)\]}]+ /x)
106: function_expected = key_expected = value_expected = false
107: kind = :operator
108:
109: elsif match = scan(/ [$a-zA-Z_][A-Za-z_0-9$]* /x)
110: kind = IDENT_KIND[match]
111: value_expected = (kind == :keyword) && KEYWORDS_EXPECTING_VALUE[match]
112: # TODO: labels
113: if kind == :ident
114: if match.index(?$) # $ allowed inside an identifier
115: kind = :predefined
116: elsif function_expected
117: kind = :function
118: elsif check(/\s*[=:]\s*function\b/)
119: kind = :function
120: elsif key_expected && check(/\s*:/)
121: kind = :key
122: end
123: end
124: function_expected = (kind == :keyword) && (match == 'function')
125: key_expected = false
126:
127: elsif match = scan(/["']/)
128: if key_expected && check(KEY_CHECK_PATTERN[match])
129: state = :key
130: else
131: state = :string
132: end
133: tokens << [:open, state]
134: string_delimiter = match
135: kind = :delimiter
136:
137: elsif value_expected && (match = scan(/\/(?=\S)/))
138: tokens << [:open, :regexp]
139: state = :regexp
140: string_delimiter = '/'
141: kind = :delimiter
142:
143: elsif scan(/ \/ /x)
144: value_expected = true
145: key_expected = false
146: kind = :operator
147:
148: else
149: getch
150: kind = :error
151:
152: end
153:
154: when :string, :regexp, :key
155: if scan(STRING_CONTENT_PATTERN[string_delimiter])
156: kind = :content
157: elsif match = scan(/["'\/]/)
158: tokens << [match, :delimiter]
159: if state == :regexp
160: modifiers = scan(/[gim]+/)
161: tokens << [modifiers, :modifier] if modifiers && !modifiers.empty?
162: end
163: tokens << [:close, state]
164: string_delimiter = nil
165: key_expected = value_expected = false
166: state = :initial
167: next
168: elsif state != :regexp && (match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox))
169: if string_delimiter == "'" && !(match == "\\\\" || match == "\\'")
170: kind = :content
171: else
172: kind = :char
173: end
174: elsif state == :regexp && scan(/ \\ (?: #{ESCAPE} | #{REGEXP_ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
175: kind = :char
176: elsif scan(/\\./m)
177: kind = :content
178: elsif scan(/ \\ | $ /x)
179: tokens << [:close, state]
180: kind = :error
181: key_expected = value_expected = false
182: state = :initial
183: else
184: raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
185: end
186:
187: else
188: raise_inspect 'Unknown state', tokens
189:
190: end
191:
192: match ||= matched
193: if $CODERAY_DEBUG and not kind
194: raise_inspect 'Error token %p in line %d' %
195: [[match, kind], line], tokens
196: end
197: raise_inspect 'Empty token', tokens unless match
198:
199: tokens << [match, kind]
200:
201: end
202:
203: if [:string, :regexp].include? state
204: tokens << [:close, state]
205: end
206:
207: tokens
208: end
# File lib/coderay/scanners/java_script.rb, line 212
212: def reset_instance
213: super
214: @xml_scanner.reset if defined? @xml_scanner
215: end