Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# -*- coding: utf-8 -*- 

2""" 

3 pygments.token 

4 ~~~~~~~~~~~~~~ 

5 

6 Basic token types and the standard tokens. 

7 

8 :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. 

9 :license: BSD, see LICENSE for details. 

10""" 

11 

12 

13class _TokenType(tuple): 

14 parent = None 

15 

16 def split(self): 

17 buf = [] 

18 node = self 

19 while node is not None: 

20 buf.append(node) 

21 node = node.parent 

22 buf.reverse() 

23 return buf 

24 

25 def __init__(self, *args): 

26 # no need to call super.__init__ 

27 self.subtypes = set() 

28 

29 def __contains__(self, val): 

30 return self is val or ( 

31 type(val) is self.__class__ and 

32 val[:len(self)] == self 

33 ) 

34 

35 def __getattr__(self, val): 

36 if not val or not val[0].isupper(): 

37 return tuple.__getattribute__(self, val) 

38 new = _TokenType(self + (val,)) 

39 setattr(self, val, new) 

40 self.subtypes.add(new) 

41 new.parent = self 

42 return new 

43 

44 def __repr__(self): 

45 return 'Token' + (self and '.' or '') + '.'.join(self) 

46 

47 def __copy__(self): 

48 # These instances are supposed to be singletons 

49 return self 

50 

51 def __deepcopy__(self, memo): 

52 # These instances are supposed to be singletons 

53 return self 

54 

55 

56Token = _TokenType() 

57 

58# Special token types 

59Text = Token.Text 

60Whitespace = Text.Whitespace 

61Escape = Token.Escape 

62Error = Token.Error 

63# Text that doesn't belong to this lexer (e.g. HTML in PHP) 

64Other = Token.Other 

65 

66# Common token types for source code 

67Keyword = Token.Keyword 

68Name = Token.Name 

69Literal = Token.Literal 

70String = Literal.String 

71Number = Literal.Number 

72Punctuation = Token.Punctuation 

73Operator = Token.Operator 

74Comment = Token.Comment 

75 

76# Generic types for non-source code 

77Generic = Token.Generic 

78 

79# String and some others are not direct children of Token. 

80# alias them: 

81Token.Token = Token 

82Token.String = String 

83Token.Number = Number 

84 

85 

86def is_token_subtype(ttype, other): 

87 """ 

88 Return True if ``ttype`` is a subtype of ``other``. 

89 

90 exists for backwards compatibility. use ``ttype in other`` now. 

91 """ 

92 return ttype in other 

93 

94 

95def string_to_tokentype(s): 

96 """ 

97 Convert a string into a token type:: 

98 

99 >>> string_to_token('String.Double') 

100 Token.Literal.String.Double 

101 >>> string_to_token('Token.Literal.Number') 

102 Token.Literal.Number 

103 >>> string_to_token('') 

104 Token 

105 

106 Tokens that are already tokens are returned unchanged: 

107 

108 >>> string_to_token(String) 

109 Token.Literal.String 

110 """ 

111 if isinstance(s, _TokenType): 

112 return s 

113 if not s: 

114 return Token 

115 node = Token 

116 for item in s.split('.'): 

117 node = getattr(node, item) 

118 return node 

119 

120 

121# Map standard token types to short names, used in CSS class naming. 

122# If you add a new item, please be sure to run this file to perform 

123# a consistency check for duplicate values. 

124STANDARD_TYPES = { 

125 Token: '', 

126 

127 Text: '', 

128 Whitespace: 'w', 

129 Escape: 'esc', 

130 Error: 'err', 

131 Other: 'x', 

132 

133 Keyword: 'k', 

134 Keyword.Constant: 'kc', 

135 Keyword.Declaration: 'kd', 

136 Keyword.Namespace: 'kn', 

137 Keyword.Pseudo: 'kp', 

138 Keyword.Reserved: 'kr', 

139 Keyword.Type: 'kt', 

140 

141 Name: 'n', 

142 Name.Attribute: 'na', 

143 Name.Builtin: 'nb', 

144 Name.Builtin.Pseudo: 'bp', 

145 Name.Class: 'nc', 

146 Name.Constant: 'no', 

147 Name.Decorator: 'nd', 

148 Name.Entity: 'ni', 

149 Name.Exception: 'ne', 

150 Name.Function: 'nf', 

151 Name.Function.Magic: 'fm', 

152 Name.Property: 'py', 

153 Name.Label: 'nl', 

154 Name.Namespace: 'nn', 

155 Name.Other: 'nx', 

156 Name.Tag: 'nt', 

157 Name.Variable: 'nv', 

158 Name.Variable.Class: 'vc', 

159 Name.Variable.Global: 'vg', 

160 Name.Variable.Instance: 'vi', 

161 Name.Variable.Magic: 'vm', 

162 

163 Literal: 'l', 

164 Literal.Date: 'ld', 

165 

166 String: 's', 

167 String.Affix: 'sa', 

168 String.Backtick: 'sb', 

169 String.Char: 'sc', 

170 String.Delimiter: 'dl', 

171 String.Doc: 'sd', 

172 String.Double: 's2', 

173 String.Escape: 'se', 

174 String.Heredoc: 'sh', 

175 String.Interpol: 'si', 

176 String.Other: 'sx', 

177 String.Regex: 'sr', 

178 String.Single: 's1', 

179 String.Symbol: 'ss', 

180 

181 Number: 'm', 

182 Number.Bin: 'mb', 

183 Number.Float: 'mf', 

184 Number.Hex: 'mh', 

185 Number.Integer: 'mi', 

186 Number.Integer.Long: 'il', 

187 Number.Oct: 'mo', 

188 

189 Operator: 'o', 

190 Operator.Word: 'ow', 

191 

192 Punctuation: 'p', 

193 

194 Comment: 'c', 

195 Comment.Hashbang: 'ch', 

196 Comment.Multiline: 'cm', 

197 Comment.Preproc: 'cp', 

198 Comment.PreprocFile: 'cpf', 

199 Comment.Single: 'c1', 

200 Comment.Special: 'cs', 

201 

202 Generic: 'g', 

203 Generic.Deleted: 'gd', 

204 Generic.Emph: 'ge', 

205 Generic.Error: 'gr', 

206 Generic.Heading: 'gh', 

207 Generic.Inserted: 'gi', 

208 Generic.Output: 'go', 

209 Generic.Prompt: 'gp', 

210 Generic.Strong: 'gs', 

211 Generic.Subheading: 'gu', 

212 Generic.Traceback: 'gt', 

213}