Package cssutils :: Package tests :: Module test_tokenize2
[hide private]
[frames] | no frames]

Source Code for Module cssutils.tests.test_tokenize2

  1  # -*- coding: utf-8 -*- 
  2  """testcases for new cssutils.tokenize.Tokenizer 
  3   
  4  TODO:: 
  5   
  6      - escape ends with explicit space but \r\n as single space 
  7      - ur'"\""': [('STRING', ur'"\""', 1, 1)], 
  8      - font-face with escaped "-" 
  9   
 10  + old tests as new ones are **not complete**! 
 11  """ 
 12  __author__ = '$LastChangedBy: cthedot $' 
 13  __date__ = '$LastChangedDate: 2007-09-01 15:56:36 +0200 (Sa, 01 Sep 2007) $' 
 14  __version__ = '$LastChangedRevision: 302 $' 
 15   
 16  import xml.dom 
 17  import basetest 
 18  from cssutils.tokenize2 import * 
 19   
20 -class TokenizerTestCase(basetest.BaseTestCase):
21 22 testsall = { 23 # IDENT 24 u'äöü߀': [('IDENT', u'äöü߀', 1, 1)], 25 u' a ': [('S', u' ', 1, 1), 26 ('IDENT', u'a', 1, 2), 27 ('S', u' ', 1, 3)], 28 u'_a': [('IDENT', u'_a', 1, 1)], 29 u'-a': [('IDENT', u'-a', 1, 1)], 30 u'aA-_\200\377': [('IDENT', u'aA-_\200\377', 1, 1)], 31 u'a1': [('IDENT', u'a1', 1, 1)], 32 # escapes must end with S or max 6 digits: 33 u'\\44\nb': [('IDENT', u'Db', 1, 1)], 34 u'\\44\rb': [('IDENT', u'Db', 1, 1)], 35 u'\\44\fb': [('IDENT', u'Db', 1, 1)], 36 u'\\44\n*': [('IDENT', u'D', 1, 1), 37 ('CHAR', u'*', 2, 1)], 38 u'\\44 a': [('IDENT', u'D', 1, 1), 39 ('S', u' ', 1, 5), 40 ('IDENT', u'a', 1, 6)], 41 # TODO: 42 # Note that this means that a "real" space after the escape sequence 43 # must itself either be escaped or doubled: 44 u'\\44\ x': [('IDENT', u'D\\ x', 1, 1)], 45 u'\\44 ': [('IDENT', u'D', 1, 1), 46 ('S', u' ', 1, 5)], 47 48 ur'\44': [('IDENT', u'D', 1, 1)], 49 ur'\\': [('IDENT', ur'\\', 1, 1)], 50 ur'\{': [('IDENT', ur'\{', 1, 1)], 51 ur'\"': [('IDENT', ur'\"', 1, 1)], 52 ur'\(': [('IDENT', ur'\(', 1, 1)], 53 ur'\1 \22 \333 \4444 \55555 \666666 \777777 7 \7777777': [ 54 ('IDENT', u'\x01"\u0333\u4444\\55555 \\666666 \\777777 7', 1, 1), 55 ('S', ' ', 1, 43), 56 ('IDENT', '\\7777777', 1, 44)], 57 58 59 u'\\1 b': [('IDENT', u'\x01b', 1, 1)], 60 u'\\44 b': [('IDENT', u'Db', 1, 1)], 61 u'\\123 b': [('IDENT', u'\u0123b', 1, 1)], 62 u'\\1234 b': [('IDENT', u'\u1234b', 1, 1)], 63 u'\\12345 b': [('IDENT', u'\\12345 b', 1, 1)], 64 u'\\123456 b': [('IDENT', u'\\123456 b', 1, 1)], 65 u'\\1234567 b': [('IDENT', u'\\1234567', 1, 1), 66 ('S', u' ', 1, 9), 67 ('IDENT', u'b', 1, 10)], 68 u'\\{\\}\\(\\)\\[\\]\\#\\@\\.\\,': 69 [('IDENT', u'\\{\\}\\(\\)\\[\\]\\#\\@\\.\\,', 1, 1)], 70 71 # STRING 72 u' "" ': [('S', u' ', 1, 1), 73 ('STRING', u'""', 1, 2), 74 ('S', u' ', 1, 4)], 75 u' "\'" ': [('S', u' ', 1, 1), 76 ('STRING', u'"\'"', 1, 2), 77 ('S', u' ', 1, 5)], 78 u" '' ": [('S', u' ', 1, 1), 79 ('STRING', u"''", 1, 2), 80 ('S', u' ', 1, 4)], 81 u" '' ": [('S', u' ', 1, 1), 82 ('STRING', u"''", 1, 2), 83 ('S', u' ', 1, 4)], 84 u"'\\\n'": [('STRING', u"'\\\n'", 1, 1)], 85 u"'\\\n\\\n\\\n'": [('STRING', u"'\\\n\\\n\\\n'", 1, 1)], 86 u"'\\\f'": [('STRING', u"'\\\f'", 1, 1)], 87 u"'\\\r'": [('STRING', u"'\\\r'", 1, 1)], 88 u"'\\\r\n'": [('STRING', u"'\\\r\n'", 1, 1)], 89 u"'1\\\n2'": [('STRING', u"'1\\\n2'", 1, 1)], 90 91 # HASH 92 u' #a ': [('S', u' ', 1, 1), 93 ('HASH', u'#a', 1, 2), 94 ('S', u' ', 1, 4)], 95 96 u'#ccc': [('HASH', u'#ccc', 1, 1)], 97 u'#111': [('HASH', u'#111', 1, 1)], 98 u'#a1a1a1': [('HASH', u'#a1a1a1', 1, 1)], 99 u'#1a1a1a': [('HASH', u'#1a1a1a', 1, 1)], 100 101 # NUMBER, for plus see CSS3 102 u' 0 ': [('S', u' ', 1, 1), 103 ('NUMBER', u'0', 1, 2), 104 ('S', u' ', 1, 3)], 105 u' 0.1 ': [('S', u' ', 1, 1), 106 ('NUMBER', u'0.1', 1, 2), 107 ('S', u' ', 1, 5)], 108 u' .0 ': [('S', u' ', 1, 1), 109 ('NUMBER', u'.0', 1, 2), 110 ('S', u' ', 1, 4)], 111 112 u' -0 ': [('S', u' ', 1, 1), 113 ('CHAR', u'-', 1, 2), 114 ('NUMBER', u'0', 1, 3), 115 ('S', u' ', 1, 4)], 116 117 # PERCENTAGE 118 u' 0% ': [('S', u' ', 1, 1), 119 ('PERCENTAGE', u'0%', 1, 2), 120 ('S', u' ', 1, 4)], 121 u' .5% ': [('S', u' ', 1, 1), 122 ('PERCENTAGE', u'.5%', 1, 2), 123 ('S', u' ', 1, 5)], 124 125 # URI 126 u' url() ': [('S', u' ', 1, 1), 127 ('URI', u'url()', 1, 2), 128 ('S', u' ', 1, 7)], 129 u' url(a) ': [('S', u' ', 1, 1), 130 ('URI', u'url(a)', 1, 2), 131 ('S', u' ', 1, 8)], 132 u' url("a") ': [('S', u' ', 1, 1), 133 ('URI', u'url("a")', 1, 2), 134 ('S', u' ', 1, 10)], 135 u' url( a ) ': [('S', u' ', 1, 1), 136 ('URI', u'url( a )', 1, 2), 137 ('S', u' ', 1, 10)], 138 u'ur\\l(': [('FUNCTION', u'ur\\l(', 1, 1)], 139 140 # UNICODE-RANGE 141 142 # CDO 143 u' <!-- ': [('S', u' ', 1, 1), 144 ('CDO', u'<!--', 1, 2), 145 ('S', u' ', 1, 6)], 146 u'"<!--""-->"': [('STRING', u'"<!--"', 1, 1), 147 ('STRING', u'"-->"', 1, 7)], 148 149 # CDC 150 u' --> ': [('S', u' ', 1, 1), 151 ('CDC', u'-->', 1, 2), 152 ('S', u' ', 1, 5)], 153 154 # S 155 u' ': [('S', u' ', 1, 1)], 156 u' ': [('S', u' ', 1, 1)], 157 u'\r': [('S', u'\r', 1, 1)], 158 u'\n': [('S', u'\n', 1, 1)], 159 u'\r\n': [('S', u'\r\n', 1, 1)], 160 u'\f': [('S', u'\f', 1, 1)], 161 u'\r': [('S', u'\r', 1, 1)], 162 u'\t': [('S', u'\t', 1, 1)], 163 u'\r\n\r\n\f\t ': [('S', u'\r\n\r\n\f\t ', 1, 1)], 164 165 # COMMENT, for incomplete see later 166 u'/*x*/ ': [('COMMENT', u'/*x*/', 1, 1), 167 ('S', u' ', 1, 6)], 168 169 # FUNCTION 170 u' x( ': [('S', u' ', 1, 1), 171 ('FUNCTION', u'x(', 1, 2), 172 ('S', u' ', 1, 4)], 173 # only url( is a valid URI so this must be a function 174 u'URL(': [('FUNCTION', u'URL(', 1, 1)], 175 u'uRl(': [('FUNCTION', u'uRl(', 1, 1)], 176 177 # INCLUDES 178 u' ~= ': [('S', u' ', 1, 1), 179 ('INCLUDES', u'~=', 1, 2), 180 ('S', u' ', 1, 4)], 181 u'~==': [('INCLUDES', u'~=', 1, 1), ('CHAR', u'=', 1, 3)], 182 183 # DASHMATCH 184 u' |= ': [('S', u' ', 1, 1), 185 ('DASHMATCH', u'|=', 1, 2), 186 ('S', u' ', 1, 4)], 187 u'|==': [('DASHMATCH', u'|=', 1, 1), ('CHAR', u'=', 1, 3)], 188 189 # CHAR 190 u' @ ': [('S', u' ', 1, 1), 191 ('CHAR', u'@', 1, 2), 192 ('S', u' ', 1, 3)], 193 194 # --- overwritten for CSS 2.1 --- 195 # LBRACE 196 u' { ': [('S', u' ', 1, 1), 197 ('CHAR', u'{', 1, 2), 198 ('S', u' ', 1, 3)], 199 # PLUS 200 u' + ': [('S', u' ', 1, 1), 201 ('CHAR', u'+', 1, 2), 202 ('S', u' ', 1, 3)], 203 # GREATER 204 u' > ': [('S', u' ', 1, 1), 205 ('CHAR', u'>', 1, 2), 206 ('S', u' ', 1, 3)], 207 # COMMA 208 u' , ': [('S', u' ', 1, 1), 209 ('CHAR', u',', 1, 2), 210 ('S', u' ', 1, 3)], 211 212 # class 213 u' . ': [('S', u' ', 1, 1), 214 ('CHAR', u'.', 1, 2), 215 ('S', u' ', 1, 3)], 216 217 } 218 219 tests3 = { 220 # specials 221 u'c\\olor': [('IDENT', u'c\\olor', 1, 1)], 222 u'-1': [('CHAR', u'-', 1, 1), ('NUMBER', u'1', 1, 2)], 223 u'-1px': [('CHAR', u'-', 1, 1), ('DIMENSION', u'1px', 1, 2)], 224 225 # ATKEYWORD 226 u' @x ': [('S', u' ', 1, 1), 227 ('ATKEYWORD', u'@x', 1, 2), 228 ('S', u' ', 1, 4)], 229 u'@X': [('ATKEYWORD', u'@X', 1, 1)], 230 u'@\\x': [('ATKEYWORD', u'@\\x', 1, 1)], 231 # - 232 u'@1x': [('CHAR', u'@', 1, 1), 233 ('DIMENSION', u'1x', 1, 2)], 234 235 # DIMENSION 236 u' 0px ': [('S', u' ', 1, 1), 237 ('DIMENSION', u'0px', 1, 2), 238 ('S', u' ', 1, 5)], 239 u' 1s ': [('S', u' ', 1, 1), 240 ('DIMENSION', u'1s', 1, 2), 241 ('S', u' ', 1, 4)], 242 u'0.2EM': [('DIMENSION', u'0.2EM', 1, 1)], 243 u'1p\\x': [('DIMENSION', u'1p\\x', 1, 1)], 244 u'1PX': [('DIMENSION', u'1PX', 1, 1)], 245 246 # NUMBER 247 u' - 0 ': [('S', u' ', 1, 1), 248 ('CHAR', u'-', 1, 2), 249 ('S', u' ', 1, 3), 250 ('NUMBER', u'0', 1, 4), 251 ('S', u' ', 1, 5)], 252 u' + 0 ': [('S', u' ', 1, 1), 253 ('CHAR', u'+', 1, 2), 254 ('S', u' ', 1, 3), 255 ('NUMBER', u'0', 1, 4), 256 ('S', u' ', 1, 5)], 257 258 # PREFIXMATCH 259 u' ^= ': [('S', u' ', 1, 1), 260 ('PREFIXMATCH', u'^=', 1, 2), 261 ('S', u' ', 1, 4)], 262 u'^==': [('PREFIXMATCH', u'^=', 1, 1), ('CHAR', u'=', 1, 3)], 263 264 # SUFFIXMATCH 265 u' $= ': [('S', u' ', 1, 1), 266 ('SUFFIXMATCH', u'$=', 1, 2), 267 ('S', u' ', 1, 4)], 268 u'$==': [('SUFFIXMATCH', u'$=', 1, 1), ('CHAR', u'=', 1, 3)], 269 270 # SUBSTRINGMATCH 271 u' *= ': [('S', u' ', 1, 1), 272 ('SUBSTRINGMATCH', u'*=', 1, 2), 273 ('S', u' ', 1, 4)], 274 u'*==': [('SUBSTRINGMATCH', u'*=', 1, 1), ('CHAR', u'=', 1, 3)], 275 276 # BOM 277 u' \xFEFF ': [('S', u' ', 1, 1), 278 ('BOM', u'\xFEFF', 1, 2), # len=3 279 ('S', u' ', 1, 5)], 280 281 } 282 283 tests2 = { 284 # escapes work not for a-f! 285 # IMPORT_SYM 286 u' @import ': [('S', u' ', 1, 1), 287 ('IMPORT_SYM', u'@import', 1, 2), 288 ('S', u' ', 1, 9)], 289 u'@IMPORT': [('IMPORT_SYM', u'@IMPORT', 1, 1)], 290 ur'@\i\m\p\o\r\t': [('IMPORT_SYM', ur'@\i\m\p\o\r\t', 1, 1)], 291 ur'@\I\M\P\O\R\T': [('IMPORT_SYM', ur'@\I\M\P\O\R\T', 1, 1)], 292 ur'@\49 \04d\0050\0004f\000052\54': [('IMPORT_SYM', 293 ur'@\49 \04d\0050\0004f\000052\54', 294 1, 1)], 295 ur'@\69 \06d\0070\0006f\000072\74': [('IMPORT_SYM', 296 ur'@\69 \06d\0070\0006f\000072\74', 297 1, 1)], 298 299 # PAGE_SYM 300 u' @page ': [('S', u' ', 1, 1), 301 ('PAGE_SYM', u'@page', 1, 2), 302 ('S', u' ', 1, 7)], 303 u'@PAGE': [('PAGE_SYM', u'@PAGE', 1, 1)], 304 ur'@\pa\ge': [('PAGE_SYM', ur'@\pa\ge', 1, 1)], 305 ur'@\PA\GE': [('PAGE_SYM', ur'@\PA\GE', 1, 1)], 306 ur'@\50\41\47\45': [('PAGE_SYM', ur'@\50\41\47\45', 1, 1)], 307 ur'@\70\61\67\65': [('PAGE_SYM', ur'@\70\61\67\65', 1, 1)], 308 309 # MEDIA_SYM 310 u' @media ': [('S', u' ', 1, 1), 311 ('MEDIA_SYM', u'@media', 1, 2), 312 ('S', u' ', 1, 8)], 313 u'@MEDIA': [('MEDIA_SYM', u'@MEDIA', 1, 1)], 314 ur'@\med\ia': [('MEDIA_SYM', ur'@\med\ia', 1, 1)], 315 ur'@\MED\IA': [('MEDIA_SYM', ur'@\MED\IA', 1, 1)], 316 u'@\\4d\n\\45\r\\44\t\\49\r\n\\41\f': [('MEDIA_SYM', 317 u'@\\4d\n\\45\r\\44\t\\49\r\n\\41\f', 318 1, 1)], 319 u'@\\6d\n\\65\r\\64\t\\69\r\n\\61\f': [('MEDIA_SYM', 320 u'@\\6d\n\\65\r\\64\t\\69\r\n\\61\f', 321 1, 1)], 322 323 # FONT_FACE_SYM 324 u' @font-face ': [('S', u' ', 1, 1), 325 ('FONT_FACE_SYM', u'@font-face', 1, 2), 326 ('S', u' ', 1, 12)], 327 u'@FONT-FACE': [('FONT_FACE_SYM', u'@FONT-FACE', 1, 1)], 328 ur'@f\o\n\t\-face': [('FONT_FACE_SYM', ur'@f\o\n\t\-face', 1, 1)], 329 ur'@F\O\N\T\-FACE': [('FONT_FACE_SYM', ur'@F\O\N\T\-FACE', 1, 1)], 330 # TODO: "-" as hex! 331 ur'@\46\4f\4e\54\-\46\41\43\45': [('FONT_FACE_SYM', 332 ur'@\46\4f\4e\54\-\46\41\43\45', 1, 1)], 333 ur'@\66\6f\6e\74\-\66\61\63\65': [('FONT_FACE_SYM', 334 ur'@\66\6f\6e\74\-\66\61\63\65', 1, 1)], 335 336 # CHARSET_SYM only if "@charset "! 337 u' @charset ': [('S', u' ', 1, 1), 338 ('CHARSET_SYM', u'@charset ', 1, 2), 339 ('S', u' ', 1, 11)], 340 u'@charset': [('ATKEYWORD', u'@charset', 1, 1)], # no ending S 341 u'@CHARSET ': [('ATKEYWORD', u'@CHARSET', 1, 1),# uppercase 342 ('S', u' ', 1, 9)], 343 u'@cha\\rset ': [('ATKEYWORD', u'@cha\\rset', 1, 1), # not literal 344 ('S', u' ', 1, 10)], 345 346 # NAMESPACE_SYM 347 u' @namespace ': [('S', u' ', 1, 1), 348 ('NAMESPACE_SYM', u'@namespace', 1, 2), 349 ('S', u' ', 1, 12)], 350 ur'@NAMESPACE': [('NAMESPACE_SYM', ur'@NAMESPACE', 1, 1)], 351 ur'@\na\me\s\pace': [('NAMESPACE_SYM', ur'@\na\me\s\pace', 1, 1)], 352 ur'@\NA\ME\S\PACE': [('NAMESPACE_SYM', ur'@\NA\ME\S\PACE', 1, 1)], 353 ur'@\4e\41\4d\45\53\50\41\43\45': [('NAMESPACE_SYM', 354 ur'@\4e\41\4d\45\53\50\41\43\45', 1, 1)], 355 ur'@\6e\61\6d\65\73\70\61\63\65': [('NAMESPACE_SYM', 356 ur'@\6e\61\6d\65\73\70\61\63\65', 1, 1)], 357 358 # ATKEYWORD 359 u' @unknown ': [('S', u' ', 1, 1), 360 ('ATKEYWORD', u'@unknown', 1, 2), 361 ('S', u' ', 1, 10)], 362 363 # STRING 364 # strings with linebreak in it 365 u' "\\na"\na': [('S', u' ', 1, 1), 366 ('STRING', u'"\\na"', 1, 2), 367 ('S', u'\n', 1, 7), 368 ('IDENT', u'a', 2, 1)], 369 u" '\\na'\na": [('S', u' ', 1, 1), 370 ('STRING', u"'\\na'", 1, 2), 371 ('S', u'\n', 1, 7), 372 ('IDENT', u'a', 2, 1)], 373 u' "\\r\\n\\t\\n\\ra"a': [('S', u' ', 1, 1), 374 ('STRING', u'"\\r\\n\\t\\n\\ra"', 1, 2), 375 ('IDENT', u'a', 1, 15)], 376 377 # IMPORTANT_SYM is not IDENT!!! 378 u' !important ': [('S', u' ', 1, 1), 379 ('CHAR', u'!', 1, 2), 380 ('IDENT', u'important', 1, 3), 381 ('S', u' ', 1, 12)], 382 u'! /*1*/ important ': [ 383 ('CHAR', u'!', 1, 1), 384 ('S', u' ', 1, 2), 385 ('COMMENT', u'/*1*/', 1, 3), 386 ('S', u' ', 1, 8), 387 ('IDENT', u'important', 1, 9), 388 ('S', u' ', 1, 18)], 389 u'! important': [('CHAR', u'!', 1, 1), 390 ('S', u' ', 1, 2), 391 ('IDENT', u'important', 1, 3)], 392 u'!\n\timportant': [('CHAR', u'!', 1, 1), 393 ('S', u'\n\t', 1, 2), 394 ('IDENT', u'important', 2, 2)], 395 u'!IMPORTANT': [('CHAR', u'!', 1, 1), 396 ('IDENT', u'IMPORTANT', 1, 2)], 397 ur'!\i\m\p\o\r\ta\n\t': [('CHAR', u'!', 1, 1), 398 ('IDENT', 399 ur'\i\m\p\o\r\ta\n\t', 1, 2)], 400 ur'!\I\M\P\O\R\Ta\N\T': [('CHAR', u'!', 1, 1), 401 ('IDENT', 402 ur'\I\M\P\O\R\Ta\N\T', 1, 2)], 403 ur'!\49\4d\50\4f\52\54\41\4e\54': [('CHAR', u'!', 1, 1), 404 ('IDENT', 405 ur'IMPORTANT', 406 1, 2)], 407 ur'!\69\6d\70\6f\72\74\61\6e\74': [('CHAR', u'!', 1, 1), 408 ('IDENT', 409 ur'important', 410 1, 2)], 411 } 412 413 # overwriting tests in testsall 414 tests2only = { 415 # LBRACE 416 u' { ': [('S', u' ', 1, 1), 417 ('LBRACE', u'{', 1, 2), 418 ('S', u' ', 1, 3)], 419 # PLUS 420 u' + ': [('S', u' ', 1, 1), 421 ('PLUS', u'+', 1, 2), 422 ('S', u' ', 1, 3)], 423 # GREATER 424 u' > ': [('S', u' ', 1, 1), 425 ('GREATER', u'>', 1, 2), 426 ('S', u' ', 1, 3)], 427 # COMMA 428 u' , ': [('S', u' ', 1, 1), 429 ('COMMA', u',', 1, 2), 430 ('S', u' ', 1, 3)], 431 # class 432 u' . ': [('S', u' ', 1, 1), 433 ('CLASS', u'.', 1, 2), 434 ('S', u' ', 1, 3)], 435 } 436 437 testsfullsheet = { 438 # TODO: escape ends with explicit space but \r\n as single space 439 #u'\\1\r\nb': [('IDENT', u'\\1\r', 1, 1), ('IDENT', u'b', 1, 4)], 440 441 # STRING 442 ur'"\" "': [('STRING', ur'"\" "', 1, 1)], 443 u"""'\\''""": [('STRING', u"""'\\''""", 1, 1)], 444 u'''"\\""''': [('STRING', u'''"\\""''', 1, 1)], 445 u' "\na': [('S', u' ', 1, 1), 446 ('INVALID', u'"', 1, 2), 447 ('S', u'\n', 1, 3), 448 ('IDENT', u'a', 2, 1)], 449 450 # strings with linebreak in it 451 u' "\\na\na': [('S', u' ', 1, 1), 452 ('INVALID', u'"\\na', 1, 2), 453 ('S', u'\n', 1, 6), 454 ('IDENT', u'a', 2, 1)], 455 u' "\\r\\n\\t\\n\\ra\na': [('S', u' ', 1, 1), 456 ('INVALID', u'"\\r\\n\\t\\n\\ra', 1, 2), 457 ('S', u'\n', 1, 14), 458 ('IDENT', u'a', 2, 1)], 459 } 460 461 # tests if fullsheet=False is set on tokenizer 462 testsfullsheetfalse = { 463 # COMMENT incomplete 464 u'/*': [('CHAR', u'/', 1, 1), 465 ('CHAR', u'*', 1, 2)], 466 467 # INVALID incomplete 468 u' " ': [('S', u' ', 1, 1), 469 ('INVALID', u'" ', 1, 2)], 470 u" 'abc\"with quote\" in it": [('S', u' ', 1, 1), 471 ('INVALID', u"'abc\"with quote\" in it", 1, 2)], 472 473 # URI incomplete 474 u'url(a': [('FUNCTION', u'url(', 1, 1), 475 ('IDENT', u'a', 1, 5)], 476 u'url("a': [('FUNCTION', u'url(', 1, 1), 477 ('INVALID', u'"a', 1, 5)], 478 u"url('a": [('FUNCTION', u'url(', 1, 1), 479 ('INVALID', u"'a", 1, 5)], 480 } 481 482 # tests if fullsheet=True is set on tokenizer 483 testsfullsheettrue = { 484 # COMMENT incomplete 485 u'/*': [('COMMENT', u'/**/', 1, 1)], 486 487 # INVALID incomplete => STRING 488 u' " ': [('S', u' ', 1, 1), 489 ('STRING', u'" "', 1, 2)], 490 u" 'abc\"with quote\" in it": [('S', u' ', 1, 1), 491 ('STRING', u"'abc\"with quote\" in it'", 1, 2)], 492 493 # URI incomplete FUNC => URI 494 u'url(a': [('URI', u'url(a)', 1, 1)], 495 u'url( a': [('URI', u'url( a)', 1, 1)], 496 u'url("a': [('URI', u'url("a")', 1, 1)], 497 u'url( "a ': [('URI', u'url( "a ")', 1, 1)], 498 u"url('a": [('URI', u"url('a')", 1, 1)], 499 u'url("a"': [('URI', u'url("a")', 1, 1)], 500 u"url('a'": [('URI', u"url('a')", 1, 1)], 501 502 } 503
504 - def setUp(self):
505 #log = cssutils.errorhandler.ErrorHandler() 506 self.tokenizer = Tokenizer()
507
508 - def test_linenumbers(self):
509 "Tokenizer line + col" 510 pass
511
512 - def test_tokenize(self):
513 "cssutils Tokenizer().tokenize()" 514 import cssutils.cssproductions 515 tokenizer = Tokenizer(cssutils.cssproductions.MACROS, 516 cssutils.cssproductions.PRODUCTIONS) 517 tests = {} 518 tests.update(self.testsall) 519 tests.update(self.tests2) 520 tests.update(self.tests3) 521 tests.update(self.testsfullsheet) 522 tests.update(self.testsfullsheetfalse) 523 for css in tests: 524 # check token format 525 tokens = tokenizer.tokenize(css) 526 for i, actual in enumerate(tokens): 527 expected = tests[css][i] 528 self.assertEqual(expected, actual) 529 530 # check if all same number of tokens 531 tokens = [t for t in tokenizer.tokenize(css)] 532 self.assertEqual(len(tokens), len(tests[css]))
533
534 - def test_tokenizefullsheet(self):
535 "cssutils Tokenizer().tokenize(fullsheet=True)" 536 import cssutils.cssproductions 537 tokenizer = Tokenizer(cssutils.cssproductions.MACROS, 538 cssutils.cssproductions.PRODUCTIONS) 539 tests = {} 540 tests.update(self.testsall) 541 tests.update(self.tests2) 542 tests.update(self.tests3) 543 tests.update(self.testsfullsheet) 544 tests.update(self.testsfullsheettrue) 545 for css in tests: 546 # check token format 547 tokens = tokenizer.tokenize(css, fullsheet=True) 548 for i, actual in enumerate(tokens): 549 try: 550 expected = tests[css][i] 551 except IndexError: 552 # EOF is added 553 self.assertEqual(actual[0], 'EOF') 554 else: 555 self.assertEqual(expected, actual) 556 557 # check if all same number of tokens 558 tokens = [t for t in tokenizer.tokenize(css, fullsheet=True)] 559 # EOF is added so -1 560 self.assertEqual(len(tokens) - 1, len(tests[css]))
561 562 563 # not really needed
564 - def test_tokenizeCSS3(self):
565 "CSS3 Tokenizer().tokenize()" 566 import cssutils.css3productions 567 tokenizer = Tokenizer(cssutils.css3productions.MACROS, 568 cssutils.css3productions.PRODUCTIONS) 569 tests = {} 570 tests.update(self.testsall) 571 tests.update(self.tests3) 572 for css in tests: 573 tokens = tokenizer.tokenize(css) 574 for i, actual in enumerate(tokens): 575 expected = tests[css][i] 576 self.assertEqual(expected, actual)
577 578 # not really needed
579 - def test_tokenizeCSS2_1(self):
580 "CSS2 Tokenizer().tokenize()" 581 import cssutils.css2productions 582 tokenizer = Tokenizer(cssutils.css2productions.MACROS, 583 cssutils.css2productions.PRODUCTIONS) 584 tests = {} 585 tests.update(self.testsall) 586 #tests.update(self.tests2) 587 tests.update(self.tests2only) 588 for css in tests: 589 tokens = tokenizer.tokenize(css) 590 for i, actual in enumerate(tokens): 591 expected = tests[css][i] 592 self.assertEqual(expected, actual)
593 594 # -------------- 595
596 - def __old(self):
597 598 testsOLD = { 599 u'x x1 -x .-x #_x -': [(1, 1, tt.IDENT, u'x'), 600 (1, 2, 'S', u' '), 601 (1, 3, tt.IDENT, u'x1'), 602 (1, 5, 'S', u' '), 603 (1, 6, tt.IDENT, u'-x'), 604 (1, 8, 'S', u' '), 605 (1, 9, tt.CLASS, u'.'), 606 (1, 10, tt.IDENT, u'-x'), 607 (1, 12, 'S', u' '), 608 (1, 13, tt.HASH, u'#_x'), 609 (1, 16, 'S', u' '), 610 (1, 17, 'DELIM', u'-')], 611 612 # num 613 u'1 1.1 -1 -1.1 .1 -.1 1.': [(1, 1, tt.NUMBER, u'1'), 614 (1, 2, 'S', u' '), (1, 3, tt.NUMBER, u'1.1'), 615 (1, 6, 'S', u' '), (1, 7, tt.NUMBER, u'-1'), 616 (1, 9, 'S', u' '), (1, 10, tt.NUMBER, u'-1.1'), 617 (1, 14, 'S', u' '), (1, 15, tt.NUMBER, u'0.1'), 618 (1, 17, 'S', u' '), (1, 18, tt.NUMBER, u'-0.1'), 619 (1, 21, 'S', u' '), 620 (1, 22, tt.NUMBER, u'1'), (1, 23, tt.CLASS, u'.') 621 ], 622 # CSS3 pseudo 623 u'::': [(1, 1, tt.PSEUDO_ELEMENT, u'::')], 624 625 # SPECIALS 626 u'*+>~{},': [(1, 1, tt.UNIVERSAL, u'*'), 627 (1, 2, tt.PLUS, u'+'), 628 (1, 3, tt.GREATER, u'>'), 629 (1, 4, tt.TILDE, u'~'), 630 (1, 5, tt.LBRACE, u'{'), 631 (1, 6, tt.RBRACE, u'}'), 632 (1, 7, tt.COMMA, u',')], 633 634 # DELIM 635 u'!%:&$|': [(1, 1, 'DELIM', u'!'), 636 (1, 2, 'DELIM', u'%'), 637 (1, 3, 'DELIM', u':'), 638 (1, 4, 'DELIM', u'&'), 639 (1, 5, 'DELIM', u'$'), 640 (1, 6, 'DELIM', u'|')], 641 642 643 # DIMENSION 644 u'5em': [(1, 1, tt.DIMENSION, u'5em')], 645 u' 5em': [(1, 1, 'S', u' '), (1, 2, tt.DIMENSION, u'5em')], 646 u'5em ': [(1, 1, tt.DIMENSION, u'5em'), (1, 4, 'S', u' ')], 647 648 u'-5em': [(1, 1, tt.DIMENSION, u'-5em')], 649 u' -5em': [(1, 1, 'S', u' '), (1, 2, tt.DIMENSION, u'-5em')], 650 u'-5em ': [(1, 1, tt.DIMENSION, u'-5em'), (1, 5, 'S', u' ')], 651 652 u'.5em': [(1, 1, tt.DIMENSION, u'0.5em')], 653 u' .5em': [(1, 1, 'S', u' '), (1, 2, tt.DIMENSION, u'0.5em')], 654 u'.5em ': [(1, 1, tt.DIMENSION, u'0.5em'), (1, 5, 'S', u' ')], 655 656 u'-.5em': [(1, 1, tt.DIMENSION, u'-0.5em')], 657 u' -.5em': [(1, 1, 'S', u' '), (1, 2, tt.DIMENSION, u'-0.5em')], 658 u'-.5em ': [(1, 1, tt.DIMENSION, u'-0.5em'), (1, 6, 'S', u' ')], 659 660 u'5em5_-': [(1, 1, tt.DIMENSION, u'5em5_-')], 661 662 u'a a5 a5a 5 5a 5a5': [(1, 1, tt.IDENT, u'a'), 663 (1, 2, 'S', u' '), 664 (1, 3, tt.IDENT, u'a5'), 665 (1, 5, 'S', u' '), 666 (1, 6, tt.IDENT, u'a5a'), 667 (1, 9, 'S', u' '), 668 (1, 10, tt.NUMBER, u'5'), 669 (1, 11, 'S', u' '), 670 (1, 12, tt.DIMENSION, u'5a'), 671 (1, 14, 'S', u' '), 672 (1, 15, tt.DIMENSION, u'5a5')], 673 674 # URI 675 u'url()': [(1, 1, tt.URI, u'url()')], 676 u'url();': [(1, 1, tt.URI, u'url()'), (1, 6, tt.SEMICOLON, ';')], 677 u'url("x")': [(1, 1, tt.URI, u'url("x")')], 678 u'url( "x")': [(1, 1, tt.URI, u'url("x")')], 679 u'url("x" )': [(1, 1, tt.URI, u'url("x")')], 680 u'url( "x" )': [(1, 1, tt.URI, u'url("x")')], 681 u' url("x")': [ 682 (1, 1, 'S', u' '), 683 (1, 2, tt.URI, u'url("x")')], 684 u'url("x") ': [ 685 (1, 1, tt.URI, u'url("x")'), 686 (1, 9, 'S', u' '), 687 ], 688 u'url(ab)': [(1, 1, tt.URI, u'url(ab)')], 689 u'url($#/ab)': [(1, 1, tt.URI, u'url($#/ab)')], 690 u'url(\1233/a/b)': [(1, 1, tt.URI, u'url(\1233/a/b)')], 691 # not URI 692 u'url("1""2")': [ 693 (1, 1, tt.FUNCTION, u'url('), 694 (1, 5, tt.STRING, u'"1"'), 695 (1, 8, tt.STRING, u'"2"'), 696 (1, 11, tt.RPARANTHESIS, u')'), 697 ], 698 u'url(a"2")': [ 699 (1, 1, tt.FUNCTION, u'url('), 700 (1, 5, tt.IDENT, u'a'), 701 (1, 6, tt.STRING, u'"2"'), 702 (1, 9, tt.RPARANTHESIS, u')'), 703 ], 704 u'url(a b)': [ 705 (1, 1, tt.FUNCTION, u'url('), 706 (1, 5, tt.IDENT, u'a'), 707 (1, 6, 'S', u' '), 708 (1, 7, tt.IDENT, u'b'), 709 (1, 8, tt.RPARANTHESIS, u')'), 710 ], 711 712 # FUNCTION 713 u' counter("x")': [ 714 (1,1, 'S', u' '), 715 (1, 2, tt.FUNCTION, u'counter('), 716 (1, 10, tt.STRING, u'"x"'), 717 (1, 13, tt.RPARANTHESIS, u')')], 718 # HASH 719 u'# #a #_a #-a #1': [ 720 (1, 1, 'DELIM', u'#'), 721 (1, 2, 'S', u' '), 722 (1, 3, tt.HASH, u'#a'), 723 (1, 5, 'S', u' '), 724 (1, 6, tt.HASH, u'#_a'), 725 (1, 9, 'S', u' '), 726 (1, 10, tt.HASH, u'#-a'), 727 (1, 13, 'S', u' '), 728 (1, 14, tt.HASH, u'#1') 729 ], 730 u'#1a1 ': [ 731 (1, 1, tt.HASH, u'#1a1'), 732 (1, 5, 'S', u' '), 733 ], 734 u'#1a1\n': [ 735 (1, 1, tt.HASH, u'#1a1'), 736 (1, 5, 'S', u'\n'), 737 ], 738 u'#1a1{': [ 739 (1, 1, tt.HASH, u'#1a1'), 740 (1, 5, tt.LBRACE, u'{'), 741 ], 742 u'#1a1 {': [ 743 (1, 1, tt.HASH, u'#1a1'), 744 (1, 5, 'S', u' '), 745 (1, 6, tt.LBRACE, u'{'), 746 ], 747 u'#1a1\n{': [ 748 (1, 1, tt.HASH, u'#1a1'), 749 (1, 5, 'S', u'\n'), 750 (2, 1, tt.LBRACE, u'{'), 751 ], 752 u'#1a1\n {': [ 753 (1, 1, tt.HASH, u'#1a1'), 754 (1, 5, 'S', u'\n '), 755 (2, 2, tt.LBRACE, u'{'), 756 ], 757 u'#1a1 \n{': [ 758 (1, 1, tt.HASH, u'#1a1'), 759 (1, 5, 'S', u' \n'), 760 (2, 1, tt.LBRACE, u'{'), 761 ], 762 # STRINGS with NL 763 u'"x\n': [(1,1, tt.INVALID, u'"x\n')], 764 u'"x\r': [(1,1, tt.INVALID, u'"x\r')], 765 u'"x\f': [(1,1, tt.INVALID, u'"x\f')], 766 u'"x\n ': [ 767 (1,1, tt.INVALID, u'"x\n'), 768 (2,1, 'S', u' ') 769 ] 770 771 }
772 773 # tests = { 774 # u'/*a': xml.dom.SyntaxErr, 775 # u'"a': xml.dom.SyntaxErr, 776 # u"'a": xml.dom.SyntaxErr, 777 # u"\\0 a": xml.dom.SyntaxErr, 778 # u"\\00": xml.dom.SyntaxErr, 779 # u"\\000": xml.dom.SyntaxErr, 780 # u"\\0000": xml.dom.SyntaxErr, 781 # u"\\00000": xml.dom.SyntaxErr, 782 # u"\\000000": xml.dom.SyntaxErr, 783 # u"\\0000001": xml.dom.SyntaxErr 784 # } 785 # self.tokenizer.log.raiseExceptions = True #!! 786 # for css, exception in tests.items(): 787 # self.assertRaises(exception, self.tokenizer.tokenize, css) 788 789 790 if __name__ == '__main__': 791 import unittest 792 unittest.main() 793