1 /** 2 * Defines lexical tokens. 3 * 4 * Specification: $(LINK2 https://dlang.org/spec/lex.html#tokens, Tokens) 5 * 6 * Copyright: Copyright (C) 1999-2023 by The D Language Foundation, All Rights Reserved 7 * Authors: $(LINK2 https://www.digitalmars.com, Walter Bright) 8 * License: $(LINK2 https://www.boost.org/LICENSE_1_0.txt, Boost License 1.0) 9 * Source: $(LINK2 https://github.com/dlang/dmd/blob/master/src/dmd/tokens.d, _tokens.d) 10 * Documentation: https://dlang.org/phobos/dmd_tokens.html 11 * Coverage: https://codecov.io/gh/dlang/dmd/src/master/src/dmd/tokens.d 12 */ 13 14 module dmd.tokens; 15 16 import core.stdc.ctype; 17 import core.stdc.stdio; 18 import core.stdc.string; 19 import dmd.identifier; 20 import dmd.location; 21 import dmd.root.ctfloat; 22 import dmd.common.outbuffer; 23 import dmd.root.rmem; 24 import dmd.root.utf; 25 26 enum TOK : ubyte 27 { 28 reserved, 29 30 // Other 31 leftParenthesis, 32 rightParenthesis, 33 leftBracket, 34 rightBracket, 35 leftCurly, 36 rightCurly, 37 colon, 38 semicolon, 39 dotDotDot, 40 endOfFile, 41 cast_, 42 null_, 43 assert_, 44 true_, 45 false_, 46 throw_, 47 new_, 48 delete_, 49 variable, 50 slice, 51 version_, 52 module_, 53 dollar, 54 template_, 55 typeof_, 56 pragma_, 57 typeid_, 58 comment, 59 60 // Operators 61 lessThan, 62 greaterThan, 63 lessOrEqual, 64 greaterOrEqual, 65 equal, 66 notEqual, 67 identity, 68 notIdentity, 69 is_, 70 71 leftShift, 72 rightShift, 73 leftShiftAssign, 74 rightShiftAssign, 75 unsignedRightShift, 76 unsignedRightShiftAssign, 77 concatenateAssign, // ~= 78 add, 79 min, 80 addAssign, 81 minAssign, 82 mul, 83 div, 84 mod, 85 mulAssign, 86 divAssign, 87 modAssign, 88 and, 89 or, 90 xor, 91 andAssign, 92 orAssign, 93 xorAssign, 94 assign, 95 not, 96 tilde, 97 plusPlus, 98 minusMinus, 99 dot, 100 comma, 101 question, 102 andAnd, 103 orOr, 104 105 // Numeric literals 106 int32Literal, 107 uns32Literal, 108 int64Literal, 109 uns64Literal, 110 int128Literal, 111 uns128Literal, 112 float32Literal, 113 float64Literal, 114 float80Literal, 115 imaginary32Literal, 116 imaginary64Literal, 117 imaginary80Literal, 118 119 // Char constants 120 charLiteral, 121 wcharLiteral, 122 dcharLiteral, 123 124 // Leaf operators 125 identifier, 126 string_, 127 hexadecimalString, 128 this_, 129 super_, 130 error, 131 132 // Basic types 133 void_, 134 int8, 135 uns8, 136 int16, 137 uns16, 138 int32, 139 uns32, 140 int64, 141 uns64, 142 int128, 143 uns128, 144 float32, 145 float64, 146 float80, 147 imaginary32, 148 imaginary64, 149 imaginary80, 150 complex32, 151 complex64, 152 complex80, 153 char_, 154 wchar_, 155 dchar_, 156 bool_, 157 158 // Aggregates 159 struct_, 160 class_, 161 interface_, 162 union_, 163 enum_, 164 import_, 165 alias_, 166 override_, 167 delegate_, 168 function_, 169 mixin_, 170 align_, 171 extern_, 172 private_, 173 protected_, 174 public_, 175 export_, 176 static_, 177 final_, 178 const_, 179 abstract_, 180 debug_, 181 deprecated_, 182 in_, 183 out_, 184 inout_, 185 lazy_, 186 auto_, 187 package_, 188 immutable_, 189 190 // Statements 191 if_, 192 else_, 193 while_, 194 for_, 195 do_, 196 switch_, 197 case_, 198 default_, 199 break_, 200 continue_, 201 with_, 202 synchronized_, 203 return_, 204 goto_, 205 try_, 206 catch_, 207 finally_, 208 asm_, 209 foreach_, 210 foreach_reverse_, 211 scope_, 212 onScopeExit, 213 onScopeFailure, 214 onScopeSuccess, 215 216 // Contracts 217 invariant_, 218 219 // Testing 220 unittest_, 221 222 // Added after 1.0 223 argumentTypes, 224 ref_, 225 macro_, 226 227 parameters, 228 traits, 229 pure_, 230 nothrow_, 231 gshared, 232 line, 233 file, 234 fileFullPath, 235 moduleString, // __MODULE__ 236 functionString, // __FUNCTION__ 237 prettyFunction, // __PRETTY_FUNCTION__ 238 shared_, 239 at, 240 pow, 241 powAssign, 242 goesTo, 243 vector, 244 pound, 245 246 arrow, // -> 247 colonColon, // :: 248 wchar_tLiteral, 249 endOfLine, // \n, \r, \u2028, \u2029 250 whitespace, 251 252 // C only keywords 253 inline, 254 register, 255 restrict, 256 signed, 257 sizeof_, 258 typedef_, 259 unsigned, 260 volatile, 261 _Alignas, 262 _Alignof, 263 _Atomic, 264 _Bool, 265 _Complex, 266 _Generic, 267 _Imaginary, 268 _Noreturn, 269 _Static_assert, 270 _Thread_local, 271 272 // C only extended keywords 273 _assert, 274 _import, 275 __cdecl, 276 __declspec, 277 __stdcall, 278 __thread, 279 __pragma, 280 __int128, 281 __attribute__, 282 } 283 284 /// Expression nodes 285 enum EXP : ubyte 286 { 287 reserved, 288 289 // Other 290 negate, 291 cast_, 292 null_, 293 assert_, 294 array, 295 call, 296 address, 297 type, 298 throw_, 299 new_, 300 delete_, 301 star, 302 symbolOffset, 303 variable, 304 dotVariable, 305 dotIdentifier, 306 dotTemplateInstance, 307 dotType, 308 slice, 309 arrayLength, 310 dollar, 311 template_, 312 dotTemplateDeclaration, 313 declaration, 314 dSymbol, 315 typeid_, 316 uadd, 317 remove, 318 newAnonymousClass, 319 arrayLiteral, 320 assocArrayLiteral, 321 structLiteral, 322 classReference, 323 thrownException, 324 delegatePointer, 325 delegateFunctionPointer, 326 327 // Operators 328 lessThan, 329 greaterThan, 330 lessOrEqual, 331 greaterOrEqual, 332 equal, 333 notEqual, 334 identity, 335 notIdentity, 336 index, 337 is_, 338 339 leftShift, 340 rightShift, 341 leftShiftAssign, 342 rightShiftAssign, 343 unsignedRightShift, 344 unsignedRightShiftAssign, 345 concatenate, 346 concatenateAssign, // ~= 347 concatenateElemAssign, 348 concatenateDcharAssign, 349 add, 350 min, 351 addAssign, 352 minAssign, 353 mul, 354 div, 355 mod, 356 mulAssign, 357 divAssign, 358 modAssign, 359 and, 360 or, 361 xor, 362 andAssign, 363 orAssign, 364 xorAssign, 365 assign, 366 not, 367 tilde, 368 plusPlus, 369 minusMinus, 370 construct, 371 blit, 372 dot, 373 comma, 374 question, 375 andAnd, 376 orOr, 377 prePlusPlus, 378 preMinusMinus, 379 380 // Leaf operators 381 identifier, 382 string_, 383 this_, 384 super_, 385 halt, 386 tuple, 387 error, 388 389 // Basic types 390 void_, 391 int64, 392 float64, 393 complex80, 394 import_, 395 delegate_, 396 function_, 397 mixin_, 398 in_, 399 break_, 400 continue_, 401 goto_, 402 scope_, 403 404 traits, 405 overloadSet, 406 line, 407 file, 408 fileFullPath, 409 moduleString, // __MODULE__ 410 functionString, // __FUNCTION__ 411 prettyFunction, // __PRETTY_FUNCTION__ 412 pow, 413 powAssign, 414 vector, 415 416 voidExpression, 417 cantExpression, 418 showCtfeContext, 419 objcClassReference, 420 vectorArray, 421 compoundLiteral, // ( type-name ) { initializer-list } 422 _Generic, 423 interval, 424 425 loweredAssignExp, 426 } 427 428 enum FirstCKeyword = TOK.inline; 429 430 // Assert that all token enum members have consecutive values and 431 // that none of them overlap 432 static assert(() { 433 foreach (idx, enumName; __traits(allMembers, TOK)) { 434 static if (idx != __traits(getMember, TOK, enumName)) { 435 pragma(msg, "Error: Expected TOK.", enumName, " to be ", idx, " but is ", __traits(getMember, TOK, enumName)); 436 static assert(0); 437 } 438 } 439 return true; 440 }()); 441 442 /**************************************** 443 */ 444 445 private immutable TOK[] keywords = 446 [ 447 TOK.this_, 448 TOK.super_, 449 TOK.assert_, 450 TOK.null_, 451 TOK.true_, 452 TOK.false_, 453 TOK.cast_, 454 TOK.new_, 455 TOK.delete_, 456 TOK.throw_, 457 TOK.module_, 458 TOK.pragma_, 459 TOK.typeof_, 460 TOK.typeid_, 461 TOK.template_, 462 TOK.void_, 463 TOK.int8, 464 TOK.uns8, 465 TOK.int16, 466 TOK.uns16, 467 TOK.int32, 468 TOK.uns32, 469 TOK.int64, 470 TOK.uns64, 471 TOK.int128, 472 TOK.uns128, 473 TOK.float32, 474 TOK.float64, 475 TOK.float80, 476 TOK.bool_, 477 TOK.char_, 478 TOK.wchar_, 479 TOK.dchar_, 480 TOK.imaginary32, 481 TOK.imaginary64, 482 TOK.imaginary80, 483 TOK.complex32, 484 TOK.complex64, 485 TOK.complex80, 486 TOK.delegate_, 487 TOK.function_, 488 TOK.is_, 489 TOK.if_, 490 TOK.else_, 491 TOK.while_, 492 TOK.for_, 493 TOK.do_, 494 TOK.switch_, 495 TOK.case_, 496 TOK.default_, 497 TOK.break_, 498 TOK.continue_, 499 TOK.synchronized_, 500 TOK.return_, 501 TOK.goto_, 502 TOK.try_, 503 TOK.catch_, 504 TOK.finally_, 505 TOK.with_, 506 TOK.asm_, 507 TOK.foreach_, 508 TOK.foreach_reverse_, 509 TOK.scope_, 510 TOK.struct_, 511 TOK.class_, 512 TOK.interface_, 513 TOK.union_, 514 TOK.enum_, 515 TOK.import_, 516 TOK.mixin_, 517 TOK.static_, 518 TOK.final_, 519 TOK.const_, 520 TOK.alias_, 521 TOK.override_, 522 TOK.abstract_, 523 TOK.debug_, 524 TOK.deprecated_, 525 TOK.in_, 526 TOK.out_, 527 TOK.inout_, 528 TOK.lazy_, 529 TOK.auto_, 530 TOK.align_, 531 TOK.extern_, 532 TOK.private_, 533 TOK.package_, 534 TOK.protected_, 535 TOK.public_, 536 TOK.export_, 537 TOK.invariant_, 538 TOK.unittest_, 539 TOK.version_, 540 TOK.argumentTypes, 541 TOK.parameters, 542 TOK.ref_, 543 TOK.macro_, 544 TOK.pure_, 545 TOK.nothrow_, 546 TOK.gshared, 547 TOK.traits, 548 TOK.vector, 549 TOK.file, 550 TOK.fileFullPath, 551 TOK.line, 552 TOK.moduleString, 553 TOK.functionString, 554 TOK.prettyFunction, 555 TOK.shared_, 556 TOK.immutable_, 557 558 // C only keywords 559 TOK.inline, 560 TOK.register, 561 TOK.restrict, 562 TOK.signed, 563 TOK.sizeof_, 564 TOK.typedef_, 565 TOK.unsigned, 566 TOK..volatile, 567 TOK._Alignas, 568 TOK._Alignof, 569 TOK._Atomic, 570 TOK._Bool, 571 TOK._Complex, 572 TOK._Generic, 573 TOK._Imaginary, 574 TOK._Noreturn, 575 TOK._Static_assert, 576 TOK._Thread_local, 577 578 // C only extended keywords 579 TOK._assert, 580 TOK._import, 581 TOK.__cdecl, 582 TOK.__declspec, 583 TOK.__stdcall, 584 TOK.__thread, 585 TOK.__pragma, 586 TOK.__int128, 587 TOK.__attribute__, 588 ]; 589 590 // Initialize the identifier pool 591 shared static this() nothrow 592 { 593 Identifier.initTable(); 594 foreach (kw; keywords) 595 { 596 //printf("keyword[%d] = '%s'\n",kw, Token.tochars[kw].ptr); 597 Identifier.idPool(Token.tochars[kw], kw); 598 } 599 } 600 601 /************************************ 602 * This is used to pick the C keywords out of the tokens. 603 * If it's not a C keyword, then it's an identifier. 604 */ 605 static immutable TOK[TOK.max + 1] Ckeywords = 606 () { 607 with (TOK) 608 { 609 TOK[TOK.max + 1] tab = identifier; // default to identifier 610 enum Ckwds = [ auto_, break_, case_, char_, const_, continue_, default_, do_, float64, else_, 611 enum_, extern_, float32, for_, goto_, if_, inline, int32, int64, register, 612 restrict, return_, int16, signed, sizeof_, static_, struct_, switch_, typedef_, 613 union_, unsigned, void_, volatile, while_, asm_, typeof_, 614 _Alignas, _Alignof, _Atomic, _Bool, _Complex, _Generic, _Imaginary, _Noreturn, 615 _Static_assert, _Thread_local, 616 _import, __cdecl, __declspec, __stdcall, __thread, __pragma, __int128, __attribute__, 617 _assert ]; 618 619 foreach (kw; Ckwds) 620 tab[kw] = cast(TOK) kw; 621 622 return tab; 623 } 624 } (); 625 626 627 /*********************************************************** 628 */ 629 extern (C++) struct Token 630 { 631 Token* next; 632 Loc loc; 633 const(char)* ptr; // pointer to first character of this token within buffer 634 TOK value; 635 const(char)[] blockComment; // doc comment string prior to this token 636 const(char)[] lineComment; // doc comment for previous token 637 638 union 639 { 640 // Integers 641 long intvalue; 642 ulong unsvalue; 643 // Floats 644 real_t floatvalue; 645 646 struct 647 { 648 const(char)* ustring; // UTF8 string 649 uint len; 650 ubyte postfix; // 'c', 'w', 'd' 651 } 652 653 Identifier ident; 654 } 655 656 extern (D) private static immutable string[TOK.max + 1] tochars = 657 [ 658 // Keywords 659 TOK.this_: "this", 660 TOK.super_: "super", 661 TOK.assert_: "assert", 662 TOK.null_: "null", 663 TOK.true_: "true", 664 TOK.false_: "false", 665 TOK.cast_: "cast", 666 TOK.new_: "new", 667 TOK.delete_: "delete", 668 TOK.throw_: "throw", 669 TOK.module_: "module", 670 TOK.pragma_: "pragma", 671 TOK.typeof_: "typeof", 672 TOK.typeid_: "typeid", 673 TOK.template_: "template", 674 TOK.void_: "void", 675 TOK.int8: "byte", 676 TOK.uns8: "ubyte", 677 TOK.int16: "short", 678 TOK.uns16: "ushort", 679 TOK.int32: "int", 680 TOK.uns32: "uint", 681 TOK.int64: "long", 682 TOK.uns64: "ulong", 683 TOK.int128: "cent", 684 TOK.uns128: "ucent", 685 TOK.float32: "float", 686 TOK.float64: "double", 687 TOK.float80: "real", 688 TOK.bool_: "bool", 689 TOK.char_: "char", 690 TOK.wchar_: "wchar", 691 TOK.dchar_: "dchar", 692 TOK.imaginary32: "ifloat", 693 TOK.imaginary64: "idouble", 694 TOK.imaginary80: "ireal", 695 TOK.complex32: "cfloat", 696 TOK.complex64: "cdouble", 697 TOK.complex80: "creal", 698 TOK.delegate_: "delegate", 699 TOK.function_: "function", 700 TOK.is_: "is", 701 TOK.if_: "if", 702 TOK.else_: "else", 703 TOK.while_: "while", 704 TOK.for_: "for", 705 TOK.do_: "do", 706 TOK.switch_: "switch", 707 TOK.case_: "case", 708 TOK.default_: "default", 709 TOK.break_: "break", 710 TOK.continue_: "continue", 711 TOK.synchronized_: "synchronized", 712 TOK.return_: "return", 713 TOK.goto_: "goto", 714 TOK.try_: "try", 715 TOK.catch_: "catch", 716 TOK.finally_: "finally", 717 TOK.with_: "with", 718 TOK.asm_: "asm", 719 TOK.foreach_: "foreach", 720 TOK.foreach_reverse_: "foreach_reverse", 721 TOK.scope_: "scope", 722 TOK.struct_: "struct", 723 TOK.class_: "class", 724 TOK.interface_: "interface", 725 TOK.union_: "union", 726 TOK.enum_: "enum", 727 TOK.import_: "import", 728 TOK.mixin_: "mixin", 729 TOK.static_: "static", 730 TOK.final_: "final", 731 TOK.const_: "const", 732 TOK.alias_: "alias", 733 TOK.override_: "override", 734 TOK.abstract_: "abstract", 735 TOK.debug_: "debug", 736 TOK.deprecated_: "deprecated", 737 TOK.in_: "in", 738 TOK.out_: "out", 739 TOK.inout_: "inout", 740 TOK.lazy_: "lazy", 741 TOK.auto_: "auto", 742 TOK.align_: "align", 743 TOK.extern_: "extern", 744 TOK.private_: "private", 745 TOK.package_: "package", 746 TOK.protected_: "protected", 747 TOK.public_: "public", 748 TOK.export_: "export", 749 TOK.invariant_: "invariant", 750 TOK.unittest_: "unittest", 751 TOK.version_: "version", 752 TOK.argumentTypes: "__argTypes", 753 TOK.parameters: "__parameters", 754 TOK.ref_: "ref", 755 TOK.macro_: "macro", 756 TOK.pure_: "pure", 757 TOK.nothrow_: "nothrow", 758 TOK.gshared: "__gshared", 759 TOK.traits: "__traits", 760 TOK.vector: "__vector", 761 TOK.file: "__FILE__", 762 TOK.fileFullPath: "__FILE_FULL_PATH__", 763 TOK.line: "__LINE__", 764 TOK.moduleString: "__MODULE__", 765 TOK.functionString: "__FUNCTION__", 766 TOK.prettyFunction: "__PRETTY_FUNCTION__", 767 TOK.shared_: "shared", 768 TOK.immutable_: "immutable", 769 770 TOK.endOfFile: "End of File", 771 TOK.leftCurly: "{", 772 TOK.rightCurly: "}", 773 TOK.leftParenthesis: "(", 774 TOK.rightParenthesis: ")", 775 TOK.leftBracket: "[", 776 TOK.rightBracket: "]", 777 TOK.semicolon: ";", 778 TOK.colon: ":", 779 TOK.comma: ",", 780 TOK.dot: ".", 781 TOK.xor: "^", 782 TOK.xorAssign: "^=", 783 TOK.assign: "=", 784 TOK.lessThan: "<", 785 TOK.greaterThan: ">", 786 TOK.lessOrEqual: "<=", 787 TOK.greaterOrEqual: ">=", 788 TOK.equal: "==", 789 TOK.notEqual: "!=", 790 TOK.not: "!", 791 TOK.leftShift: "<<", 792 TOK.rightShift: ">>", 793 TOK.unsignedRightShift: ">>>", 794 TOK.add: "+", 795 TOK.min: "-", 796 TOK.mul: "*", 797 TOK.div: "/", 798 TOK.mod: "%", 799 TOK.slice: "..", 800 TOK.dotDotDot: "...", 801 TOK.and: "&", 802 TOK.andAnd: "&&", 803 TOK.or: "|", 804 TOK.orOr: "||", 805 TOK.tilde: "~", 806 TOK.dollar: "$", 807 TOK.plusPlus: "++", 808 TOK.minusMinus: "--", 809 TOK.question: "?", 810 TOK.variable: "var", 811 TOK.addAssign: "+=", 812 TOK.minAssign: "-=", 813 TOK.mulAssign: "*=", 814 TOK.divAssign: "/=", 815 TOK.modAssign: "%=", 816 TOK.leftShiftAssign: "<<=", 817 TOK.rightShiftAssign: ">>=", 818 TOK.unsignedRightShiftAssign: ">>>=", 819 TOK.andAssign: "&=", 820 TOK.orAssign: "|=", 821 TOK.concatenateAssign: "~=", 822 TOK.identity: "is", 823 TOK.notIdentity: "!is", 824 TOK.identifier: "identifier", 825 TOK.at: "@", 826 TOK.pow: "^^", 827 TOK.powAssign: "^^=", 828 TOK.goesTo: "=>", 829 TOK.pound: "#", 830 TOK.arrow: "->", 831 TOK.colonColon: "::", 832 833 // For debugging 834 TOK.error: "error", 835 TOK.string_: "string", 836 TOK.onScopeExit: "scope(exit)", 837 TOK.onScopeSuccess: "scope(success)", 838 TOK.onScopeFailure: "scope(failure)", 839 840 // Finish up 841 TOK.reserved: "reserved", 842 TOK.comment: "comment", 843 TOK.int32Literal: "int32v", 844 TOK.uns32Literal: "uns32v", 845 TOK.int64Literal: "int64v", 846 TOK.uns64Literal: "uns64v", 847 TOK.int128Literal: "int128v", 848 TOK.uns128Literal: "uns128v", 849 TOK.float32Literal: "float32v", 850 TOK.float64Literal: "float64v", 851 TOK.float80Literal: "float80v", 852 TOK.imaginary32Literal: "imaginary32v", 853 TOK.imaginary64Literal: "imaginary64v", 854 TOK.imaginary80Literal: "imaginary80v", 855 TOK.charLiteral: "charv", 856 TOK.wcharLiteral: "wcharv", 857 TOK.dcharLiteral: "dcharv", 858 TOK.wchar_tLiteral: "wchar_tv", 859 TOK.hexadecimalString: "xstring", 860 TOK.endOfLine: "\\n", 861 TOK.whitespace: "whitespace", 862 863 // C only keywords 864 TOK.inline : "inline", 865 TOK.register : "register", 866 TOK.restrict : "restrict", 867 TOK.signed : "signed", 868 TOK.sizeof_ : "sizeof", 869 TOK.typedef_ : "typedef", 870 TOK.unsigned : "unsigned", 871 TOK..volatile : "volatile", 872 TOK._Alignas : "_Alignas", 873 TOK._Alignof : "_Alignof", 874 TOK._Atomic : "_Atomic", 875 TOK._Bool : "_Bool", 876 TOK._Complex : "_Complex", 877 TOK._Generic : "_Generic", 878 TOK._Imaginary: "_Imaginary", 879 TOK._Noreturn : "_Noreturn", 880 TOK._Static_assert : "_Static_assert", 881 TOK._Thread_local : "_Thread_local", 882 883 // C only extended keywords 884 TOK._assert : "__check", 885 TOK._import : "__import", 886 TOK.__cdecl : "__cdecl", 887 TOK.__declspec : "__declspec", 888 TOK.__stdcall : "__stdcall", 889 TOK.__thread : "__thread", 890 TOK.__pragma : "__pragma", 891 TOK.__int128 : "__int128", 892 TOK.__attribute__ : "__attribute__", 893 ]; 894 895 static assert(() { 896 foreach (s; tochars) 897 assert(s.length); 898 return true; 899 }()); 900 901 nothrow: 902 903 extern (D) int isKeyword() pure const @safe @nogc 904 { 905 foreach (kw; keywords) 906 { 907 if (kw == value) 908 return 1; 909 } 910 return 0; 911 } 912 913 /**** 914 * Set to contents of ptr[0..length] 915 * Params: 916 * ptr = pointer to string 917 * length = length of string 918 */ 919 void setString(const(char)* ptr, size_t length) 920 { 921 auto s = cast(char*)mem.xmalloc_noscan(length + 1); 922 memcpy(s, ptr, length); 923 s[length] = 0; 924 ustring = s; 925 len = cast(uint)length; 926 postfix = 0; 927 } 928 929 /**** 930 * Set to contents of buf 931 * Params: 932 * buf = string (not zero terminated) 933 */ 934 void setString(const ref OutBuffer buf) 935 { 936 setString(cast(const(char)*)buf[].ptr, buf.length); 937 } 938 939 /**** 940 * Set to empty string 941 */ 942 void setString() 943 { 944 ustring = ""; 945 len = 0; 946 postfix = 0; 947 } 948 949 extern (C++) const(char)* toChars() const 950 { 951 const bufflen = 3 + 3 * floatvalue.sizeof + 1; 952 __gshared char[bufflen] buffer; 953 const(char)* p = &buffer[0]; 954 switch (value) 955 { 956 case TOK.int32Literal: 957 snprintf(&buffer[0], bufflen, "%d", cast(int)intvalue); 958 break; 959 case TOK.uns32Literal: 960 case TOK.wchar_tLiteral: 961 snprintf(&buffer[0], bufflen, "%uU", cast(uint)unsvalue); 962 break; 963 case TOK.wcharLiteral: 964 case TOK.dcharLiteral: 965 case TOK.charLiteral: 966 { 967 OutBuffer buf; 968 buf.writeSingleCharLiteral(cast(dchar) intvalue); 969 buf.writeByte('\0'); 970 p = buf.extractChars(); 971 } 972 break; 973 case TOK.int64Literal: 974 snprintf(&buffer[0], bufflen, "%lldL", cast(long)intvalue); 975 break; 976 case TOK.uns64Literal: 977 snprintf(&buffer[0], bufflen, "%lluUL", cast(ulong)unsvalue); 978 break; 979 case TOK.float32Literal: 980 CTFloat.sprint(&buffer[0], bufflen, 'g', floatvalue); 981 strcat(&buffer[0], "f"); 982 break; 983 case TOK.float64Literal: 984 CTFloat.sprint(&buffer[0], bufflen, 'g', floatvalue); 985 break; 986 case TOK.float80Literal: 987 CTFloat.sprint(&buffer[0], bufflen, 'g', floatvalue); 988 strcat(&buffer[0], "L"); 989 break; 990 case TOK.imaginary32Literal: 991 CTFloat.sprint(&buffer[0], bufflen, 'g', floatvalue); 992 strcat(&buffer[0], "fi"); 993 break; 994 case TOK.imaginary64Literal: 995 CTFloat.sprint(&buffer[0], bufflen, 'g', floatvalue); 996 strcat(&buffer[0], "i"); 997 break; 998 case TOK.imaginary80Literal: 999 CTFloat.sprint(&buffer[0], bufflen, 'g', floatvalue); 1000 strcat(&buffer[0], "Li"); 1001 break; 1002 case TOK.string_: 1003 { 1004 OutBuffer buf; 1005 buf.writeByte('"'); 1006 for (size_t i = 0; i < len;) 1007 { 1008 dchar c; 1009 utf_decodeChar(ustring[0 .. len], i, c); 1010 writeCharLiteral(buf, c); 1011 } 1012 buf.writeByte('"'); 1013 if (postfix) 1014 buf.writeByte(postfix); 1015 buf.writeByte(0); 1016 p = buf.extractChars(); 1017 } 1018 break; 1019 case TOK.hexadecimalString: 1020 { 1021 OutBuffer buf; 1022 buf.writeByte('x'); 1023 buf.writeByte('"'); 1024 foreach (size_t i; 0 .. len) 1025 { 1026 if (i) 1027 buf.writeByte(' '); 1028 buf.printf("%02x", ustring[i]); 1029 } 1030 buf.writeByte('"'); 1031 if (postfix) 1032 buf.writeByte(postfix); 1033 buf.writeByte(0); 1034 p = buf.extractData(); 1035 break; 1036 } 1037 case TOK.identifier: 1038 case TOK.enum_: 1039 case TOK.struct_: 1040 case TOK.import_: 1041 case TOK.wchar_: 1042 case TOK.dchar_: 1043 case TOK.bool_: 1044 case TOK.char_: 1045 case TOK.int8: 1046 case TOK.uns8: 1047 case TOK.int16: 1048 case TOK.uns16: 1049 case TOK.int32: 1050 case TOK.uns32: 1051 case TOK.int64: 1052 case TOK.uns64: 1053 case TOK.int128: 1054 case TOK.uns128: 1055 case TOK.float32: 1056 case TOK.float64: 1057 case TOK.float80: 1058 case TOK.imaginary32: 1059 case TOK.imaginary64: 1060 case TOK.imaginary80: 1061 case TOK.complex32: 1062 case TOK.complex64: 1063 case TOK.complex80: 1064 case TOK.void_: 1065 p = ident.toChars(); 1066 break; 1067 default: 1068 p = toChars(value); 1069 break; 1070 } 1071 return p; 1072 } 1073 1074 static const(char)* toChars(TOK value) 1075 { 1076 return toString(value).ptr; 1077 } 1078 1079 extern (D) static string toString(TOK value) pure nothrow @nogc @safe 1080 { 1081 return tochars[value]; 1082 } 1083 } 1084 1085 /** 1086 * Write a character, using a readable escape sequence if needed 1087 * 1088 * Useful for printing "" string literals in e.g. error messages, ddoc, or the `.stringof` property 1089 * 1090 * Params: 1091 * buf = buffer to append character in 1092 * c = code point to write 1093 */ 1094 nothrow 1095 void writeCharLiteral(ref OutBuffer buf, dchar c) 1096 { 1097 switch (c) 1098 { 1099 case '\0': 1100 buf.writestring("\\0"); 1101 break; 1102 case '\n': 1103 buf.writestring("\\n"); 1104 break; 1105 case '\r': 1106 buf.writestring("\\r"); 1107 break; 1108 case '\t': 1109 buf.writestring("\\t"); 1110 break; 1111 case '\b': 1112 buf.writestring("\\b"); 1113 break; 1114 case '\f': 1115 buf.writestring("\\f"); 1116 break; 1117 case '"': 1118 case '\\': 1119 buf.writeByte('\\'); 1120 goto default; 1121 default: 1122 if (c <= 0xFF) 1123 { 1124 if (isprint(c)) 1125 buf.writeByte(c); 1126 else 1127 buf.printf("\\x%02x", c); 1128 } 1129 else if (c <= 0xFFFF) 1130 buf.printf("\\u%04x", c); 1131 else 1132 buf.printf("\\U%08x", c); 1133 break; 1134 } 1135 } 1136 1137 unittest 1138 { 1139 OutBuffer buf; 1140 foreach(dchar d; "a\n\r\t\b\f\0\x11\u7233\U00017233"d) 1141 { 1142 writeCharLiteral(buf, d); 1143 } 1144 assert(buf[] == `a\n\r\t\b\f\0\x11\u7233\U00017233`); 1145 } 1146 1147 /** 1148 * Write a single-quoted character literal 1149 * 1150 * Useful for printing '' char literals in e.g. error messages, ddoc, or the `.stringof` property 1151 * 1152 * Params: 1153 * buf = buffer to append character in 1154 * c = code point to write 1155 */ 1156 nothrow 1157 void writeSingleCharLiteral(ref OutBuffer buf, dchar c) 1158 { 1159 buf.writeByte('\''); 1160 if (c == '\'') 1161 buf.writeByte('\\'); 1162 1163 if (c == '"') 1164 buf.writeByte('"'); 1165 else 1166 writeCharLiteral(buf, c); 1167 1168 buf.writeByte('\''); 1169 } 1170 1171 unittest 1172 { 1173 OutBuffer buf; 1174 writeSingleCharLiteral(buf, '\''); 1175 assert(buf[] == `'\''`); 1176 buf.reset(); 1177 writeSingleCharLiteral(buf, '"'); 1178 assert(buf[] == `'"'`); 1179 buf.reset(); 1180 writeSingleCharLiteral(buf, '\n'); 1181 assert(buf[] == `'\n'`); 1182 }