Add support for a single quote in a character literal and a double quote in a string literal

Author: Sebastian Kaupper

pyVHDLParser/Token/Parser.py

@@ -66,13 +66,14 @@ class TokenKind(IntEnum):
 		PossibleRealLiteral =             7   #: Last char was a ``.``
 		PossibleCharacterLiteral =        8   #: Last char was a ``'``
 		PossibleStringLiteralStart =      9   #: Last char was a ``"``
-		PossibleExtendedIdentifierStart = 10  #: Last char was a ``\``
-		SingleLineComment =               11  #: Found ``--`` before
-		MultiLineComment =                12  #: Found ``/*`` before
-		Linebreak =                       13  #: Last char was a ``\n``
-		Directive =                       14  #: Last char was a `` ` ``
-		FuseableCharacter =               15  #: Last char was a character that could be fused
-		OtherChars =                      16  #: Anything else
+		PossibleStringLiteralEnd =        10  #: Last char was a ``"`` while being in state ``PossibleStringLiteralStart``
+		PossibleExtendedIdentifierStart = 11  #: Last char was a ``\``
+		SingleLineComment =               12  #: Found ``--`` before
+		MultiLineComment =                13  #: Found ``/*`` before
+		Linebreak =                       14  #: Last char was a ``\n``
+		Directive =                       15  #: Last char was a `` ` ``
+		FuseableCharacter =               16  #: Last char was a character that could be fused
+		OtherChars =                      17  #: Anything else
 
 	@classmethod
 	def GetVHDLTokenizer(cls, iterable: Iterator[str]):
@@ -336,24 +337,23 @@ def GetVHDLTokenizer(cls, iterable: Iterator[str]):
 				buffer += char
 				if len(buffer) == 2:
 					if buffer[1] == "(" and isinstance(previousToken, WordToken):
+						# An input of the form `<word>'(` always must be a qualified expression
+						# in order to be valid VHDL. There is not case where `'('` would be a valid character literal
+						# if preceeded by a word token
 						previousToken =   CharacterToken(previousToken, "'", start)
 						yield previousToken
 						previousToken =   CharacterToken(previousToken, "(", SourceCodePosition(row, column, absolute))
 						yield previousToken
 						tokenKind =       cls.TokenKind.OtherChars
-					elif buffer[1] == "'":
-						previousToken =   CharacterToken(previousToken, "'", start)
-						yield previousToken
-						previousToken =   CharacterToken(previousToken, "'", SourceCodePosition(row, column, absolute))
-						yield previousToken
-						tokenKind =       cls.TokenKind.OtherChars
 					else:
 						continue
 				elif (len(buffer) == 3) and (buffer[2] == "'"):
+					# Whatever is enclosed in single quotes, is the content of a character literal
 					previousToken =   CharacterLiteralToken(previousToken, buffer, start, SourceCodePosition(row, column, absolute))
 					yield previousToken
 					tokenKind = cls.TokenKind.OtherChars
 				else:
+					# If the third entry of the buffer is not a closing single quote, the single quote must belong to an attribute
 					previousToken =   CharacterToken(previousToken, "'", start)
 					yield previousToken
 
@@ -371,9 +371,41 @@ def GetVHDLTokenizer(cls, iterable: Iterator[str]):
 			elif tokenKind is cls.TokenKind.PossibleStringLiteralStart:
 				buffer += char
 				if char == "\"":
-					previousToken = StringLiteralToken(previousToken, buffer, start, SourceCodePosition(row, column, absolute))
+					tokenKind = cls.TokenKind.PossibleStringLiteralEnd
+
+			# State: PossibleStringLiteralEnd
+			elif tokenKind is cls.TokenKind.PossibleStringLiteralEnd:
+				if char == "\"":
+					buffer += char
+					tokenKind = cls.TokenKind.PossibleStringLiteralStart
+				else:
+					previousToken = StringLiteralToken(previousToken, buffer, start, SourceCodePosition(row, column-1, absolute-1))
 					yield previousToken
-					tokenKind = cls.TokenKind.OtherChars
+
+					start =   SourceCodePosition(row, column, absolute)
+					buffer =  char
+					if char in __WHITESPACE_CHARACTERS__: tokenKind = cls.TokenKind.SpaceChars
+					elif char in __NUMBER_CHARACTERS__:   tokenKind = cls.TokenKind.IntegerChars
+					elif char in __ALPHA_CHARACTERS__:    tokenKind = cls.TokenKind.AlphaChars
+					elif char == "'":                     tokenKind = cls.TokenKind.PossibleCharacterLiteral
+					elif char == "\"":                    tokenKind = cls.TokenKind.PossibleStringLiteralStart
+					elif char == "-":                     tokenKind = cls.TokenKind.PossibleSingleLineCommentStart
+					elif char == "\r":                    tokenKind = cls.TokenKind.PossibleLinebreak
+					elif char == "\n":
+						previousToken = LinebreakToken(previousToken, char, start, start)
+						yield previousToken
+						tokenKind = cls.TokenKind.OtherChars
+					elif char in __FUSEABLE_CHARS__:
+						buffer =        char
+						tokenKind =     cls.TokenKind.FuseableCharacter
+					elif char == ".":                     tokenKind = cls.TokenKind.PossibleRealLiteral
+					elif char == "\\":                    tokenKind = cls.TokenKind.PossibleExtendedIdentifierStart
+					elif (char == "`") and isinstance(previousToken, (WhitespaceToken, LinebreakToken)):
+						tokenKind = cls.TokenKind.Directive
+					else:
+						previousToken = CharacterToken(previousToken, char, start)
+						yield previousToken
+						tokenKind =     cls.TokenKind.OtherChars
 
 			# State: PossibleExtendedIdentifierStart
 			elif tokenKind is cls.TokenKind.PossibleExtendedIdentifierStart:

tests/unit/Tokenizer/Tokens.py

@@ -130,7 +130,7 @@ class Sequence_1(TestCase, ExpectedDataMixin, TokenSequence):
 	)
 
 class Sequence_2(TestCase, ExpectedDataMixin, TokenSequence):
-	code = """abc   \\def\\ \t 'a' "abc" /* help */ -- foo\n """
+	code = """abc   \\def\\ \t 'a' ''' "abc" "\"\"" "foo\"\"" /* help */ -- foo\n """
 	tokenStream = ExpectedTokenStream(
 		[(StartOfDocumentToken, None),
          (WordToken,               "abc"),
@@ -139,8 +139,14 @@ class Sequence_2(TestCase, ExpectedDataMixin, TokenSequence):
          (WhitespaceToken, " \t "),
          (CharacterLiteralToken,   "a"),
          (WhitespaceToken, " "),
+         (CharacterLiteralToken,   "'"),
+         (WhitespaceToken, " "),
          (StringLiteralToken,      "abc"),
          (WhitespaceToken, " "),
+         (StringLiteralToken,      "\"\""),
+         (WhitespaceToken, " "),
+         (StringLiteralToken,      "foo\"\""),
+         (WhitespaceToken, " "),
          (MultiLineCommentToken,   "/* help */"),
          (WhitespaceToken, " "),
          (SingleLineCommentToken,  "-- foo\n"),
@@ -303,7 +309,7 @@ class Sequence_6(TestCase, ExpectedDataMixin, TokenSequence):
 
 
 class Sequence_7(TestCase, ExpectedDataMixin, TokenSequence):
-	code = """constant BIT_STRING : UNSIGNED(0 downto 0) := UNSIGNED'(x\"0\");\nconstant LPAREN_CHAR : character := '(';\nfoo'('0')\nbar'('(')"""
+	code = """constant BIT_STRING : UNSIGNED(0 downto 0) := UNSIGNED'(x\"0\");\nconstant LPAREN_CHAR : character := '(';\nfoo'('0')\nbar'('(')\ncharacter'(''')"""
 	tokenStream = ExpectedTokenStream(
 		[ (StartOfDocumentToken,   None),
 			(WordToken,               "constant"),
@@ -355,11 +361,16 @@ class Sequence_7(TestCase, ExpectedDataMixin, TokenSequence):
 			(CharacterToken,          "("),
 			(CharacterLiteralToken,   "("),
 			(CharacterToken,          ")"),
+			(LinebreakToken,          None),
+			(WordToken,               "character"),
+			(CharacterToken,          "'"),
+			(CharacterToken,          "("),
+			(CharacterLiteralToken,   "'"),
+			(CharacterToken,          ")"),
 			(EndOfDocumentToken,     None)
 		]
 	)
 
-
 class Tokenizer_ExceptionInKeyword(TestCase, ExpectedDataMixin, TokenSequence):
 	code = """keyword"""
 	tokenStream = ExpectedTokenStream(