Verified Commit 1089719f authored by Dmitry Volodin's avatar Dmitry Volodin
Browse files

LineTokenizer: Fix string_quote processing

parent 5a61c07b
......@@ -111,7 +111,7 @@ class LineTokenizer(BaseTokenizer):
while start < llen:
qi = line.find(self.string_quote, start)
if qi == -1:
qi = llen
qi = llen + 1
if in_string:
yield line[start:qi]
in_string = False
......
......@@ -68,9 +68,43 @@ TOKENS1 = [
("services", "isis", "interface", "ge-0/0/1.0")
]
CFG2 = """snmp {
community pub1ic {
authorization read-only;
}
community nnnn {
authorization read-only;
}
community "@default" {
authorization read-only;
}
community TTT {
authorization read-only;
clients {
10.10.10.0/26;
}
}
}
"""
TOKENS2 = [
("snmp",),
("snmp", "community", "pub1ic"),
("snmp", "community", "pub1ic", "authorization", "read-only"),
("snmp", "community", "nnnn"),
("snmp", "community", "nnnn", "authorization", "read-only"),
("snmp", "community", "@default"),
("snmp", "community", "@default", "authorization", "read-only"),
("snmp", "community", "TTT"),
("snmp", "community", "TTT", "authorization", "read-only"),
("snmp", "community", "TTT", "clients"),
("snmp", "community", "TTT", "clients", "10.10.10.0/26")
]
@pytest.mark.parametrize("input,config,expected", [
(CFG1, {"line_comment": "#", "explicit_eol": ";"}, TOKENS1),
(CFG2, {"line_comment": "#", "explicit_eol": ";", "string_quote": "\""}, TOKENS2),
])
def test_tokenizer(input, config, expected):
tokenizer = CurlyTokenizer(input, **config)
......
......@@ -70,6 +70,9 @@ interface e1
interface e2
description "multi line" interface e2 "description here"
interface e3
description "multi line" Z
"""
TOKENS5 = [
......@@ -79,7 +82,9 @@ TOKENS5 = [
("interface", "e1"),
(" ", "description", "unterminated description"),
("interface", "e2"),
(" ", "description", "multi line", "interface", "e2", "description here")
(" ", "description", "multi line", "interface", "e2", "description here"),
("interface", "e3"),
(" ", "description", "multi line", "Z")
]
CFG6 = """! Config
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment