From d37fe0e897f384a8a661a9642231daa9e13ad881 Mon Sep 17 00:00:00 2001 From: furunkel Date: Wed, 16 Jun 2021 13:55:05 +0200 Subject: [PATCH] Expose a token's underlying input range --- javalang/test/test_tokenizer.py | 13 +++++++++++++ javalang/tokenizer.py | 4 ++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/javalang/test/test_tokenizer.py b/javalang/test/test_tokenizer.py index 2fbc7d4..b0f767c 100644 --- a/javalang/test/test_tokenizer.py +++ b/javalang/test/test_tokenizer.py @@ -145,12 +145,15 @@ def test_inline_comment_position(self): self.assertEqual(int_token.position.line, 1) self.assertEqual(int_token.position.column, 1) + self.assertEqual(code[int_token.position.range], 'int') self.assertEqual(j_token.position.line, 1) self.assertEqual(j_token.position.column, 19) + self.assertEqual(code[j_token.position.range], 'j') self.assertEqual(semi_token.position.line, 1) self.assertEqual(semi_token.position.column, 20) + self.assertEqual(code[semi_token.position.range], ';') def test_multiline_inline_comment(self): code = """int /* @@ -167,6 +170,8 @@ def test_multiline_inline_comment(self): self.assertEqual(token_j.position.line, 4) self.assertEqual(token_j.position.column, 4) + self.assertEqual(token_j.position.range, slice(22, 23)) + self.assertEqual(code[token_j.position.range], 'j') def test_multiline_inline_comment_end_of_input(self): code = """int /* @@ -180,6 +185,8 @@ def test_multiline_inline_comment_end_of_input(self): self.assertEqual(token_int.position.line, 1) self.assertEqual(token_int.position.column, 1) + self.assertEqual(token_int.position.range, slice(0, 3)) + def test_column_starts_at_one(self): code = """int j; int k; @@ -188,5 +195,11 @@ def test_column_starts_at_one(self): self.assertEqual(token[0].position.column, 1) self.assertEqual(token[3].position.column, 1) + self.assertEqual(token[0].position.range, slice(0, 3)) + self.assertEqual(token[3].position.range, slice(7, 10)) + + self.assertEqual(code[token[0].position.range], "int") + self.assertEqual(code[token[3].position.range], "int") + if __name__=="__main__": unittest.main() diff --git a/javalang/tokenizer.py b/javalang/tokenizer.py index d725292..ba9b040 100644 --- a/javalang/tokenizer.py +++ b/javalang/tokenizer.py @@ -8,7 +8,7 @@ class LexerError(Exception): pass -Position = namedtuple('Position', ['line', 'column']) +Position = namedtuple('Position', ['line', 'column', 'range']) class JavaToken(object): def __init__(self, value, position=None, javadoc=None): @@ -548,7 +548,7 @@ def tokenize(self): self.i = self.i + 1 continue - position = Position(self.current_line, self.i - self.start_of_line) + position = Position(self.current_line, self.i - self.start_of_line, slice(self.i, self.j)) token = token_type(self.data[self.i:self.j], position, self.javadoc) yield token