diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 34dd4fb5..46f96f88 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,7 +11,7 @@ New features: Fixes: - +* #1057: Preserve the indentation of pasted code Changes to dependencies: diff --git a/bpython/curtsiesfrontend/preprocess.py b/bpython/curtsiesfrontend/preprocess.py index f48a79bf..53403663 100644 --- a/bpython/curtsiesfrontend/preprocess.py +++ b/bpython/curtsiesfrontend/preprocess.py @@ -12,6 +12,10 @@ indent_empty_lines_re = LazyReCompile(r"\s*") tabs_to_spaces_re = LazyReCompile(r"^\t+") +# keywords introducing a clause that continues the block above it rather than +# starting a new statement, even though they sit at the same indentation +BLOCK_CONTINUATION_CLAUSES = frozenset(("elif", "else", "except", "finally")) + def indent_empty_lines(s: str, compiler: CommandCompiler) -> str: """Indents blank lines that would otherwise cause early compilation @@ -49,5 +53,58 @@ def tab_to_space(m: Match[str]) -> str: ) +def _compiles(lines: list[str], compiler: CommandCompiler) -> bool: + try: + return bool(compiler("\n".join(lines))) + except (ValueError, SyntaxError, OverflowError): + return False + + +def starts_new_statement(line: str) -> bool: + """Whether a line is a statement of its own rather than part of a block""" + if not line.strip() or line[:1].isspace(): + return False + # a comment is inert, and may well be sitting above an 'else:' + if line.startswith("#"): + return False + return ( + line.split(maxsplit=1)[0].rstrip(":") not in BLOCK_CONTINUATION_CLAUSES + ) + + +def separate_blocks(s: str, compiler: CommandCompiler) -> str: + """Insert the blank lines needed to close indented blocks + + Interactively a block is ended by a blank line, but pasted code has none + between a block and the statement that follows it, so without this that + statement would be swallowed into the block instead of being run. + + Only really works if starting on a new line""" + result_lines: list[str] = [] + # the lines the REPL would currently be holding in its buffer + buffer: list[str] = [] + + for line in s.split("\n"): + # A block is merely waiting for its blank line, whereas a buffer held + # open by an unclosed bracket stays unfinished however many follow. + if ( + buffer + and starts_new_statement(line) + and _compiles(buffer + ["", ""], compiler) + ): + result_lines.append("") + buffer = [] + + result_lines.append(line) + buffer.append(line) + if not line.strip() or _compiles(buffer, compiler): + # the REPL runs the buffer and starts collecting afresh + buffer = [] + + return "\n".join(result_lines) + + def preprocess(s: str, compiler: CommandCompiler) -> str: - return indent_empty_lines(leading_tabs_to_spaces(s), compiler) + return separate_blocks( + indent_empty_lines(leading_tabs_to_spaces(s), compiler), compiler + ) diff --git a/bpython/curtsiesfrontend/repl.py b/bpython/curtsiesfrontend/repl.py index 928be253..8dad46e0 100644 --- a/bpython/curtsiesfrontend/repl.py +++ b/bpython/curtsiesfrontend/repl.py @@ -81,6 +81,21 @@ # i.e. control characters like '' will be stripped MAX_EVENTS_POSSIBLY_NOT_PASTE = 20 +# Keys that insert text instead of performing an action, mapped to the +# character they stand for. In a paste these carry the content of the pasted +# code - in particular the whitespace making up its indentation - so unlike +# keys such as '' they must never be dropped or deduplicated. +CONTENT_KEYS = { + "": "\n", + "": "\n", + "": "\n", + "\n": "\n", + "\r": "\n", + "": " ", + "": "\t", + "\t": "\t", +} + class SearchMode(Enum): NO_SEARCH = 0 @@ -1081,6 +1096,11 @@ def process_simple_keypress(self, e: str): pass # ignore events elif e == "": self.add_normal_character(" ") + elif e in ("", "\t"): + # add_normal_character ignores control characters, so expand tabs + # here rather than losing the indentation they stand for + for _ in range(self.config.tab_length): + self.add_normal_character(" ") else: self.add_normal_character(e) @@ -2213,6 +2233,10 @@ def compress_paste_event(paste_event): if not all(paste_event.events[0] == e for e in paste_event.events): return None event = paste_event.events[0] + # A run of identical content keys is ordinary pasted text - indentation or + # blank lines - rather than a held down key, so it must not be collapsed. + if is_simple_event(event): + return None # basically "is there a special curtsies names for this key?" if len(event) > 1: return event @@ -2225,11 +2249,9 @@ def just_simple_events(event_list: Iterable[str | events.Event]) -> list[str]: for e in event_list: if isinstance(e, events.Event): continue # ignore events - # '\n' necessary for pastes - elif e in ("", "", "", "\n", "\r"): - simple_events.append("\n") - elif e == "": - simple_events.append(" ") + # content keys carry the text of a paste, including its indentation + elif e in CONTENT_KEYS: + simple_events.append(CONTENT_KEYS[e]) elif len(e) > 1: continue # get rid of etc. else: @@ -2240,7 +2262,4 @@ def just_simple_events(event_list: Iterable[str | events.Event]) -> list[str]: def is_simple_event(e: str | events.Event) -> bool: if isinstance(e, events.Event): return False - return ( - e in ("", "", "", "\n", "\r", "") - or len(e) <= 1 - ) + return e in CONTENT_KEYS or len(e) <= 1 diff --git a/bpython/test/test_curtsies_repl.py b/bpython/test/test_curtsies_repl.py index 59102f9e..c640fafc 100644 --- a/bpython/test/test_curtsies_repl.py +++ b/bpython/test/test_curtsies_repl.py @@ -483,6 +483,51 @@ def test_control_events_in_large_paste(self): "a" + "e" * curtsiesrepl.MAX_EVENTS_POSSIBLY_NOT_PASTE, ) + def paste(self, source): + """Paste source as the key events curtsies would report for it""" + p = events.PasteEvent() + p.events = [ + {" ": "", "\t": "", "\n": ""}.get(c, c) + for c in source + ] + self.repl.process_event(p) + return [ + line + for line, line_type in self.repl.all_logical_lines + if line_type == LineType.INPUT + ] + + def test_repeated_content_keys_are_not_compressed(self): + """A run of spaces is indentation, not a key being held down""" + for key in ("", "", ""): + with self.subTest(key=key): + p = events.PasteEvent() + p.events = [key] * 10 + self.assertIsNone(curtsiesrepl.compress_paste_event(p)) + + def test_repeated_control_keys_are_still_compressed(self): + p = events.PasteEvent() + p.events = [""] * 10 + self.assertEqual(curtsiesrepl.compress_paste_event(p), "") + + def test_deep_indentation_is_preserved(self): + """Indentation long enough to fill a read must survive (#1057)""" + self.assertEqual(self.paste(" " * 12 + "x"), []) + self.assertEqual(self.repl.current_line, " " * 12 + "x") + + def test_space_indented_paste(self): + self.assertEqual( + self.paste("for i in (1, 2):\n x = i\n y = x\n"), + ["for i in (1, 2):", " x = i", " y = x"], + ) + + def test_tab_indented_paste(self): + """Tabs carry indentation and must not be dropped (#1057)""" + self.assertEqual( + self.paste("for i in (1, 2):\n\tx = i\n\ty = x\n"), + ["for i in (1, 2):", " x = i", " y = x"], + ) + if __name__ == "__main__": unittest.main() diff --git a/bpython/test/test_preprocess.py b/bpython/test/test_preprocess.py index 8e8a3630..57b0a990 100644 --- a/bpython/test/test_preprocess.py +++ b/bpython/test/test_preprocess.py @@ -105,3 +105,76 @@ def test_blank_trailing_line(self): def test_tabs(self): self.assertIndented(original.tabs) + + +class TestSeparateBlocks(unittest.TestCase): + """Pasted code has no blank line after a block, so one has to be added + for the statement that follows it to be run separately (#1057)""" + + def test_statement_after_block(self): + self.assertEqual( + preproc("for i in r:\n f(i)\nprint(1)\n"), + "for i in r:\n f(i)\n\nprint(1)\n", + ) + + def test_statement_after_nested_blocks(self): + self.assertEqual( + preproc("for i in r:\n if i:\n f(i)\nprint(1)\n"), + "for i in r:\n if i:\n f(i)\n\nprint(1)\n", + ) + + def test_consecutive_blocks(self): + self.assertEqual( + preproc("if a:\n f()\nwhile b:\n g()\n"), + "if a:\n f()\n\nwhile b:\n g()\n", + ) + + def test_clauses_continuing_a_block_are_left_alone(self): + for source in ( + "if a:\n f()\nelif b:\n g()\nelse:\n h()\n", + "try:\n f()\nexcept E:\n g()\nfinally:\n h()\n", + "for i in r:\n f(i)\nelse:\n g()\n", + ): + with self.subTest(source=source): + self.assertEqual(preproc(source), source) + + def test_unindented_lines_inside_brackets_are_left_alone(self): + # an unclosed bracket keeps the statement going, so these lines are + # continuations rather than new statements + for source in ( + "total = (\n1 +\n2\n)\n", + "items = [\n'a',\n'b',\n]\n", + "f(\nx,\n)\n", + ): + with self.subTest(source=source): + self.assertEqual(preproc(source), source) + + def test_consecutive_simple_statements_are_left_alone(self): + source = "a = 1\nb = 2\nprint(a + b)\n" + self.assertEqual(preproc(source), source) + + def test_existing_blank_line_is_not_doubled(self): + source = "for i in r:\n f(i)\n\nprint(1)\n" + self.assertEqual(preproc(source), source) + + def test_block_after_statement(self): + source = "a = 1\nfor i in r:\n f(i)\n" + self.assertEqual(preproc(source), source) + + def test_comment_above_a_continuing_clause_is_left_alone(self): + source = "if a:\n f()\n# why not\nelse:\n g()\n" + self.assertEqual(preproc(source), source) + + def test_decorator_is_not_separated_from_its_definition(self): + self.assertEqual( + preproc("@deco\ndef f():\n pass\nprint(1)\n"), + "@deco\ndef f():\n pass\n\nprint(1)\n", + ) + + def test_lines_inside_a_string_are_left_alone(self): + source = 's = """\nabc\ndef\n"""\nprint(s)\n' + self.assertEqual(preproc(source), source) + + def test_backslash_continuation_is_left_alone(self): + source = "a = 1 + \\\n 2\nprint(a)\n" + self.assertEqual(preproc(source), source)