Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion CHANGELOG.rst
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ New features:


Fixes:

* #1057: Preserve the indentation of pasted code

Changes to dependencies:

Expand Down
59 changes: 58 additions & 1 deletion bpython/curtsiesfrontend/preprocess.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,10 @@
indent_empty_lines_re = LazyReCompile(r"\s*")
tabs_to_spaces_re = LazyReCompile(r"^\t+")

# keywords introducing a clause that continues the block above it rather than
# starting a new statement, even though they sit at the same indentation
BLOCK_CONTINUATION_CLAUSES = frozenset(("elif", "else", "except", "finally"))


def indent_empty_lines(s: str, compiler: CommandCompiler) -> str:
"""Indents blank lines that would otherwise cause early compilation
Expand Down Expand Up @@ -49,5 +53,58 @@ def tab_to_space(m: Match[str]) -> str:
)


def _compiles(lines: list[str], compiler: CommandCompiler) -> bool:
try:
return bool(compiler("\n".join(lines)))
except (ValueError, SyntaxError, OverflowError):
return False


def starts_new_statement(line: str) -> bool:
"""Whether a line is a statement of its own rather than part of a block"""
if not line.strip() or line[:1].isspace():
return False
# a comment is inert, and may well be sitting above an 'else:'
if line.startswith("#"):
return False
return (
line.split(maxsplit=1)[0].rstrip(":") not in BLOCK_CONTINUATION_CLAUSES
)


def separate_blocks(s: str, compiler: CommandCompiler) -> str:
"""Insert the blank lines needed to close indented blocks

Interactively a block is ended by a blank line, but pasted code has none
between a block and the statement that follows it, so without this that
statement would be swallowed into the block instead of being run.

Only really works if starting on a new line"""
result_lines: list[str] = []
# the lines the REPL would currently be holding in its buffer
buffer: list[str] = []

for line in s.split("\n"):
# A block is merely waiting for its blank line, whereas a buffer held
# open by an unclosed bracket stays unfinished however many follow.
if (
buffer
and starts_new_statement(line)
and _compiles(buffer + ["", ""], compiler)
):
result_lines.append("")
buffer = []

result_lines.append(line)
buffer.append(line)
if not line.strip() or _compiles(buffer, compiler):
# the REPL runs the buffer and starts collecting afresh
buffer = []

return "\n".join(result_lines)


def preprocess(s: str, compiler: CommandCompiler) -> str:
return indent_empty_lines(leading_tabs_to_spaces(s), compiler)
return separate_blocks(
indent_empty_lines(leading_tabs_to_spaces(s), compiler), compiler
)
37 changes: 28 additions & 9 deletions bpython/curtsiesfrontend/repl.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,21 @@
# i.e. control characters like '<Ctrl-a>' will be stripped
MAX_EVENTS_POSSIBLY_NOT_PASTE = 20

# Keys that insert text instead of performing an action, mapped to the
# character they stand for. In a paste these carry the content of the pasted
# code - in particular the whitespace making up its indentation - so unlike
# keys such as '<Ctrl-a>' they must never be dropped or deduplicated.
CONTENT_KEYS = {
"<Ctrl-j>": "\n",
"<Ctrl-m>": "\n",
"<PADENTER>": "\n",
"\n": "\n",
"\r": "\n",
"<SPACE>": " ",
"<TAB>": "\t",
"\t": "\t",
}


class SearchMode(Enum):
NO_SEARCH = 0
Expand Down Expand Up @@ -1081,6 +1096,11 @@ def process_simple_keypress(self, e: str):
pass # ignore events
elif e == "<SPACE>":
self.add_normal_character(" ")
elif e in ("<TAB>", "\t"):
# add_normal_character ignores control characters, so expand tabs
# here rather than losing the indentation they stand for
for _ in range(self.config.tab_length):
self.add_normal_character(" ")
else:
self.add_normal_character(e)

Expand Down Expand Up @@ -2213,6 +2233,10 @@ def compress_paste_event(paste_event):
if not all(paste_event.events[0] == e for e in paste_event.events):
return None
event = paste_event.events[0]
# A run of identical content keys is ordinary pasted text - indentation or
# blank lines - rather than a held down key, so it must not be collapsed.
if is_simple_event(event):
return None
# basically "is there a special curtsies names for this key?"
if len(event) > 1:
return event
Expand All @@ -2225,11 +2249,9 @@ def just_simple_events(event_list: Iterable[str | events.Event]) -> list[str]:
for e in event_list:
if isinstance(e, events.Event):
continue # ignore events
# '\n' necessary for pastes
elif e in ("<Ctrl-j>", "<Ctrl-m>", "<PADENTER>", "\n", "\r"):
simple_events.append("\n")
elif e == "<SPACE>":
simple_events.append(" ")
# content keys carry the text of a paste, including its indentation
elif e in CONTENT_KEYS:
simple_events.append(CONTENT_KEYS[e])
elif len(e) > 1:
continue # get rid of <Ctrl-a> etc.
else:
Expand All @@ -2240,7 +2262,4 @@ def just_simple_events(event_list: Iterable[str | events.Event]) -> list[str]:
def is_simple_event(e: str | events.Event) -> bool:
if isinstance(e, events.Event):
return False
return (
e in ("<Ctrl-j>", "<Ctrl-m>", "<PADENTER>", "\n", "\r", "<SPACE>")
or len(e) <= 1
)
return e in CONTENT_KEYS or len(e) <= 1
45 changes: 45 additions & 0 deletions bpython/test/test_curtsies_repl.py
Original file line number Diff line number Diff line change
Expand Up @@ -483,6 +483,51 @@ def test_control_events_in_large_paste(self):
"a" + "e" * curtsiesrepl.MAX_EVENTS_POSSIBLY_NOT_PASTE,
)

def paste(self, source):
"""Paste source as the key events curtsies would report for it"""
p = events.PasteEvent()
p.events = [
{" ": "<SPACE>", "\t": "<TAB>", "\n": "<Ctrl-j>"}.get(c, c)
for c in source
]
self.repl.process_event(p)
return [
line
for line, line_type in self.repl.all_logical_lines
if line_type == LineType.INPUT
]

def test_repeated_content_keys_are_not_compressed(self):
"""A run of spaces is indentation, not a key being held down"""
for key in ("<SPACE>", "<TAB>", "<Ctrl-j>"):
with self.subTest(key=key):
p = events.PasteEvent()
p.events = [key] * 10
self.assertIsNone(curtsiesrepl.compress_paste_event(p))

def test_repeated_control_keys_are_still_compressed(self):
p = events.PasteEvent()
p.events = ["<Ctrl-a>"] * 10
self.assertEqual(curtsiesrepl.compress_paste_event(p), "<Ctrl-a>")

def test_deep_indentation_is_preserved(self):
"""Indentation long enough to fill a read must survive (#1057)"""
self.assertEqual(self.paste(" " * 12 + "x"), [])
self.assertEqual(self.repl.current_line, " " * 12 + "x")

def test_space_indented_paste(self):
self.assertEqual(
self.paste("for i in (1, 2):\n x = i\n y = x\n"),
["for i in (1, 2):", " x = i", " y = x"],
)

def test_tab_indented_paste(self):
"""Tabs carry indentation and must not be dropped (#1057)"""
self.assertEqual(
self.paste("for i in (1, 2):\n\tx = i\n\ty = x\n"),
["for i in (1, 2):", " x = i", " y = x"],
)


if __name__ == "__main__":
unittest.main()
73 changes: 73 additions & 0 deletions bpython/test/test_preprocess.py
Original file line number Diff line number Diff line change
Expand Up @@ -105,3 +105,76 @@ def test_blank_trailing_line(self):

def test_tabs(self):
self.assertIndented(original.tabs)


class TestSeparateBlocks(unittest.TestCase):
"""Pasted code has no blank line after a block, so one has to be added
for the statement that follows it to be run separately (#1057)"""

def test_statement_after_block(self):
self.assertEqual(
preproc("for i in r:\n f(i)\nprint(1)\n"),
"for i in r:\n f(i)\n\nprint(1)\n",
)

def test_statement_after_nested_blocks(self):
self.assertEqual(
preproc("for i in r:\n if i:\n f(i)\nprint(1)\n"),
"for i in r:\n if i:\n f(i)\n\nprint(1)\n",
)

def test_consecutive_blocks(self):
self.assertEqual(
preproc("if a:\n f()\nwhile b:\n g()\n"),
"if a:\n f()\n\nwhile b:\n g()\n",
)

def test_clauses_continuing_a_block_are_left_alone(self):
for source in (
"if a:\n f()\nelif b:\n g()\nelse:\n h()\n",
"try:\n f()\nexcept E:\n g()\nfinally:\n h()\n",
"for i in r:\n f(i)\nelse:\n g()\n",
):
with self.subTest(source=source):
self.assertEqual(preproc(source), source)

def test_unindented_lines_inside_brackets_are_left_alone(self):
# an unclosed bracket keeps the statement going, so these lines are
# continuations rather than new statements
for source in (
"total = (\n1 +\n2\n)\n",
"items = [\n'a',\n'b',\n]\n",
"f(\nx,\n)\n",
):
with self.subTest(source=source):
self.assertEqual(preproc(source), source)

def test_consecutive_simple_statements_are_left_alone(self):
source = "a = 1\nb = 2\nprint(a + b)\n"
self.assertEqual(preproc(source), source)

def test_existing_blank_line_is_not_doubled(self):
source = "for i in r:\n f(i)\n\nprint(1)\n"
self.assertEqual(preproc(source), source)

def test_block_after_statement(self):
source = "a = 1\nfor i in r:\n f(i)\n"
self.assertEqual(preproc(source), source)

def test_comment_above_a_continuing_clause_is_left_alone(self):
source = "if a:\n f()\n# why not\nelse:\n g()\n"
self.assertEqual(preproc(source), source)

def test_decorator_is_not_separated_from_its_definition(self):
self.assertEqual(
preproc("@deco\ndef f():\n pass\nprint(1)\n"),
"@deco\ndef f():\n pass\n\nprint(1)\n",
)

def test_lines_inside_a_string_are_left_alone(self):
source = 's = """\nabc\ndef\n"""\nprint(s)\n'
self.assertEqual(preproc(source), source)

def test_backslash_continuation_is_left_alone(self):
source = "a = 1 + \\\n 2\nprint(a)\n"
self.assertEqual(preproc(source), source)
Loading