feat: Add content aware line chunking (#2707)

tests/data/0_to_99.py

@@ -0,0 +1,100 @@
+x0 = 0
+x1 = 1
+x2 = 2
+x3 = 3
+x4 = 4
+x5 = 5
+x6 = 6
+x7 = 7
+x8 = 8
+x9 = 9
+x10 = 10
+x11 = 11
+x12 = 12
+x13 = 13
+x14 = 14
+x15 = 15
+x16 = 16
+x17 = 17
+x18 = 18
+x19 = 19
+x20 = 20
+x21 = 21
+x22 = 22
+x23 = 23
+x24 = 24
+x25 = 25
+x26 = 26
+x27 = 27
+x28 = 28
+x29 = 29
+x30 = 30
+x31 = 31
+x32 = 32
+x33 = 33
+x34 = 34
+x35 = 35
+x36 = 36
+x37 = 37
+x38 = 38
+x39 = 39
+x40 = 40
+x41 = 41
+x42 = 42
+x43 = 43
+x44 = 44
+x45 = 45
+x46 = 46
+x47 = 47
+x48 = 48
+x49 = 49
+x50 = 50
+x51 = 51
+x52 = 52
+x53 = 53
+x54 = 54
+x55 = 55
+x56 = 56
+x57 = 57
+x58 = 58
+x59 = 59
+x60 = 60
+x61 = 61
+x62 = 62
+x63 = 63
+x64 = 64
+x65 = 65
+x66 = 66
+x67 = 67
+x68 = 68
+x69 = 69
+x70 = 70
+x71 = 71
+x72 = 72
+x73 = 73
+x74 = 74
+x75 = 75
+x76 = 76
+x77 = 77
+x78 = 78
+x79 = 79
+x80 = 80
+x81 = 81
+x82 = 82
+x83 = 83
+x84 = 84
+x85 = 85
+x86 = 86
+x87 = 87
+x88 = 88
+x89 = 89
+x90 = 90
+x91 = 91
+x92 = 92
+x93 = 93
+x94 = 94
+x95 = 95
+x96 = 96
+x97 = 97
+x98 = 98
+x99 = 99

tests/data/lines_1_to_100.txt

@@ -1,100 +0,0 @@
-Line 1
-Line 2
-Line 3
-Line 4
-Line 5
-Line 6
-Line 7
-Line 8
-Line 9
-Line 10
-Line 11
-Line 12
-Line 13
-Line 14
-Line 15
-Line 16
-Line 17
-Line 18
-Line 19
-Line 20
-Line 21
-Line 22
-Line 23
-Line 24
-Line 25
-Line 26
-Line 27
-Line 28
-Line 29
-Line 30
-Line 31
-Line 32
-Line 33
-Line 34
-Line 35
-Line 36
-Line 37
-Line 38
-Line 39
-Line 40
-Line 41
-Line 42
-Line 43
-Line 44
-Line 45
-Line 46
-Line 47
-Line 48
-Line 49
-Line 50
-Line 51
-Line 52
-Line 53
-Line 54
-Line 55
-Line 56
-Line 57
-Line 58
-Line 59
-Line 60
-Line 61
-Line 62
-Line 63
-Line 64
-Line 65
-Line 66
-Line 67
-Line 68
-Line 69
-Line 70
-Line 71
-Line 72
-Line 73
-Line 74
-Line 75
-Line 76
-Line 77
-Line 78
-Line 79
-Line 80
-Line 81
-Line 82
-Line 83
-Line 84
-Line 85
-Line 86
-Line 87
-Line 88
-Line 89
-Line 90
-Line 91
-Line 92
-Line 93
-Line 94
-Line 95
-Line 96
-Line 97
-Line 98
-Line 99
-Line 100