Skip to content

Match newlines in gitignore globstar path segments #139

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
cpburnz merged 1 commit into from
Sep 10, 2026
Merged

Match newlines in gitignore globstar path segments #139

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGES_1.in.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ New features:

Bug fixes:

- Match newline characters in directory names when expanding gitignore globstars, including implicit leading globstars.

- `Pull #123`_: Ignore invalid gitignore bracket ranges for `GitIgnoreSpec`.
- `Pull #128`_: Support POSIX character classes (e.g. `[[:alpha:]]`) in gitignore bracket expressions.
- `Issue #129`_ / `Pull #132`_: Fix GitIgnoreSpec re-including files under an excluded directory
Expand Down
8 changes: 4 additions & 4 deletions pathspec/patterns/gitignore/basic.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ def __normalize_segments(
return (None, '/')
else:
# The pattern "**" will match every path. Special case this pattern.
return (None, '.')
return (None, '(?s:.)')

elif (
seg_count == 2
Expand All @@ -107,7 +107,7 @@ def __normalize_segments(
):
# The pattern "*" will be normalized to "**/*" and will match every
# path. Special case this pattern for efficiency.
return (None, '.')
return (None, '(?s:.)')

elif (
seg_count == 3
Expand Down Expand Up @@ -265,12 +265,12 @@ def __translate_segments(cls, pattern_segs: list[str]) -> list[str]:
# match any leading path segments.
# - NOTICE: '(?:^|/)' benchmarks slower using p15 (sm=0.9382,
# hs=0.9966, re2=0.9337).
out_parts.append('^(?:.+/)?')
out_parts.append('^(?:(?s:.)+/)?')

elif i < end:
# A pattern with inner double-asterisks ('**') will match multiple (or
# zero) inner path segments.
out_parts.append('(?:/.+)?')
out_parts.append('(?:/(?s:.)+)?')
need_slash = True

else:
Expand Down
6 changes: 3 additions & 3 deletions pathspec/patterns/gitignore/spec.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
This regular expression matches the optional directory marker and sub-path.
"""

_MATCH_ALL = f'^(?:.+/)?[^/]+{_DIR_MARK_OPT}'
_MATCH_ALL = f'^(?:(?s:.)+/)?[^/]+{_DIR_MARK_OPT}'
"""
This regular expression matches every path. It is the expansion of the patterns
"*" and "**" (i.e., "**/{any name}"), and it has to capture the directory marker
Expand Down Expand Up @@ -306,12 +306,12 @@ def __translate_segments(
if i == 0:
# A normalized pattern beginning with double-asterisks ('**') will
# match any leading path segments.
out_parts.append('^(?:.+/)?')
out_parts.append('^(?:(?s:.)+/)?')

elif i < end:
# A pattern with inner double-asterisks ('**') will match multiple (or
# zero) inner path segments.
out_parts.append('(?:/.+)?')
out_parts.append('(?:/(?s:.)+)?')
need_slash = True

else:
Expand Down
95 changes: 53 additions & 42 deletions tests/test_03_gitignore_basic.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ def test_01_absolute_root_1(self):
# GitIgnoreSpecPattern.
regex, include = GitIgnoreBasicPattern.pattern_to_regex('/')
self.assertTrue(include)
self.assertEqual(regex, '.')
self.assertEqual(regex, '(?s:.)')

def test_01_absolute_root_2_double_asterisk(self):
"""
Expand Down Expand Up @@ -138,7 +138,7 @@ def test_01_relative(self):
"""
regex, include = GitIgnoreBasicPattern.pattern_to_regex('spam')
self.assertTrue(include)
self.assertEqual(regex, f'^(?:.+/)?spam{_DIR_OPT}')
self.assertEqual(regex, f'^(?:(?s:.)+/)?spam{_DIR_OPT}')

pattern = GitIgnoreBasicPattern(re.compile(regex), include)
results = set(filter(pattern.match_file, [
Expand Down Expand Up @@ -202,7 +202,7 @@ def test_02_ignore(self):
"""
regex, include = GitIgnoreBasicPattern.pattern_to_regex('!temp')
self.assertIs(include, False)
self.assertEqual(regex, f'^(?:.+/)?temp{_DIR_OPT}')
self.assertEqual(regex, f'^(?:(?s:.)+/)?temp{_DIR_OPT}')

# NOTE: The pattern match is backwards because the pattern itself
# does not consider the include attribute.
Expand Down Expand Up @@ -255,7 +255,7 @@ def test_03_inner_double_asterisk(self):
"""
regex, include = GitIgnoreBasicPattern.pattern_to_regex('left/**/right')
self.assertTrue(include)
self.assertEqual(regex, f'^left(?:/.+)?/right{_DIR_OPT}')
self.assertEqual(regex, f'^left(?:/(?s:.)+)?/right{_DIR_OPT}')

pattern = GitIgnoreBasicPattern(re.compile(regex), include)
results = set(filter(pattern.match_file, [
Expand All @@ -278,7 +278,7 @@ def test_03_only_double_asterisk(self):
"""
regex, include = GitIgnoreBasicPattern.pattern_to_regex('**')
self.assertTrue(include)
self.assertEqual(regex, '.')
self.assertEqual(regex, '(?s:.)')

pattern = GitIgnoreBasicPattern(re.compile(regex), include)
results = set(filter(pattern.match_file, [
Expand Down Expand Up @@ -315,7 +315,7 @@ def test_03_parent_double_asterisk(self):
"""
regex, include = GitIgnoreBasicPattern.pattern_to_regex('**/spam')
self.assertTrue(include)
self.assertEqual(regex, f'^(?:.+/)?spam{_DIR_OPT}')
self.assertEqual(regex, f'^(?:(?s:.)+/)?spam{_DIR_OPT}')

pattern = GitIgnoreBasicPattern(re.compile(regex), include)
results = set(filter(pattern.match_file, [
Expand All @@ -335,7 +335,7 @@ def test_03_duplicate_leading_double_asterisk_edge_case(self):
"""
regex, include = GitIgnoreBasicPattern.pattern_to_regex('**')
self.assertTrue(include)
self.assertEqual(regex, '.')
self.assertEqual(regex, '(?s:.)')

equiv_regex, include = GitIgnoreBasicPattern.pattern_to_regex('**/**')
self.assertTrue(include)
Expand All @@ -347,23 +347,23 @@ def test_03_duplicate_leading_double_asterisk_edge_case(self):

regex, include = GitIgnoreBasicPattern.pattern_to_regex('**/api')
self.assertTrue(include)
self.assertEqual(regex, f'^(?:.+/)?api{_DIR_OPT}')
self.assertEqual(regex, f'^(?:(?s:.)+/)?api{_DIR_OPT}')

equiv_regex, include = GitIgnoreBasicPattern.pattern_to_regex('**/**/api')
self.assertTrue(include)
self.assertEqual(equiv_regex, regex)

regex, include = GitIgnoreBasicPattern.pattern_to_regex('**/api/')
self.assertTrue(include)
self.assertEqual(regex, '^(?:.+/)?api/')
self.assertEqual(regex, '^(?:(?s:.)+/)?api/')

equiv_regex, include = GitIgnoreBasicPattern.pattern_to_regex('**/**/api/')
self.assertTrue(include)
self.assertEqual(equiv_regex, regex)

regex, include = GitIgnoreBasicPattern.pattern_to_regex('**/api/**')
self.assertTrue(include)
self.assertEqual(regex, '^(?:.+/)?api/')
self.assertEqual(regex, '^(?:(?s:.)+/)?api/')

equiv_regex, include = GitIgnoreBasicPattern.pattern_to_regex('**/**/api/**/**')
self.assertTrue(include)
Expand Down Expand Up @@ -397,7 +397,7 @@ def test_04_infix_wildcard(self):
"""
regex, include = GitIgnoreBasicPattern.pattern_to_regex('foo-*-bar')
self.assertTrue(include)
self.assertEqual(regex, f'^(?:.+/)?foo\\-[^/]*\\-bar{_DIR_OPT}')
self.assertEqual(regex, f'^(?:(?s:.)+/)?foo\\-[^/]*\\-bar{_DIR_OPT}')

pattern = GitIgnoreBasicPattern(re.compile(regex), include)
results = set(filter(pattern.match_file, [
Expand Down Expand Up @@ -429,7 +429,7 @@ def test_04_postfix_wildcard(self):
"""
regex, include = GitIgnoreBasicPattern.pattern_to_regex('~temp-*')
self.assertTrue(include)
self.assertEqual(regex, f'^(?:.+/)?\\~temp\\-[^/]*{_DIR_OPT}')
self.assertEqual(regex, f'^(?:(?s:.)+/)?\\~temp\\-[^/]*{_DIR_OPT}')

pattern = GitIgnoreBasicPattern(re.compile(regex), include)
results = set(filter(pattern.match_file, [
Expand Down Expand Up @@ -460,7 +460,7 @@ def test_04_prefix_wildcard(self):
"""
regex, include = GitIgnoreBasicPattern.pattern_to_regex('*.py')
self.assertTrue(include)
self.assertEqual(regex, f'^(?:.+/)?[^/]*\\.py{_DIR_OPT}')
self.assertEqual(regex, f'^(?:(?s:.)+/)?[^/]*\\.py{_DIR_OPT}')

pattern = GitIgnoreBasicPattern(re.compile(regex), include)
results = set(filter(pattern.match_file, [
Expand Down Expand Up @@ -492,7 +492,7 @@ def test_05_directory(self):
"""
regex, include = GitIgnoreBasicPattern.pattern_to_regex('dir/')
self.assertTrue(include)
self.assertEqual(regex, '^(?:.+/)?dir/')
self.assertEqual(regex, '^(?:(?s:.)+/)?dir/')

pattern = GitIgnoreBasicPattern(re.compile(regex), include)
results = set(filter(pattern.match_file, [
Expand Down Expand Up @@ -750,7 +750,7 @@ def test_12_asterisk_1_regex(self):
"""
regex, include = GitIgnoreBasicPattern.pattern_to_regex('*')
self.assertTrue(include)
self.assertEqual(regex, '.')
self.assertEqual(regex, '(?s:.)')

def test_12_asterisk_2_regex_equivalent(self):
"""
Expand Down Expand Up @@ -865,7 +865,7 @@ def test_14_issue_81_c(self):
# GitIgnoreSpecPattern should not.
pattern = GitIgnoreBasicPattern('!libfoo/')

self.assertEqual(pattern.regex.pattern, '^(?:.+/)?libfoo/')
self.assertEqual(pattern.regex.pattern, '^(?:(?s:.)+/)?libfoo/')
self.assertIs(pattern.include, False)
self.assertTrue(pattern.match_file('libfoo/__init__.py'))

Expand All @@ -875,7 +875,7 @@ def test_15_issue_93_a_1(self):
"""
pattern = GitIgnoreBasicPattern('foo**')
self.assertIs(pattern.include, True)
self.assertEqual(pattern.regex.pattern, f'^(?:.+/)?foo[^/]*[^/]*{_DIR_OPT}')
self.assertEqual(pattern.regex.pattern, f'^(?:(?s:.)+/)?foo[^/]*[^/]*{_DIR_OPT}')
self.assertTrue(pattern.match_file('foosrodah'))

def test_15_issue_93_a_2(self):
Expand All @@ -894,7 +894,7 @@ def test_15_issue_93_b_1_single(self):
"""
pattern = GitIgnoreBasicPattern(' foo')
self.assertIs(pattern.include, True)
self.assertEqual(pattern.regex.pattern, f'^(?:.+/)?\\ foo{_DIR_OPT}')
self.assertEqual(pattern.regex.pattern, f'^(?:(?s:.)+/)?\\ foo{_DIR_OPT}')
self.assertFalse(pattern.match_file('foo'))
self.assertTrue(pattern.match_file(' foo'))

Expand All @@ -904,7 +904,7 @@ def test_15_issue_93_b_2_double(self):
"""
pattern = GitIgnoreBasicPattern(' foo')
self.assertIs(pattern.include, True)
self.assertEqual(pattern.regex.pattern, f'^(?:.+/)?\\ \\ foo{_DIR_OPT}')
self.assertEqual(pattern.regex.pattern, f'^(?:(?s:.)+/)?\\ \\ foo{_DIR_OPT}')
self.assertFalse(pattern.match_file('foo'))
self.assertFalse(pattern.match_file(' foo'))
self.assertTrue(pattern.match_file(' foo'))
Expand All @@ -914,12 +914,12 @@ def test_15_issue_93_c_1_valid(self):
Test patterns with valid range notation.
"""
for raw_pattern, regex in [
('[!a-z]', f'^(?:.+/)?[^a-z]{_DIR_OPT}'),
('[^a-z]', f'^(?:.+/)?[^a-z]{_DIR_OPT}'),
('[a-z]', f'^(?:.+/)?[a-z]{_DIR_OPT}'),
('a[!a-z]', f'^(?:.+/)?a[^a-z]{_DIR_OPT}'),
('a[^a-z]', f'^(?:.+/)?a[^a-z]{_DIR_OPT}'),
('a[a-z]', f'^(?:.+/)?a[a-z]{_DIR_OPT}'),
('[!a-z]', f'^(?:(?s:.)+/)?[^a-z]{_DIR_OPT}'),
('[^a-z]', f'^(?:(?s:.)+/)?[^a-z]{_DIR_OPT}'),
('[a-z]', f'^(?:(?s:.)+/)?[a-z]{_DIR_OPT}'),
('a[!a-z]', f'^(?:(?s:.)+/)?a[^a-z]{_DIR_OPT}'),
('a[^a-z]', f'^(?:(?s:.)+/)?a[^a-z]{_DIR_OPT}'),
('a[a-z]', f'^(?:(?s:.)+/)?a[a-z]{_DIR_OPT}'),
]:
with self.subTest(f"p={raw_pattern!r}"):
pattern = GitIgnoreBasicPattern(raw_pattern)
Expand All @@ -932,10 +932,10 @@ def test_15_issue_93_c_2_invalid(self):
"""
# The basic pattern treats invalid range notation as a literal.
for raw_pattern, regex in [
('[!]', f'^(?:.+/)?\\[!\\]{_DIR_OPT}'),
('[^]', f'^(?:.+/)?\\[\\^\\]{_DIR_OPT}'),
('a[!]', f'^(?:.+/)?a\\[!\\]{_DIR_OPT}'),
('a[^]', f'^(?:.+/)?a\\[\\^\\]{_DIR_OPT}'),
('[!]', f'^(?:(?s:.)+/)?\\[!\\]{_DIR_OPT}'),
('[^]', f'^(?:(?s:.)+/)?\\[\\^\\]{_DIR_OPT}'),
('a[!]', f'^(?:(?s:.)+/)?a\\[!\\]{_DIR_OPT}'),
('a[^]', f'^(?:(?s:.)+/)?a\\[\\^\\]{_DIR_OPT}'),
]:
with self.subTest(f"p={raw_pattern!r}"):
pattern = GitIgnoreBasicPattern(raw_pattern)
Expand All @@ -956,19 +956,19 @@ def test_15_issue_93_c_3_unclosed(self):
Test patterns with unclosed range notation.
"""
for raw_pattern, regex in [
('[!', f'^(?:.+/)?\\[!{_DIR_OPT}'),
('[', f'^(?:.+/)?\\[{_DIR_OPT}'),
('[-', f'^(?:.+/)?\\[\\-{_DIR_OPT}'),
('[^', f'^(?:.+/)?\\[\\^{_DIR_OPT}'),
('[a', f'^(?:.+/)?\\[a{_DIR_OPT}'),
('[a-', f'^(?:.+/)?\\[a\\-{_DIR_OPT}'),
('[a-z', f'^(?:.+/)?\\[a\\-z{_DIR_OPT}'),
('a[!', f'^(?:.+/)?a\\[!{_DIR_OPT}'),
('a[', f'^(?:.+/)?a\\[{_DIR_OPT}'),
('a[-', f'^(?:.+/)?a\\[\\-{_DIR_OPT}'),
('a[^', f'^(?:.+/)?a\\[\\^{_DIR_OPT}'),
('a[a-', f'^(?:.+/)?a\\[a\\-{_DIR_OPT}'),
('a[a-z', f'^(?:.+/)?a\\[a\\-z{_DIR_OPT}'),
('[!', f'^(?:(?s:.)+/)?\\[!{_DIR_OPT}'),
('[', f'^(?:(?s:.)+/)?\\[{_DIR_OPT}'),
('[-', f'^(?:(?s:.)+/)?\\[\\-{_DIR_OPT}'),
('[^', f'^(?:(?s:.)+/)?\\[\\^{_DIR_OPT}'),
('[a', f'^(?:(?s:.)+/)?\\[a{_DIR_OPT}'),
('[a-', f'^(?:(?s:.)+/)?\\[a\\-{_DIR_OPT}'),
('[a-z', f'^(?:(?s:.)+/)?\\[a\\-z{_DIR_OPT}'),
('a[!', f'^(?:(?s:.)+/)?a\\[!{_DIR_OPT}'),
('a[', f'^(?:(?s:.)+/)?a\\[{_DIR_OPT}'),
('a[-', f'^(?:(?s:.)+/)?a\\[\\-{_DIR_OPT}'),
('a[^', f'^(?:(?s:.)+/)?a\\[\\^{_DIR_OPT}'),
('a[a-', f'^(?:(?s:.)+/)?a\\[a\\-{_DIR_OPT}'),
('a[a-z', f'^(?:(?s:.)+/)?a\\[a\\-z{_DIR_OPT}'),
]:
with self.subTest(f"p={raw_pattern!r}"):
pattern = GitIgnoreBasicPattern(raw_pattern)
Expand All @@ -982,3 +982,14 @@ def test_16_repr_str(self):
pattern = GitIgnoreBasicPattern('*.py')
self.assertEqual(repr(pattern), "GitIgnoreBasicPattern(pattern='*.py', include=True)")
self.assertEqual(str(pattern), '*.py')

def test_globstars_match_newlines(self):
for pattern, path in [
("target", "line\nbreak/target"),
("**/target", "line\nbreak/target"),
("root/**/target", "root/line\nbreak/target"),
("**", "\n"),
("*", "\n"),
]:
with self.subTest(pattern=pattern, path=path):
self.assertIsNotNone(GitIgnoreBasicPattern(pattern).match_file(path))
Loading