posix: fnmatch: fix bugs in fnmatch(), see issue #55186

all tests in tests/posix/c_lib_ext/src/fnmatch.c seem to pass
except one (which is still commented)

Signed-off-by: Nimish Tambe <nimisht@gmail.com>

lib/posix/options/fnmatch.c

@@ -84,13 +84,16 @@ static const char *rangematch(const char *pattern, int test, int flags)
 	for (need = true, ok = false, c = FOLDCASE(*pattern++, flags); c != ']' || need;
 	     c = FOLDCASE(*pattern++, flags)) {
 		need = false;
-		if (c == '/') {
+
+		if (c == '/' && (flags & FNM_PATHNAME)) {
 			return (void *)-1;
 		}
 
 		if (c == '\\' && !(flags & FNM_NOESCAPE)) {
+			if (*pattern != ']' && *pattern != EOS) {
 				c = FOLDCASE(*pattern++, flags);
 			}
+		}
 
 		if (c == EOS) {
 			return NULL;
@@ -221,10 +224,21 @@ static int fnmatchx(const char *pattern, const char *string, int flags, size_t r
 				return FNM_NOMATCH;
 			}
 
+			if (*string == '.' && (flags & FNM_PERIOD) &&
+			    (string == stringstart ||
+			     ((flags & FNM_PATHNAME) && *(string - 1) == '/'))) {
+				return FNM_NOMATCH;
+			}
+
 			r = rangematch(pattern, FOLDCASE(*string, flags), flags);
+
 			if (r == NULL) {
+				if (FOLDCASE('[', flags) != FOLDCASE(*string, flags)) {
 					return FNM_NOMATCH;
 				}
+				++string;
+				break;
+			}
 
 			if (r == (void *)-1) {
 				if (*string != '[') {

tests/posix/c_lib_ext/src/fnmatch.c

@@ -30,14 +30,14 @@ ZTEST(posix_c_lib_ext, test_fnmatch)
 	zassert_equal(fnmatch("a*.c", "a/x.c", FNM_PATHNAME), FNM_NOMATCH);
 	zassert_ok(fnmatch("*/foo", "/foo", FNM_PATHNAME));
 	zassert_ok(fnmatch("-O[01]", "-O1", 0));
-	/* zassert_ok(fnmatch("[[?*\\]", "\\", 0)); */
-	/* zassert_ok(fnmatch("[]?*\\]", "]", 0)); */
+	zassert_ok(fnmatch("[[?*\\]", "\\", 0));
+	zassert_ok(fnmatch("[]?*\\]", "]", 0));
 	zassert_ok(fnmatch("[!]a-]", "b", 0));
 	zassert_ok(fnmatch("[]-_]", "^", 0));
 	zassert_ok(fnmatch("[!]-_]", "X", 0));
 	zassert_equal(fnmatch("??", "-", 0), FNM_NOMATCH);
 	zassert_equal(fnmatch("*LIB*", "lib", FNM_PERIOD), FNM_NOMATCH);
-	/* zassert_ok(fnmatch("a[/]b", "a/b", 0)); */
+	zassert_ok(fnmatch("a[/]b", "a/b", 0));
 	zassert_equal(fnmatch("a[/]b", "a/b", FNM_PATHNAME), FNM_NOMATCH);
 	zassert_ok(fnmatch("[a-z]/[a-z]", "a/b", 0));
 	zassert_equal(fnmatch("*", "a/b", FNM_PATHNAME), FNM_NOMATCH);
@@ -64,8 +64,8 @@ ZTEST(posix_c_lib_ext, test_fnmatch)
 	zassert_equal(fnmatch("a/?b", "a/.b", FNM_PATHNAME | FNM_PERIOD), FNM_NOMATCH);
 	zassert_equal(fnmatch("*a/b", ".a/b", FNM_PATHNAME | FNM_PERIOD), FNM_NOMATCH);
 	zassert_equal(fnmatch("a/*b", "a/.b", FNM_PATHNAME | FNM_PERIOD), FNM_NOMATCH);
-	/* zassert_equal(fnmatch("[.]a/b", ".a/b", FNM_PATHNAME | FNM_PERIOD), FNM_NOMATCH); */
-	/* zassert_equal(fnmatch("a/[.]b", "a/.b", FNM_PATHNAME | FNM_PERIOD), FNM_NOMATCH); */
+	zassert_equal(fnmatch("[.]a/b", ".a/b", FNM_PATHNAME | FNM_PERIOD), FNM_NOMATCH);
+	zassert_equal(fnmatch("a/[.]b", "a/.b", FNM_PATHNAME | FNM_PERIOD), FNM_NOMATCH);
 	zassert_ok(fnmatch("*/?", "a/b", FNM_PATHNAME | FNM_PERIOD));
 	zassert_ok(fnmatch("?/*", "a/b", FNM_PATHNAME | FNM_PERIOD));
 	zassert_ok(fnmatch(".*/?", ".a/b", FNM_PATHNAME | FNM_PERIOD));