posix: shm: fix fd leak on error paths and protect shm_list with mutex

In subsys/portability/posix/options/shm.c, shm_open() reserves a file
descriptor using zvfs_reserve_fd() before checking whether the target
shm object exists or is unlinked.

On error conditions where the shared memory object does not exist (when
O_CREAT is omitted) or where the object has already been unlinked,
shm_open() returned -1 with errno set to ENOENT or EACCES without
calling zvfs_free_fd(fd). This caused a permanent file descriptor leak
in the global ZVFS descriptor table (CONFIG_ZVFS_OPEN_MAX).

Additionally, the global doubly-linked list shm_list and object
reference counters were manipulated without synchronization across
concurrent shm_open(), shm_unlink(), and shm_close() calls.

Fix both issues by:
1. Releasing reserved file descriptors via zvfs_free_fd(fd) on all error
   return paths in shm_open().
2. Protecting shm_list traversal, insertion, removal, and reference
   counting with a dedicated mutex (shm_lock).
3. Adding a test in xsi_realtime verifying that repeated failed open
   attempts do not leak file descriptors.

Signed-off-by: harshit kudhial <harshitkudhial@gmail.com>

Author: harshitnub077

subsys/portability/posix/options/shm.c

@@ -25,6 +25,7 @@
 static const struct fd_op_vtable shm_vtable;
 
 static sys_dlist_t shm_list = SYS_DLIST_STATIC_INIT(&shm_list);
+static K_MUTEX_DEFINE(shm_lock);
 
 struct shm_obj {
 	uint8_t *mem;
@@ -239,10 +240,12 @@ static int shm_close(void *obj)
 {
 	struct shm_obj *shm = obj;
 
+	k_mutex_lock(&shm_lock, K_FOREVER);
 	shm->refs -= (shm->refs > 0) ? 1 : 0;
 	if (shm->unlinked && (shm->refs == 0)) {
 		shm_obj_remove(shm);
 	}
+	k_mutex_unlock(&shm_lock);
 
 	return 0;
 }
@@ -336,15 +339,21 @@ int shm_open(const char *name, int oflag, mode_t mode)
 	}
 
 	key = hash32(name, name_len);
+
+	k_mutex_lock(&shm_lock, K_FOREVER);
+
 	shm = shm_obj_find(key);
 	if ((shm != NULL) && shm->unlinked) {
 		/* we cannot open a shm object that has already been unlinked */
+		k_mutex_unlock(&shm_lock);
+		zvfs_free_fd(fd);
 		errno = EACCES;
 		return -1;
 	}
 
 	if (creat) {
 		if ((shm != NULL) && excl) {
+			k_mutex_unlock(&shm_lock);
 			zvfs_free_fd(fd);
 			errno = EEXIST;
 			return -1;
@@ -353,6 +362,7 @@ int shm_open(const char *name, int oflag, mode_t mode)
 		if (shm == NULL) {
 			shm = k_calloc(1, sizeof(*shm));
 			if (shm == NULL) {
+				k_mutex_unlock(&shm_lock);
 				zvfs_free_fd(fd);
 				errno = ENOSPC;
 				return -1;
@@ -362,11 +372,15 @@ int shm_open(const char *name, int oflag, mode_t mode)
 			shm_obj_add(shm);
 		}
 	} else if (shm == NULL) {
+		k_mutex_unlock(&shm_lock);
+		zvfs_free_fd(fd);
 		errno = ENOENT;
 		return -1;
 	}
 
 	++shm->refs;
+	k_mutex_unlock(&shm_lock);
+
 	zvfs_finalize_typed_fd(fd, shm, &shm_vtable, ZVFS_MODE_IFSHM);
 
 	return fd;
@@ -384,8 +398,12 @@ int shm_unlink(const char *name)
 	}
 
 	key = hash32(name, name_len);
+
+	k_mutex_lock(&shm_lock, K_FOREVER);
+
 	shm = shm_obj_find(key);
 	if ((shm == NULL) || shm->unlinked) {
+		k_mutex_unlock(&shm_lock);
 		errno = ENOENT;
 		return -1;
 	}
@@ -395,5 +413,7 @@ int shm_unlink(const char *name)
 		shm_obj_remove(shm);
 	}
 
+	k_mutex_unlock(&shm_lock);
+
 	return 0;
 }

tests/subsys/portability/posix/xsi_realtime/src/shm.c

@@ -200,3 +200,28 @@ ZTEST(xsi_realtime, test_shm_mmap)
 
 	zassert_ok(shm_unlink(VALID_SHM_PATH));
 }
+
+ZTEST(xsi_realtime, test_shm_open_fd_leak)
+{
+	int fd[N];
+
+	/* Repeatedly attempt to open nonexistent shm without O_CREAT */
+	for (size_t i = 0; i < N * 2; ++i) {
+		int bad_fd = shm_open("/nonexistent_shm", OPEN_FLAGS, VALID_MODE);
+
+		zassert_equal(bad_fd, -1, "shm_open should fail for nonexistent shm");
+		zassert_equal(errno, ENOENT, "errno should be ENOENT");
+	}
+
+	/* Verify all N file descriptor slots remain available */
+	for (size_t i = 0; i < N; ++i) {
+		fd[i] = shm_open(VALID_SHM_PATH, i == 0 ? CREATE_FLAGS : OPEN_FLAGS, VALID_MODE);
+		zassert_true(fd[i] >= 0, "shm_open(%s) failed after failed opens: fd=%d, errno=%d",
+			     VALID_SHM_PATH, fd[i], errno);
+	}
+
+	zassert_ok(shm_unlink(VALID_SHM_PATH));
+	for (size_t i = N; i > 0; --i) {
+		zassert_ok(close(fd[i - 1]));
+	}
+}