Refactor shared memory allocation for semaphores

Before e256266 spinlocks were
implemented using semaphores on some platforms. Hence
PGReserveSemaphores() was required to be called before
SpinlockSemaInit().  SpinlockSemaInit() was required to be called before
InitShmemAllocation() since the latter initialized a spinlock to
synchronize shared memory allocations.  A cyclic dependency existed
because PGReserveSemaphores() uses shared memory for semaphores on some
platforms. It was broken by letting PGReserveSemaphores() allocate
shared memory without a spinlock using ShmemAllocUnlocked().  After
e256266, spinlocks are not implemented
using semaphores at all. This allows us to eliminate the need to
estimate and allocate shared memory required by semaphores specially and
treat them similar to the other shared memory structures.

Since the semaphores are used only in PGPROC array, it makes more sense
to estimate size of memory required by semaphores to go along with the
estimation of memory required for that array in ProcGlobalShmemSize().
Similarly, it makes sense to have the actual shared memory allocations
for both to be collocated in InitProcGlobal().

With that change CalculateShmemSize() does not need to know about the
number of semaphores to be allocated, eliminating other asymmetry.
InitializeShmemGUCs() which used to get the number of semaphores through
CalculateShmemSize() now fetches that value separately.

Author: Ashutosh Bapat <[email protected]>
Discussion: https://www.postgresql.org/message-id/CAExHW5seSZpPx-znjidVZNzdagGHOk06F+Ds88MpPUbxd1kTaA@mail.gmail.com

Author: ashutosh-bapat

src/backend/port/posix_sema.c

@@ -215,12 +215,8 @@ PGReserveSemaphores(int maxSemas)
 		elog(PANIC, "out of memory");
 #else
 
-	/*
-	 * We must use ShmemAllocUnlocked(), since the spinlock protecting
-	 * ShmemAlloc() won't be ready yet.
-	 */
 	sharedSemas = (PGSemaphore)
-		ShmemAllocUnlocked(PGSemaphoreShmemSize(maxSemas));
+		ShmemAlloc(PGSemaphoreShmemSize(maxSemas));
 #endif
 
 	numSems = 0;

src/backend/port/sysv_sema.c

@@ -343,12 +343,8 @@ PGReserveSemaphores(int maxSemas)
 				 errmsg("could not stat data directory \"%s\": %m",
 						DataDir)));
 
-	/*
-	 * We must use ShmemAllocUnlocked(), since the spinlock protecting
-	 * ShmemAlloc() won't be ready yet.
-	 */
 	sharedSemas = (PGSemaphore)
-		ShmemAllocUnlocked(PGSemaphoreShmemSize(maxSemas));
+		ShmemAlloc(PGSemaphoreShmemSize(maxSemas));
 	numSharedSemas = 0;
 	maxSharedSemas = maxSemas;
 

src/backend/storage/ipc/ipci.c

@@ -86,17 +86,9 @@ RequestAddinShmemSpace(Size size)
  * required.
  */
 Size
-CalculateShmemSize(int *num_semaphores)
+CalculateShmemSize(void)
 {
 	Size		size;
-	int			numSemas;
-
-	/* Compute number of semaphores we'll need */
-	numSemas = ProcGlobalSemas();
-
-	/* Return the number of semaphores if requested by the caller */
-	if (num_semaphores)
-		*num_semaphores = numSemas;
 
 	/*
 	 * Size of the Postgres shared-memory block is estimated via moderately-
@@ -108,7 +100,6 @@ CalculateShmemSize(int *num_semaphores)
 	 * during the actual allocation phase.
 	 */
 	size = 100000;
-	size = add_size(size, PGSemaphoreShmemSize(numSemas));
 	size = add_size(size, hash_estimate_size(SHMEM_INDEX_SIZE,
 											 sizeof(ShmemIndexEnt)));
 	size = add_size(size, dsm_estimate_size());
@@ -202,12 +193,11 @@ CreateSharedMemoryAndSemaphores(void)
 	PGShmemHeader *shim;
 	PGShmemHeader *seghdr;
 	Size		size;
-	int			numSemas;
 
 	Assert(!IsUnderPostmaster);
 
 	/* Compute the size of the shared-memory block */
-	size = CalculateShmemSize(&numSemas);
+	size = CalculateShmemSize();
 	elog(DEBUG3, "invoking IpcMemoryCreate(size=%zu)", size);
 
 	/*
@@ -224,16 +214,6 @@ CreateSharedMemoryAndSemaphores(void)
 
 	InitShmemAccess(seghdr);
 
-	/*
-	 * Create semaphores. This is done here because of a now-non-existent
-	 * dependency between spinlocks, which were required for shared memory
-	 * allocation, and semaphores, which were allocated in the shared memory
-	 * on some platforms. Ideally it should be done in
-	 * CreateOrAttachShmemStructs() where we allocate other shared memory
-	 * structures.
-	 */
-	PGReserveSemaphores(numSemas);
-
 	/*
 	 * Set up shared memory allocation mechanism
 	 */
@@ -363,12 +343,11 @@ InitializeShmemGUCs(void)
 	Size		size_b;
 	Size		size_mb;
 	Size		hp_size;
-	int			num_semas;
 
 	/*
 	 * Calculate the shared memory size and round up to the nearest megabyte.
 	 */
-	size_b = CalculateShmemSize(&num_semas);
+	size_b = CalculateShmemSize();
 	size_mb = add_size(size_b, (1024 * 1024) - 1) / (1024 * 1024);
 	sprintf(buf, "%zu", size_mb);
 	SetConfigOption("shared_memory_size", buf,
@@ -388,6 +367,6 @@ InitializeShmemGUCs(void)
 						PGC_INTERNAL, PGC_S_DYNAMIC_DEFAULT);
 	}
 
-	sprintf(buf, "%d", num_semas);
+	sprintf(buf, "%d", ProcGlobalSemas());
 	SetConfigOption("num_os_semaphores", buf, PGC_INTERNAL, PGC_S_DYNAMIC_DEFAULT);
 }

src/backend/storage/lmgr/proc.c

@@ -144,6 +144,7 @@ ProcGlobalShmemSize(void)
 	size = add_size(size, sizeof(PROC_HDR));
 	size = add_size(size, sizeof(slock_t));
 
+	size = add_size(size, PGSemaphoreShmemSize(ProcGlobalSemas()));
 	size = add_size(size, PGProcShmemSize());
 	size = add_size(size, FastPathLockShmemSize());
 
@@ -286,6 +287,9 @@ InitProcGlobal(void)
 	/* For asserts checking we did not overflow. */
 	fpEndPtr = fpPtr + requestSize;
 
+	/* Reserve space for semaphores. */
+	PGReserveSemaphores(ProcGlobalSemas());
+
 	for (i = 0; i < TotalProcs; i++)
 	{
 		PGPROC	   *proc = &procs[i];

src/include/storage/ipc.h

@@ -77,7 +77,7 @@ extern void check_on_shmem_exit_lists_are_empty(void);
 /* ipci.c */
 extern PGDLLIMPORT shmem_startup_hook_type shmem_startup_hook;
 
-extern Size CalculateShmemSize(int *num_semaphores);
+extern Size CalculateShmemSize(void);
 extern void CreateSharedMemoryAndSemaphores(void);
 #ifdef EXEC_BACKEND
 extern void AttachSharedMemoryStructs(void);