[CF 5805] v2 - Improve array-element-test estimation when no array elements qualify as common

This branch was automatically generated by a robot using patches from an
email thread registered at:

https://commitfest.postgresql.org/patch/5805

The branch will be overwritten each time a new patch version is posted to
the thread, and also periodically to check for bitrot caused by changes
on the master branch.

Patch(es): https://www.postgresql.org/message-id/[email protected]
Author(s): Tom Lane

Author: Commitfest Bot

contrib/intarray/_int_selfuncs.c

@@ -210,8 +210,8 @@ _int_matchsel(PG_FUNCTION_ARGS)
 			 */
 			if (sslot.nnumbers == sslot.nvalues + 3)
 			{
-				/* Grab the lowest frequency. */
-				minfreq = sslot.numbers[sslot.nnumbers - (sslot.nnumbers - sslot.nvalues)];
+				/* Grab the minimal MCE frequency. */
+				minfreq = sslot.numbers[sslot.nvalues];
 
 				mcelems = sslot.values;
 				mcefreqs = sslot.numbers;
@@ -270,9 +270,9 @@ int_query_opr_selec(ITEM *item, Datum *mcelems, float4 *mcefreqs,
 		{
 			/*
 			 * The element is not in MCELEM.  Punt, but assume that the
-			 * selectivity cannot be more than minfreq / 2.
+			 * selectivity cannot be more than minfreq.
 			 */
-			selec = Min(DEFAULT_EQ_SEL, minfreq / 2);
+			selec = Min(DEFAULT_EQ_SEL, minfreq);
 		}
 	}
 	else if (item->type == OPR)

src/backend/commands/analyze.c

@@ -1711,10 +1711,9 @@ update_attstats(Oid relid, bool inh, int natts, VacAttrStats **vacattrstats)
 		i = Anum_pg_statistic_stanumbers1 - 1;
 		for (k = 0; k < STATISTIC_NUM_SLOTS; k++)
 		{
-			int			nnum = stats->numnumbers[k];
-
-			if (nnum > 0)
+			if (stats->stanumbers[k] != NULL)
 			{
+				int			nnum = stats->numnumbers[k];
 				Datum	   *numdatums = (Datum *) palloc(nnum * sizeof(Datum));
 				ArrayType  *arry;
 
@@ -1732,7 +1731,7 @@ update_attstats(Oid relid, bool inh, int natts, VacAttrStats **vacattrstats)
 		i = Anum_pg_statistic_stavalues1 - 1;
 		for (k = 0; k < STATISTIC_NUM_SLOTS; k++)
 		{
-			if (stats->numvalues[k] > 0)
+			if (stats->stavalues[k] != NULL)
 			{
 				ArrayType  *arry;
 

src/backend/tsearch/ts_selfuncs.c

@@ -239,8 +239,8 @@ mcelem_tsquery_selec(TSQuery query, Datum *mcelem, int nmcelem,
 	}
 
 	/*
-	 * Grab the lowest frequency. compute_tsvector_stats() stored it for us in
-	 * the one before the last cell of the Numbers array. See ts_typanalyze.c
+	 * Grab the lowest MCE frequency. compute_tsvector_stats() stored it for
+	 * us in the one before the last cell of the Numbers array.
 	 */
 	minfreq = numbers[nnumbers - 2];
 
@@ -348,7 +348,7 @@ tsquery_opr_selec(QueryItem *item, char *operand,
 			 * preserves the property that "word:*" should be estimated to
 			 * match at least as many rows as "word" would be.
 			 */
-			selec = Max(Min(DEFAULT_TS_MATCH_SEL, minfreq / 2), selec);
+			selec = Max(Min(DEFAULT_TS_MATCH_SEL, minfreq), selec);
 		}
 		else
 		{
@@ -375,9 +375,9 @@ tsquery_opr_selec(QueryItem *item, char *operand,
 			{
 				/*
 				 * The element is not in MCELEM.  Punt, but assume that the
-				 * selectivity cannot be more than minfreq / 2.
+				 * selectivity cannot be more than minfreq.
 				 */
-				selec = Min(DEFAULT_TS_MATCH_SEL, minfreq / 2);
+				selec = Min(DEFAULT_TS_MATCH_SEL, minfreq);
 			}
 		}
 	}

src/backend/tsearch/ts_typanalyze.c

@@ -312,7 +312,7 @@ compute_tsvector_stats(VacAttrStats *stats,
 		/*
 		 * Construct an array of the interesting hashtable items, that is,
 		 * those meeting the cutoff frequency (s - epsilon)*N.  Also identify
-		 * the minimum and maximum frequencies among these items.
+		 * the maximum frequency among these items.
 		 *
 		 * Since epsilon = s/10 and bucket_width = 1/epsilon, the cutoff
 		 * frequency is 9*N / bucket_width.
@@ -324,14 +324,12 @@ compute_tsvector_stats(VacAttrStats *stats,
 
 		hash_seq_init(&scan_status, lexemes_tab);
 		track_len = 0;
-		minfreq = lexeme_no;
 		maxfreq = 0;
 		while ((item = (TrackItem *) hash_seq_search(&scan_status)) != NULL)
 		{
 			if (item->frequency > cutoff_freq)
 			{
 				sort_table[track_len++] = item;
-				minfreq = Min(minfreq, item->frequency);
 				maxfreq = Max(maxfreq, item->frequency);
 			}
 		}
@@ -346,19 +344,38 @@ compute_tsvector_stats(VacAttrStats *stats,
 		 * If we obtained more lexemes than we really want, get rid of those
 		 * with least frequencies.  The easiest way is to qsort the array into
 		 * descending frequency order and truncate the array.
+		 *
+		 * If we did not find more elements than we want, then it is safe to
+		 * assume that the stored MCE array will contain every element with
+		 * frequency above the cutoff.  In that case, rather than storing the
+		 * smallest frequency we are keeping, we want to store the minimum
+		 * frequency that would have been accepted as a valid MCE.  The
+		 * selectivity functions can assume that that is an upper bound on the
+		 * frequency of elements not present in the array.
+		 *
+		 * If we found no candidate MCEs at all, we still want to record the
+		 * cutoff frequency, since it's still valid to assume that no element
+		 * has frequency more than that.
 		 */
 		if (num_mcelem < track_len)
 		{
 			qsort_interruptible(sort_table, track_len, sizeof(TrackItem *),
 								trackitem_compare_frequencies_desc, NULL);
-			/* reset minfreq to the smallest frequency we're keeping */
+			/* set minfreq to the smallest frequency we're keeping */
 			minfreq = sort_table[num_mcelem - 1]->frequency;
 		}
 		else
+		{
 			num_mcelem = track_len;
+			/* set minfreq to the minimum frequency above the cutoff */
+			minfreq = cutoff_freq + 1;
+			/* ensure maxfreq is nonzero, too */
+			if (track_len == 0)
+				maxfreq = minfreq;
+		}
 
 		/* Generate MCELEM slot entry */
-		if (num_mcelem > 0)
+		if (num_mcelem >= 0)
 		{
 			MemoryContext old_context;
 			Datum	   *mcelem_values;

src/backend/utils/adt/array_selfuncs.c

@@ -544,13 +544,13 @@ mcelem_array_contain_overlap_selec(Datum *mcelem, int nmcelem,
 
 	if (numbers)
 	{
-		/* Grab the lowest observed frequency */
+		/* Grab the minimal MCE frequency */
 		minfreq = numbers[nmcelem];
 	}
 	else
 	{
 		/* Without statistics make some default assumptions */
-		minfreq = 2 * (float4) DEFAULT_CONTAIN_SEL;
+		minfreq = (float4) DEFAULT_CONTAIN_SEL;
 	}
 
 	/* Decide whether it is faster to use binary search or not. */
@@ -622,9 +622,9 @@ mcelem_array_contain_overlap_selec(Datum *mcelem, int nmcelem,
 		{
 			/*
 			 * The element is not in MCELEM.  Punt, but assume that the
-			 * selectivity cannot be more than minfreq / 2.
+			 * selectivity cannot be more than minfreq.
 			 */
-			elem_selec = Min(DEFAULT_CONTAIN_SEL, minfreq / 2);
+			elem_selec = Min(DEFAULT_CONTAIN_SEL, minfreq);
 		}
 
 		/*
@@ -728,7 +728,7 @@ mcelem_array_contained_selec(Datum *mcelem, int nmcelem,
 
 	/*
 	 * Grab some of the summary statistics that compute_array_stats() stores:
-	 * lowest frequency, frequency of null elements, and average distinct
+	 * lowest MCE frequency, frequency of null elements, and average distinct
 	 * element count.
 	 */
 	minfreq = numbers[nmcelem];
@@ -803,10 +803,10 @@ mcelem_array_contained_selec(Datum *mcelem, int nmcelem,
 		{
 			/*
 			 * The element is not in MCELEM.  Punt, but assume that the
-			 * selectivity cannot be more than minfreq / 2.
+			 * selectivity cannot be more than minfreq.
 			 */
 			elem_selec[unique_nitems] = Min(DEFAULT_CONTAIN_SEL,
-											minfreq / 2);
+											minfreq);
 		}
 
 		unique_nitems++;

src/backend/utils/adt/array_typanalyze.c

@@ -461,7 +461,7 @@ compute_array_stats(VacAttrStats *stats, AnalyzeAttrFetchFunc fetchfunc,
 		/*
 		 * Construct an array of the interesting hashtable items, that is,
 		 * those meeting the cutoff frequency (s - epsilon)*N.  Also identify
-		 * the minimum and maximum frequencies among these items.
+		 * the maximum frequency among these items.
 		 *
 		 * Since epsilon = s/10 and bucket_width = 1/epsilon, the cutoff
 		 * frequency is 9*N / bucket_width.
@@ -473,14 +473,12 @@ compute_array_stats(VacAttrStats *stats, AnalyzeAttrFetchFunc fetchfunc,
 
 		hash_seq_init(&scan_status, elements_tab);
 		track_len = 0;
-		minfreq = element_no;
 		maxfreq = 0;
 		while ((item = (TrackItem *) hash_seq_search(&scan_status)) != NULL)
 		{
 			if (item->frequency > cutoff_freq)
 			{
 				sort_table[track_len++] = item;
-				minfreq = Min(minfreq, item->frequency);
 				maxfreq = Max(maxfreq, item->frequency);
 			}
 		}
@@ -497,19 +495,38 @@ compute_array_stats(VacAttrStats *stats, AnalyzeAttrFetchFunc fetchfunc,
 		 * If we obtained more elements than we really want, get rid of those
 		 * with least frequencies.  The easiest way is to qsort the array into
 		 * descending frequency order and truncate the array.
+		 *
+		 * If we did not find more elements than we want, then it is safe to
+		 * assume that the stored MCE array will contain every element with
+		 * frequency above the cutoff.  In that case, rather than storing the
+		 * smallest frequency we are keeping, we want to store the minimum
+		 * frequency that would have been accepted as a valid MCE.  The
+		 * selectivity functions can assume that that is an upper bound on the
+		 * frequency of elements not present in the array.
+		 *
+		 * If we found no candidate MCEs at all, we still want to record the
+		 * cutoff frequency, since it's still valid to assume that no element
+		 * has frequency more than that.
 		 */
 		if (num_mcelem < track_len)
 		{
 			qsort_interruptible(sort_table, track_len, sizeof(TrackItem *),
 								trackitem_compare_frequencies_desc, NULL);
-			/* reset minfreq to the smallest frequency we're keeping */
+			/* set minfreq to the smallest frequency we're keeping */
 			minfreq = sort_table[num_mcelem - 1]->frequency;
 		}
 		else
+		{
 			num_mcelem = track_len;
+			/* set minfreq to the minimum frequency above the cutoff */
+			minfreq = cutoff_freq + 1;
+			/* ensure maxfreq is nonzero, too */
+			if (track_len == 0)
+				maxfreq = minfreq;
+		}
 
 		/* Generate MCELEM slot entry */
-		if (num_mcelem > 0)
+		if (num_mcelem >= 0)
 		{
 			MemoryContext old_context;
 			Datum	   *mcelem_values;

src/include/catalog/pg_statistic.h

@@ -240,6 +240,10 @@ DECLARE_FOREIGN_KEY((starelid, staattnum), pg_attribute, (attrelid, attnum));
  * the fraction of non-null rows that contain at least one null element).  If
  * this member is omitted, the column is presumed to contain no null elements.
  *
+ * Starting in v19, the first extra member can be smaller than the smallest
+ * frequency of any stored MCE, indicating that it's known that no element
+ * not present in the MCE array has frequency greater than that value.
+ *
  * Note: in current usage for tsvector columns, the stavalues elements are of
  * type text, even though their representation within tsvector is not
  * exactly text.