Allow usage of huge maintenance_work_mem for GIN build.

Currently, in-memory posting list during GIN build process is limited 1GB
because of using repalloc. The patch replaces call of repalloc to repalloc_huge.
It increases limit of posting list from 180 millions
(1GB / sizeof(ItemPointerData)) to 4 billions limited by maxcount/count fields
in GinEntryAccumulator and subsequent calls. Check added.

Also, fix accounting of allocatedMemory during build to prevent integer
overflow with maintenance_work_mem > 4GB.

Robert Abraham <robert.abraham86@googlemail.com> with additions by me

diff --git a/src/backend/access/gin/ginbulk.c b/src/backend/access/gin/ginbulk.c
index e40e5be36bc77f2c6b7cc0863a0aaa93f08fd529..5a8546998d53c8335a9119d34cea6141c6a2d8c1 100644 (file)
--- a/src/backend/access/gin/ginbulk.c
+++ b/src/backend/access/gin/ginbulk.c
@@ -14,6 +14,8 @@
 
 #include "postgres.h"
 
+#include <limits.h>
+
 #include "access/gin_private.h"
 #include "utils/datum.h"
 #include "utils/memutils.h"
@@ -36,10 +38,16 @@ ginCombineData(RBNode *existing, const RBNode *newdata, void *arg)
     */
    if (eo->count >= eo->maxcount)
    {
+       if (eo->maxcount > INT_MAX)
+           ereport(ERROR,
+                   (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
+                    errmsg("posting list is too long"),
+                    errhint("Reduce maintenance_work_mem")));
+
        accum->allocatedMemory -= GetMemoryChunkSpace(eo->list);
        eo->maxcount *= 2;
        eo->list = (ItemPointerData *)
-           repalloc(eo->list, sizeof(ItemPointerData) * eo->maxcount);
+           repalloc_huge(eo->list, sizeof(ItemPointerData) * eo->maxcount);
        accum->allocatedMemory += GetMemoryChunkSpace(eo->list);
    }
 

diff --git a/src/backend/access/gin/ginfast.c b/src/backend/access/gin/ginfast.c
index c5732c3dc030ad6be789d424a09a09de9b216c48..54e593d8a974d5de9b73d00ba97a57f549bd1c86 100644 (file)
--- a/src/backend/access/gin/ginfast.c
+++ b/src/backend/access/gin/ginfast.c
@@ -814,7 +814,7 @@ ginInsertCleanup(GinState *ginstate,
         */
        if (GinPageGetOpaque(page)->rightlink == InvalidBlockNumber ||
            (GinPageHasFullRow(page) &&
-            (accum.allocatedMemory >= maintenance_work_mem * 1024L)))
+            (accum.allocatedMemory >= (Size)maintenance_work_mem * 1024L)))
        {
            ItemPointerData *list;
            uint32      nlist;

diff --git a/src/backend/access/gin/gininsert.c b/src/backend/access/gin/gininsert.c
index fc44f0205d07e3611c7a41437c2ef5d3c1958fb7..49e91853c3cec33159f8a3c64be79ed8ee60a6c5 100644 (file)
--- a/src/backend/access/gin/gininsert.c
+++ b/src/backend/access/gin/gininsert.c
@@ -281,7 +281,7 @@ ginBuildCallback(Relation index, HeapTuple htup, Datum *values,
                               &htup->t_self);
 
    /* If we've maxed out our available memory, dump everything to the index */
-   if (buildstate->accum.allocatedMemory >= maintenance_work_mem * 1024L)
+   if (buildstate->accum.allocatedMemory >= (Size)maintenance_work_mem * 1024L)
    {
        ItemPointerData *list;
        Datum       key;

diff --git a/src/include/access/gin_private.h b/src/include/access/gin_private.h
index 5095fc1dba85061970bf6cfdc8a2522ed5d6c3e7..acbe36a74147e64a5c888de409b8197a24baae79 100644 (file)
--- a/src/include/access/gin_private.h
+++ b/src/include/access/gin_private.h
@@ -903,7 +903,7 @@ typedef struct GinEntryAccumulator
 typedef struct
 {
    GinState   *ginstate;
-   long        allocatedMemory;
+   Size        allocatedMemory;
    GinEntryAccumulator *entryallocator;
    uint32      eas_used;
    RBTree     *tree;