In psql \copy from, send data to server in larger chunks.

Previously, we would send each line as a separate CopyData message.
That's pretty wasteful if the table is narrow, as each CopyData message
has 5 bytes of overhead. For efficiency, buffer up and pack 8 kB of
input data into each CopyData message.

The server also sends each line as a separate CopyData message in COPY TO
STDOUT, and that's similarly wasteful. But that's documented in the FE/BE
protocol description, so changing that would be a wire protocol break.

Reviewed-by: Aleksander Alekseev
Discussion: https://www.postgresql.org/message-id/40b2cec0-d0fb-3191-2ae1-9a3fe16a7e48%40iki.fi

diff --git a/src/bin/psql/copy.c b/src/bin/psql/copy.c
index e1fee8e0992ec16617d57d4eb4663cb04a54da87..64ab40c4f75feae9a52f81a9d464e9d81ed2dce3 100644 (file)
--- a/src/bin/psql/copy.c
+++ b/src/bin/psql/copy.c
@@ -581,13 +581,21 @@ handleCopyIn(PGconn *conn, FILE *copystream, bool isbinary, PGresult **res)
    else
    {
        bool        copydone = false;
+       int         buflen;
+       bool        at_line_begin = true;
 
+       /*
+        * In text mode, we have to read the input one line at a time, so that
+        * we can stop reading at the EOF marker (\.).  We mustn't read beyond
+        * the EOF marker, because if the data was inlined in a SQL script, we
+        * would eat up the commands after the EOF marker.
+        */
+       buflen = 0;
        while (!copydone)
-       {                       /* for each input line ... */
-           bool        firstload;
-           bool        linedone;
+       {
+           char       *fgresult;
 
-           if (showprompt)
+           if (at_line_begin && showprompt)
            {
                const char *prompt = get_prompt(PROMPT_COPY, NULL);
 
@@ -595,63 +603,68 @@ handleCopyIn(PGconn *conn, FILE *copystream, bool isbinary, PGresult **res)
                fflush(stdout);
            }
 
-           firstload = true;
-           linedone = false;
-
-           while (!linedone)
-           {                   /* for each bufferload in line ... */
-               int         linelen;
-               char       *fgresult;
-
-               /* enable longjmp while waiting for input */
-               sigint_interrupt_enabled = true;
+           /* enable longjmp while waiting for input */
+           sigint_interrupt_enabled = true;
 
-               fgresult = fgets(buf, sizeof(buf), copystream);
+           fgresult = fgets(&buf[buflen], COPYBUFSIZ - buflen, copystream);
 
-               sigint_interrupt_enabled = false;
+           sigint_interrupt_enabled = false;
 
-               if (!fgresult)
-               {
-                   copydone = true;
-                   break;
-               }
+           if (!fgresult)
+               copydone = true;
+           else
+           {
+               int         linelen;
 
-               linelen = strlen(buf);
+               linelen = strlen(fgresult);
+               buflen += linelen;
 
                /* current line is done? */
-               if (linelen > 0 && buf[linelen - 1] == '\n')
-                   linedone = true;
-
-               /* check for EOF marker, but not on a partial line */
-               if (firstload)
+               if (buf[buflen - 1] == '\n')
                {
-                   /*
-                    * This code erroneously assumes '\.' on a line alone
-                    * inside a quoted CSV string terminates the \copy.
-                    * https://www.postgresql.org/message-id/E1TdNVQ-0001ju-GO@wrigleys.postgresql.org
-                    */
-                   if (strcmp(buf, "\\.\n") == 0 ||
-                       strcmp(buf, "\\.\r\n") == 0)
+                   /* check for EOF marker, but not on a partial line */
+                   if (at_line_begin)
                    {
-                       copydone = true;
-                       break;
+                       /*
+                        * This code erroneously assumes '\.' on a line alone
+                        * inside a quoted CSV string terminates the \copy.
+                        * https://www.postgresql.org/message-id/E1TdNVQ-0001ju-GO@wrigleys.postgresql.org
+                        */
+                       if ((linelen == 3 && memcmp(fgresult, "\\.\n", 3) == 0) ||
+                           (linelen == 4 && memcmp(fgresult, "\\.\r\n", 4) == 0))
+                       {
+                           copydone = true;
+                       }
                    }
 
-                   firstload = false;
+                   if (copystream == pset.cur_cmd_source)
+                   {
+                       pset.lineno++;
+                       pset.stmt_lineno++;
+                   }
+                   at_line_begin = true;
                }
+               else
+                   at_line_begin = false;
+           }
 
-               if (PQputCopyData(conn, buf, linelen) <= 0)
+           /*
+            * If the buffer is full, or we've reached the EOF, flush it.
+            *
+            * Make sure there's always space for four more bytes in the
+            * buffer, plus a NUL terminator.  That way, an EOF marker is
+            * never split across two fgets() calls, which simplies the logic.
+            */
+           if (buflen >= COPYBUFSIZ - 5 || (copydone && buflen > 0))
+           {
+               if (PQputCopyData(conn, buf, buflen) <= 0)
                {
                    OK = false;
                    copydone = true;
                    break;
                }
-           }
 
-           if (copystream == pset.cur_cmd_source)
-           {
-               pset.lineno++;
-               pset.stmt_lineno++;
+               buflen = 0;
            }
        }
    }