diff --git a/contrib/Makefile b/contrib/Makefile index 2b314501f7..696776795e 100644 --- a/contrib/Makefile +++ b/contrib/Makefile @@ -18,6 +18,7 @@ SUBDIRS = \ dict_xsyn \ dummy_seclabel \ earthdistance \ + file_fdw \ fuzzystrmatch \ hstore \ intagg \ diff --git a/contrib/README b/contrib/README index 3c4e324271..5d5bdcec40 100644 --- a/contrib/README +++ b/contrib/README @@ -73,6 +73,9 @@ earthdistance - Functions for computing distances between two points on Earth by Bruno Wolff III and Hal Snyder +file_fdw + Foreign-data wrapper for server-side CSV/TEXT files + fuzzystrmatch - Levenshtein, metaphone, and soundex fuzzy string matching by Joe Conway and Joel Burton diff --git a/contrib/file_fdw/.gitignore b/contrib/file_fdw/.gitignore new file mode 100644 index 0000000000..36dc02fc75 --- /dev/null +++ b/contrib/file_fdw/.gitignore @@ -0,0 +1,3 @@ +/file_fdw.sql +# Generated subdirectories +/results/ diff --git a/contrib/file_fdw/Makefile b/contrib/file_fdw/Makefile new file mode 100644 index 0000000000..1d85645e39 --- /dev/null +++ b/contrib/file_fdw/Makefile @@ -0,0 +1,20 @@ +# contrib/file_fdw/Makefile + +MODULE_big = file_fdw +OBJS = file_fdw.o + +DATA_built = file_fdw.sql +DATA = uninstall_file_fdw.sql +REGRESS = file_fdw + + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = contrib/file_fdw +top_builddir = ../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/contrib/file_fdw/data/agg.bad b/contrib/file_fdw/data/agg.bad new file mode 100644 index 0000000000..3415b15007 --- /dev/null +++ b/contrib/file_fdw/data/agg.bad @@ -0,0 +1,4 @@ +56;@7.8@ +100;@99.097@ +0;@aaa@ +42;@324.78@ diff --git a/contrib/file_fdw/data/agg.csv b/contrib/file_fdw/data/agg.csv new file mode 100644 index 0000000000..3ee6bf24db --- /dev/null +++ b/contrib/file_fdw/data/agg.csv @@ -0,0 +1,4 @@ +56;@7.8@ +100;@99.097@ +0;@0.09561@ +42;@324.78@ diff --git a/contrib/file_fdw/data/agg.data b/contrib/file_fdw/data/agg.data new file mode 100644 index 0000000000..d92c7df4d0 --- /dev/null +++ b/contrib/file_fdw/data/agg.data @@ -0,0 +1,4 @@ +56 7.8 +100 99.097 +0 0.09561 +42 324.78 diff --git a/contrib/file_fdw/data/jagged.csv b/contrib/file_fdw/data/jagged.csv new file mode 100644 index 0000000000..90e392341a --- /dev/null +++ b/contrib/file_fdw/data/jagged.csv @@ -0,0 +1,6 @@ +header for a file with varying numbers of fields +one field +two,fields +three,,fields of which one is null +next line has no fields + diff --git a/contrib/file_fdw/expected/.gitignore b/contrib/file_fdw/expected/.gitignore new file mode 100644 index 0000000000..a464ad144f --- /dev/null +++ b/contrib/file_fdw/expected/.gitignore @@ -0,0 +1 @@ +/file_fdw.out diff --git a/contrib/file_fdw/file_fdw.c b/contrib/file_fdw/file_fdw.c new file mode 100644 index 0000000000..596456873b --- /dev/null +++ b/contrib/file_fdw/file_fdw.c @@ -0,0 +1,691 @@ +/*------------------------------------------------------------------------- + * + * file_fdw.c + * foreign-datga wrapper for server-side flat files. + * + * Copyright (c) 2011, PostgreSQL Global Development Group + * + * IDENTIFICATION + * contrib/file_fdw/file_fdw.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include +#include +#include + +#include "access/reloptions.h" +#include "catalog/pg_foreign_table.h" +#include "catalog/pg_foreign_server.h" +#include "catalog/pg_type.h" +#include "commands/copy.h" +#include "commands/defrem.h" +#include "foreign/foreign.h" +#include "funcapi.h" +#include "miscadmin.h" +#include "nodes/makefuncs.h" +#include "optimizer/cost.h" +#include "parser/parsetree.h" +#include "storage/fd.h" +#include "utils/array.h" +#include "utils/builtins.h" + +PG_MODULE_MAGIC; + +/* + * Describes the valid options for objects which uses this wrapper. + */ +struct FileFdwOption +{ + const char *optname; + Oid optcontext; /* Oid of catalog in which option may appear */ +}; + +/* + * Valid options for file_fdw. + * These options are based on the options for COPY FROM command. + * + * Note: If you are adding new option for user mapping, you need to modify + * fileBeginScan(). See comments of the function for detail. + */ +static struct FileFdwOption valid_options[] = { + /* File options */ + { "filename", ForeignTableRelationId }, + + /* Format options */ + /* oids option is not supported */ + { "format", ForeignTableRelationId }, + { "header", ForeignTableRelationId }, + { "delimiter", ForeignTableRelationId }, + { "quote", ForeignTableRelationId }, + { "escape", ForeignTableRelationId }, + { "null", ForeignTableRelationId }, + + /* FIXME: implement force_not_null option */ + + /* Local option */ + { "textarray", ForeignTableRelationId }, + + /* Sentinel */ + { NULL, InvalidOid } +}; + +#define FILE_FDW_TEXTARRAY_STASH_INIT 64 +/* + * FDW-specific information for FdwExecutionState. + */ +typedef struct FileFdwPrivate { + char *filename; + bool textarray; /* make a text array rather than a tuple */ + Relation rel; /* scan target relation */ + CopyState cstate; /* state of read in file */ + List *options; /* merged generic options, excluding filename */ + /* stash for processing text arrays - not used otherwise */ + int text_array_stash_size; + Datum *text_array_values; + bool *text_array_nulls; +} FileFdwPrivate; + +/* + * SQL functions + */ +extern Datum file_fdw_validator(PG_FUNCTION_ARGS); +extern Datum file_fdw_handler(PG_FUNCTION_ARGS); + +/* + * FDW routines + */ +static FdwPlan *filePlanRelScan(Oid foreigntableid, PlannerInfo *root, + RelOptInfo *baserel); +static FdwExecutionState *fileBeginScan(FdwPlan *fplan, ParamListInfo params); +static void fileIterate(FdwExecutionState *festate, TupleTableSlot *slot); +static void fileEndScan(FdwExecutionState *festate); +static void fileReScan(FdwExecutionState *festate); + +/* text array support */ +static void makeTextArray(FileFdwPrivate *fdw_private, + TupleTableSlot *slot, char **raw_fields, int nfields); +/* + * Helper functions + */ +static char *generate_explain_info(const char *filename, unsigned long size); +static unsigned long estimate_costs(const char *filename, RelOptInfo *baserel, + double *startup_cost, double *total_cost); + +/* + * Check if the provided option is one of valid options. + * context is the Oid of the catalog the option came from, or 0 if we + * don't care. + */ +static bool +is_valid_option(const char *option, Oid context) +{ + struct FileFdwOption *opt; + + for (opt = valid_options; opt->optname; opt++) + if (context == opt->optcontext && strcmp(opt->optname, option) == 0) + return true; + return false; +} + +/* + * Validate the generic option given to FOREIGN DATA WRAPPER, SERVER, USER + * MAPPING or FOREIGN TABLE which use file_fdw. + * Raise an ERROR if the option or its value is considered + * invalid. + */ +PG_FUNCTION_INFO_V1(file_fdw_validator); +Datum +file_fdw_validator(PG_FUNCTION_ARGS) +{ + List *options_list = untransformRelOptions(PG_GETARG_DATUM(0)); + Oid catalog = PG_GETARG_OID(1); + + ListCell *cell; + + char *format = NULL; + char *delimiter = NULL; + char *quote = NULL; + char *escape = NULL; + char *null = NULL; + bool header; + + /* Only superuser can change generic options of the foreign table */ + if (catalog == ForeignTableRelationId && !superuser()) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("only superuser can change foreign table options"))); + + /* Validate each options */ + foreach(cell, options_list) + { + DefElem *def = lfirst(cell); + + if (!is_valid_option(def->defname, catalog)) + { + struct FileFdwOption *opt; + StringInfoData buf; + + /* + * Unknown option specified, complain about it. Provide a hint + * with list of valid options for the object. + */ + initStringInfo(&buf); + for (opt = valid_options; opt->optname; opt++) + if (catalog == opt->optcontext) + appendStringInfo(&buf, "%s%s", (buf.len > 0) ? ", " : "", + opt->optname); + + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_OPTION_NAME), + errmsg("invalid option \"%s\"", def->defname), + errhint("Valid options in this context are: %s", buf.data))); + + PG_RETURN_BOOL(false); + } + + if (strcmp(def->defname, "format") == 0) + { + if (pg_strcasecmp(strVal(def->arg), "csv") != 0 && + pg_strcasecmp(strVal(def->arg), "text") != 0) + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_ATTRIBUTE_VALUE), + errmsg("format must be csv or text"))); + format = strVal(def->arg); + } + else if (strcmp(def->defname, "header") == 0) + { + header = defGetBoolean(def); + } + else if (strcmp(def->defname, "delimiter") == 0) + { + if (strlen(strVal(def->arg)) != 1) + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_ATTRIBUTE_VALUE), + errmsg("delimiter must be a single one-byte1 character"))); + if (strchr(strVal(def->arg), '\r') != NULL || + strchr(strVal(def->arg), '\n') != NULL) + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_ATTRIBUTE_VALUE), + errmsg("delimiter cannot be newline or carriage return"))); + delimiter = strVal(def->arg); + } + else if (strcmp(def->defname, "quote") == 0) + { + if (strlen(strVal(def->arg)) != 1) + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_ATTRIBUTE_VALUE), + errmsg("quote must be 1 byte"))); + quote = strVal(def->arg); + } + else if (strcmp(def->defname, "escape") == 0) + { + if (strlen(strVal(def->arg)) != 1) + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_ATTRIBUTE_VALUE), + errmsg("escape must be 1 byte"))); + escape = strVal(def->arg); + } + else if (strcmp(def->defname, "null") == 0) + { + if (strchr(strVal(def->arg), '\r') != NULL || + strchr(strVal(def->arg), '\n') != NULL) + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_ATTRIBUTE_VALUE), + errmsg("null representation cannot use newline or carriage return"))); + null = strVal(def->arg); + } + } + + /* Check options which depend on the file format. */ + if (format != NULL && pg_strcasecmp(format, "text") == 0) + { + if (delimiter && strchr("\\.abcdefghijklmnopqrstuvwxyz0123456789", + delimiter[0]) != NULL) + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_ATTRIBUTE_VALUE), + errmsg("delimiter cannot be \"%s\"", delimiter))); + + if (escape != NULL) + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_ATTRIBUTE_VALUE), + errmsg("escape available only in CSV mode"))); + } + else if (format != NULL && pg_strcasecmp(format, "csv") == 0) + { + if (null != NULL && quote != NULL && strchr(null, quote[0]) != NULL) + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_ATTRIBUTE_VALUE), + errmsg("quote must not appear in the NULL specification"))); + } + + if (delimiter != NULL && quote != NULL) + if (strcmp(delimiter, quote) == 0) + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_ATTRIBUTE_VALUE), + errmsg("delimiter and quote must be different"))); + + if (null != NULL && delimiter != NULL) + if (strchr(null, delimiter[0]) != NULL) + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_ATTRIBUTE_VALUE), + errmsg("delimiter must not appear in the NULL specification"))); + + PG_RETURN_BOOL(true); +} + +/* + * return foreign-data wrapper handler object to execute foreign-data wrapper + * routines. + */ +PG_FUNCTION_INFO_V1(file_fdw_handler); +Datum +file_fdw_handler(PG_FUNCTION_ARGS) +{ + static FdwRoutine file_fdw_routine = + { + filePlanRelScan, + fileBeginScan, + fileIterate, + fileReScan, + fileEndScan, + }; + + PG_RETURN_POINTER(&file_fdw_routine); +} + +/* + * Create a FdwPlan for a scan on the foreign table. + * + * FdwPlan must be able to be copied by copyObject(), so private area is a list + * of copy-able elements. The list consists of elements below: + * + * (1) oid of the target relation, Oid Const + * (2) name of the file, String Value + * (3) list of fdw options excluding filename, List of DefElem + * + * This format must be used commonly in other planning functions, such as + * PlanQuery and PlanNative. + */ +static FdwPlan * +filePlanRelScan(Oid foreigntableid, PlannerInfo *root, RelOptInfo *rel) +{ + Const *relid; + Value *filename = NULL; + bool textarray = false; + Const *textarray_param; + ulong size; + FdwPlan *fplan; + ForeignTable *table; + ForeignServer *server; + ForeignDataWrapper *wrapper; + List *options; + ListCell *lc, *prev; + + /* + * Create new relid instance because we use 'private' list as a pointer + * list. + */ + relid = makeConst(OIDOID, + -1, + sizeof(Oid), + ObjectIdGetDatum(foreigntableid), + false, true); + + /* Extract options from FDW objects */ + table = GetForeignTable(foreigntableid); + server = GetForeignServer(table->serverid); + wrapper = GetForeignDataWrapper(server->fdwid); + options = NIL; + options = list_concat(options, wrapper->options); + options = list_concat(options, server->options); + options = list_concat(options, table->options); + + /* + * Split filename option off from the list because filename should be + * passed as another parameter to BeginCopyFrom(). + */ + prev = NULL; + foreach (lc, options) + { + DefElem *def = lfirst(lc); + if (strcmp(def->defname, "filename") == 0) + { + filename = makeString(strVal(def->arg)); + options = list_delete_cell(options, lc, prev); + break; + } + prev = lc; + } + if (filename == NULL) + ereport(ERROR, + (errcode(ERRCODE_FDW_UNABLE_TO_CREATE_REPLY), + errmsg("filename is required for file_fdw scan"))); + + /* + * Split textarray option off from the list because it's handled + * here instead of being passed as another parameter to BeginCopyFrom(). + */ + prev = NULL; + foreach (lc, options) + { + DefElem *def = lfirst(lc); + if (strcmp(def->defname, "textarray") == 0) + { + textarray = defGetBoolean(def); + options = list_delete_cell(options, lc, prev); + break; + } + prev = lc; + } + textarray_param = (Const *) makeBoolConst(textarray,false); + + if (textarray) + { + /* make sure the table has one column and it's oy type text[] */ + /* XXX fill in this piece */ + } + + /* Construct FdwPlan and store relid and options in private area */ + fplan = makeNode(FdwPlan); + size = estimate_costs(strVal(filename), rel, + &fplan->startup_cost, &fplan->total_cost); + fplan->explainInfo = generate_explain_info(strVal(filename), size); + fplan->fdw_private = NIL; + fplan->fdw_private = lappend(fplan->fdw_private, relid); + fplan->fdw_private = lappend(fplan->fdw_private, filename); + fplan->fdw_private = lappend(fplan->fdw_private, textarray_param); + fplan->fdw_private = lappend(fplan->fdw_private, options); + + return fplan; +} + +/* + * BeginScan() + * - initiate access to the file with creating CopyState + * + * Parameters for parsing file such as filename and format are passed via + * generic options of FDW-related objects; foreign-data wrapper, server and + * foreign table. User mapping is not used to get options because there is no + * valid option in context of user mapping. + */ +static FdwExecutionState * +fileBeginScan(FdwPlan *fplan, ParamListInfo params) +{ + Const *relid_const; + Oid relid; + Value *filename; + Const *textarray; + List *options; + Relation rel; + CopyState cstate; + FileFdwPrivate *fdw_private; + FdwExecutionState *festate; + + elog(DEBUG3, "%s called", __FUNCTION__); + + /* Get oid of the relation and option list from private area of FdwPlan. */ + relid_const = list_nth(fplan->fdw_private, 0); + filename = list_nth(fplan->fdw_private, 1); + textarray = list_nth(fplan->fdw_private, 2); + options = list_nth(fplan->fdw_private, 3); + + relid = DatumGetObjectId(relid_const->constvalue); + + /* + * Create CopyState from FDW options. We always acquire all columns. + * We open the relation with no lock because it's assumed that appropriate + * lock has been acquired already. The rel should be closed in + * fileEndScan(). + */ + rel = heap_open(relid, NoLock); + cstate = BeginCopyFrom(rel, strVal(filename), NIL, options); + + /* + * Pack file information into private and pass it to subsequent functions. + * We also store information enough to call BeginCopyFrom() again. + */ + festate = palloc0(sizeof(FdwExecutionState)); + fdw_private = palloc0(sizeof(FileFdwPrivate)); + fdw_private->filename = strVal(filename); + fdw_private->textarray = textarray->constvalue; + fdw_private->rel = rel; + fdw_private->cstate = cstate; + fdw_private->options = options; + festate->fdw_private = (void *) fdw_private; + + if (fdw_private->textarray) + { + fdw_private->text_array_stash_size = FILE_FDW_TEXTARRAY_STASH_INIT; + fdw_private->text_array_values = + palloc(FILE_FDW_TEXTARRAY_STASH_INIT * sizeof(Datum)); + fdw_private->text_array_nulls = + palloc(FILE_FDW_TEXTARRAY_STASH_INIT * sizeof(bool)); + } + + return festate; +} + +/* + * Iterate() + * - create HeapTuple from the record in the file. + */ +static void +fileIterate(FdwExecutionState *festate, TupleTableSlot *slot) +{ + FileFdwPrivate *fdw_private = (FileFdwPrivate *) festate->fdw_private; + bool found; + ErrorContextCallback errcontext; + + elog(DEBUG3, "%s called for \"%s\"", __FUNCTION__, fdw_private->filename); + + /* Set up callback to identify error line number. */ + errcontext.callback = CopyFromErrorCallback; + errcontext.arg = (void *) fdw_private->cstate; + errcontext.previous = error_context_stack; + error_context_stack = &errcontext; + + /* + * If next tuple has been found, store it into the slot as materialized + * tuple. Otherwise, clear the slot to tell executor that we have reached + * EOF. + */ + ExecClearTuple(slot); + if (fdw_private->textarray) + { + char **raw_fields; + int nfields; + + found = NextLineCopyFrom(fdw_private->cstate, &raw_fields, &nfields, + NULL); + if (found) + makeTextArray(fdw_private, slot, raw_fields, nfields); + } + else + { + /* let the COPY code do the work */ + found = NextCopyFrom(fdw_private->cstate, slot->tts_values, + slot->tts_isnull, NULL); + } + if (found) + ExecStoreVirtualTuple(slot); + + /* + * Cleanup error callback. We must uninstall callback before leaving + * Iterate() because other scan in the same plan tree might generate error. + */ + error_context_stack = errcontext.previous; +} + +/* + * Finish scanning foreign table and dispose objects used for this scan. + */ +static void +fileEndScan(FdwExecutionState *festate) +{ + FileFdwPrivate *fdw_private; + + elog(DEBUG3, "%s called", __FUNCTION__); + + fdw_private = (FileFdwPrivate *) festate->fdw_private; + EndCopyFrom(fdw_private->cstate); + + heap_close(fdw_private->rel, NoLock); + pfree(fdw_private); + pfree(festate); +} + +/* + * Execute query with new parameter. + */ +static void +fileReScan(FdwExecutionState *festate) +{ + FileFdwPrivate *fdw_private = (FileFdwPrivate *) festate->fdw_private; + + elog(DEBUG3, "%s called for \"%s\"", __FUNCTION__, fdw_private->filename); + + EndCopyFrom(fdw_private->cstate); + fdw_private->cstate = BeginCopyFrom(fdw_private->rel, + fdw_private->filename, + NIL, + fdw_private->options); +} + +/* + * Generate explain info string from information about the file. + */ +static char * +generate_explain_info(const char *filename, unsigned long size) +{ + StringInfoData explainInfo; + + initStringInfo(&explainInfo); + + /* + * Construct explain information. + */ + appendStringInfo(&explainInfo, "file=\"%s\", size=%lu", filename, size); + + return explainInfo.data; +} + +/* + * Estimate costs of scanning on a foreign table, and return size of the file. + */ +static unsigned long +estimate_costs(const char *filename, RelOptInfo *baserel, + double *startup_cost, double *total_cost) +{ + struct stat stat_buf; + BlockNumber pages; + double run_cost = 0; + double cpu_per_tuple; + + elog(DEBUG3, "%s called", __FUNCTION__); + + /* get size of the file */ + if (stat(filename, &stat_buf) == -1) + { + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not stat file \"%s\": %m", filename))); + } + + /* + * The way to estimate costs is almost same as cost_seqscan(), but there + * are some differences: + * - DISK costs are estimated from file size. + * - CPU costs are 10x of seq scan, for overhead of parsing records. + */ + pages = stat_buf.st_size / BLCKSZ + (stat_buf.st_size % BLCKSZ > 0 ? 1 : 0); + run_cost += seq_page_cost * pages; + + /* + * file_fdw resets a scan with re-opening the file, so random_page_cost is + * added to the startup cost as "heavy disk access". + */ + *startup_cost += baserel->baserestrictcost.startup + random_page_cost; + cpu_per_tuple = cpu_tuple_cost + baserel->baserestrictcost.per_tuple; + run_cost += cpu_per_tuple * 10 * baserel->tuples; + *total_cost = *startup_cost + run_cost; + + return stat_buf.st_size; +} + +static void +makeTextArray(FileFdwPrivate *fdw_private, TupleTableSlot *slot, char **raw_fields, int nfields) +{ + Datum *values; + bool *nulls; + int dims[1]; + int lbs[1]; + int fld; + Datum result; + int fldct = nfields; + char *string; + + if (nfields == 1 && + raw_fields[0] == NULL + /* fixme - probably need to get this from fdw_private */ + /* && cstate->null_print_len == 0 */ + ) + { + /* Treat an empty line as having no fields */ + fldct = 0; + } + else if (nfields > fdw_private->text_array_stash_size) + { + while (fdw_private->text_array_stash_size < nfields) + fdw_private->text_array_stash_size *= 2; + + fdw_private->text_array_values =repalloc( + fdw_private->text_array_values, + fdw_private->text_array_stash_size * sizeof(Datum)); + fdw_private->text_array_nulls =repalloc( + fdw_private->text_array_nulls, + fdw_private->text_array_stash_size * sizeof(bool)); + } + + values = fdw_private->text_array_values; + nulls = fdw_private->text_array_nulls; + + dims[0] = fldct; + lbs[0] = 1; /* sql arrays typically start at 1 */ + + for (fld=0; fld < fldct; fld++) + { + string = raw_fields[fld]; + + if (string == NULL) + { + values[fld] = PointerGetDatum(NULL); + nulls[fld] = true; + } + else + { + nulls[fld] = false; + values[fld] = PointerGetDatum( + DirectFunctionCall1(textin, + PointerGetDatum(string))); + } + } + + result = PointerGetDatum(construct_md_array( + values, + nulls, + 1, + dims, + lbs, + TEXTOID, + -1, + false, + 'i')); + + slot->tts_values[0] = result; + slot->tts_isnull[0] = false; + +} diff --git a/contrib/file_fdw/file_fdw.sql.in b/contrib/file_fdw/file_fdw.sql.in new file mode 100644 index 0000000000..8aadd2496e --- /dev/null +++ b/contrib/file_fdw/file_fdw.sql.in @@ -0,0 +1,19 @@ +/* contrib/file_fdw/file_fdw.sql.in */ + +-- Adjust this setting to control where the objects get created. +SET search_path = public; + +-- create wrapper with validator and handler +CREATE OR REPLACE FUNCTION file_fdw_validator (text[], oid) +RETURNS bool +AS 'MODULE_PATHNAME','file_fdw_validator' +LANGUAGE C STRICT; + +CREATE OR REPLACE FUNCTION file_fdw_handler () +RETURNS fdw_handler +AS 'MODULE_PATHNAME','file_fdw_handler' +LANGUAGE C STRICT; + +CREATE FOREIGN DATA WRAPPER file_fdw +VALIDATOR file_fdw_validator HANDLER file_fdw_handler; + diff --git a/contrib/file_fdw/input/file_fdw.source b/contrib/file_fdw/input/file_fdw.source new file mode 100644 index 0000000000..fb61e9f27d --- /dev/null +++ b/contrib/file_fdw/input/file_fdw.source @@ -0,0 +1,119 @@ +-- +-- Test foreign-data wrapper file_fdw. +-- + +-- Clean up in case a prior regression run failed + +-- Suppress NOTICE messages when roles don't exist +SET client_min_messages TO 'error'; + +DROP ROLE IF EXISTS file_fdw_superuser, file_fdw_user, no_priv_user; + +RESET client_min_messages; + +CREATE ROLE file_fdw_superuser LOGIN SUPERUSER; -- is a superuser +CREATE ROLE file_fdw_user LOGIN; -- has priv and user mapping +CREATE ROLE no_priv_user LOGIN; -- has priv but no user mapping + +-- Install file_fdw +SET client_min_messages = warning; +\set ECHO none +\i file_fdw.sql +\set ECHO all +RESET client_min_messages; + +-- file_fdw_superuser owns fdw-related objects +SET ROLE file_fdw_superuser; +CREATE SERVER file_server FOREIGN DATA WRAPPER file_fdw; + +-- privilege tests +SET ROLE file_fdw_user; +CREATE FOREIGN DATA WRAPPER file_fdw2 VALIDATOR file_fdw_validator HANDLER file_fdw_handler; -- ERROR +CREATE SERVER file_server2 FOREIGN DATA WRAPPER file_fdw; -- ERROR +CREATE USER MAPPING FOR file_fdw_user SERVER file_server; -- ERROR + +SET ROLE file_fdw_superuser; +GRANT USAGE ON FOREIGN SERVER file_server TO file_fdw_user; + +SET ROLE file_fdw_user; +CREATE USER MAPPING FOR file_fdw_user SERVER file_server; + +-- create user mappings and grant privilege to test users +SET ROLE file_fdw_superuser; +CREATE USER MAPPING FOR file_fdw_superuser SERVER file_server; +CREATE USER MAPPING FOR no_priv_user SERVER file_server; + +-- validator tests +CREATE FOREIGN TABLE tbl () SERVER file_server OPTIONS (format 'xml'); -- ERROR +CREATE FOREIGN TABLE tbl () SERVER file_server OPTIONS (format 'text', delimiter 'a'); -- ERROR +CREATE FOREIGN TABLE tbl () SERVER file_server OPTIONS (format 'text', escape '-'); -- ERROR +CREATE FOREIGN TABLE tbl () SERVER file_server OPTIONS (format 'csv', quote '-', null '=-='); -- ERROR +CREATE FOREIGN TABLE tbl () SERVER file_server OPTIONS (format 'csv', delimiter '-', null '=-='); -- ERROR +CREATE FOREIGN TABLE tbl () SERVER file_server OPTIONS (format 'csv', delimiter '-', quote '-'); -- ERROR +CREATE FOREIGN TABLE agg_text ( + a int2, + b float4 +) SERVER file_server +OPTIONS (format 'text', filename '@abs_srcdir@/data/agg.data', delimiter ' ', null '\N'); +GRANT SELECT ON agg_text TO file_fdw_user; +CREATE FOREIGN TABLE agg_csv ( + a int2, + b float4 +) SERVER file_server +OPTIONS (format 'csv', filename '@abs_srcdir@/data/agg.csv', header 'true', delimiter ';', quote '@', escape '"', null ''); +CREATE FOREIGN TABLE agg_bad ( + a int2, + b float4 +) SERVER file_server +OPTIONS (format 'csv', filename '@abs_srcdir@/data/agg.bad', header 'true', delimiter ';', quote '@', escape '"', null ''); + +CREATE FOREIGN TABLE jagged_text ( + t text[] +) SERVER file_server +OPTIONS (format 'csv', filename '@abs_srcdir@/data/jagged.csv', header 'true', textarray 'true'); + +-- basic query tests +SELECT * FROM agg_text WHERE b > 10.0 ORDER BY a; +SELECT * FROM agg_csv ORDER BY a; +SELECT * FROM agg_csv c JOIN agg_text t ON (t.a = c.a) ORDER BY c.a; + +-- textarray tests +SELECT * FROM jagged_text; +SELECT t[3] AS a, t[1] AS b, t[99] AS c FROM jagged_text; + +-- error context report tests +SELECT * FROM agg_bad; -- ERROR + +-- misc query tests +\t on +EXPLAIN (VERBOSE, COSTS FALSE) SELECT * FROM agg_csv; +\t off +PREPARE st(int) AS SELECT * FROM agg_csv WHERE a = $1; +EXECUTE st(100); +EXECUTE st(100); +DEALLOCATE st; + +-- privilege tests +SET ROLE file_fdw_superuser; +SELECT * FROM agg_text ORDER BY a; +SET ROLE file_fdw_user; +SELECT * FROM agg_text ORDER BY a; +SET ROLE no_priv_user; +SELECT * FROM agg_text ORDER BY a; -- ERROR +SET ROLE file_fdw_user; +\t on +EXPLAIN (VERBOSE, COSTS FALSE) SELECT * FROM agg_text; +\t off + +-- privilege tests for object +SET ROLE file_fdw_superuser; +ALTER FOREIGN TABLE agg_text OWNER TO file_fdw_user; +ALTER FOREIGN TABLE agg_text OPTIONS (SET format 'text'); +SET ROLE file_fdw_user; +ALTER FOREIGN TABLE agg_text OPTIONS (SET format 'text'); +SET ROLE file_fdw_superuser; + +-- cleanup +RESET ROLE; +DROP FOREIGN DATA WRAPPER file_fdw CASCADE; +DROP ROLE IF EXISTS file_fdw_superuser, file_fdw_user, no_priv_user; diff --git a/contrib/file_fdw/output/file_fdw.source b/contrib/file_fdw/output/file_fdw.source new file mode 100644 index 0000000000..b548474b82 --- /dev/null +++ b/contrib/file_fdw/output/file_fdw.source @@ -0,0 +1,192 @@ +-- +-- Test foreign-data wrapper file_fdw. +-- +-- Clean up in case a prior regression run failed +-- Suppress NOTICE messages when roles don't exist +SET client_min_messages TO 'error'; +DROP ROLE IF EXISTS file_fdw_superuser, file_fdw_user, no_priv_user; +RESET client_min_messages; +CREATE ROLE file_fdw_superuser LOGIN SUPERUSER; -- is a superuser +CREATE ROLE file_fdw_user LOGIN; -- has priv and user mapping +CREATE ROLE no_priv_user LOGIN; -- has priv but no user mapping +-- Install file_fdw +SET client_min_messages = warning; +\set ECHO none +RESET client_min_messages; +-- file_fdw_superuser owns fdw-related objects +SET ROLE file_fdw_superuser; +CREATE SERVER file_server FOREIGN DATA WRAPPER file_fdw; +-- privilege tests +SET ROLE file_fdw_user; +CREATE FOREIGN DATA WRAPPER file_fdw2 VALIDATOR file_fdw_validator HANDLER file_fdw_handler; -- ERROR +ERROR: permission denied to create foreign-data wrapper "file_fdw2" +HINT: Must be superuser to create a foreign-data wrapper. +CREATE SERVER file_server2 FOREIGN DATA WRAPPER file_fdw; -- ERROR +ERROR: permission denied for foreign-data wrapper file_fdw +CREATE USER MAPPING FOR file_fdw_user SERVER file_server; -- ERROR +ERROR: permission denied for foreign server file_server +SET ROLE file_fdw_superuser; +GRANT USAGE ON FOREIGN SERVER file_server TO file_fdw_user; +SET ROLE file_fdw_user; +CREATE USER MAPPING FOR file_fdw_user SERVER file_server; +-- create user mappings and grant privilege to test users +SET ROLE file_fdw_superuser; +CREATE USER MAPPING FOR file_fdw_superuser SERVER file_server; +CREATE USER MAPPING FOR no_priv_user SERVER file_server; +-- validator tests +CREATE FOREIGN TABLE tbl () SERVER file_server OPTIONS (format 'xml'); -- ERROR +ERROR: format must be csv or text +CREATE FOREIGN TABLE tbl () SERVER file_server OPTIONS (format 'text', delimiter 'a'); -- ERROR +ERROR: delimiter cannot be "a" +CREATE FOREIGN TABLE tbl () SERVER file_server OPTIONS (format 'text', escape '-'); -- ERROR +ERROR: escape available only in CSV mode +CREATE FOREIGN TABLE tbl () SERVER file_server OPTIONS (format 'csv', quote '-', null '=-='); -- ERROR +ERROR: quote must not appear in the NULL specification +CREATE FOREIGN TABLE tbl () SERVER file_server OPTIONS (format 'csv', delimiter '-', null '=-='); -- ERROR +ERROR: delimiter must not appear in the NULL specification +CREATE FOREIGN TABLE tbl () SERVER file_server OPTIONS (format 'csv', delimiter '-', quote '-'); -- ERROR +ERROR: delimiter and quote must be different +CREATE FOREIGN TABLE agg_text ( + a int2, + b float4 +) SERVER file_server +OPTIONS (format 'text', filename '@abs_srcdir@/data/agg.data', delimiter ' ', null '\N'); +GRANT SELECT ON agg_text TO file_fdw_user; +CREATE FOREIGN TABLE agg_csv ( + a int2, + b float4 +) SERVER file_server +OPTIONS (format 'csv', filename '@abs_srcdir@/data/agg.csv', header 'true', delimiter ';', quote '@', escape '"', null ''); +CREATE FOREIGN TABLE agg_bad ( + a int2, + b float4 +) SERVER file_server +OPTIONS (format 'csv', filename '@abs_srcdir@/data/agg.bad', header 'true', delimiter ';', quote '@', escape '"', null ''); +CREATE FOREIGN TABLE jagged_text ( + t text[] +) SERVER file_server +OPTIONS (format 'csv', filename '@abs_srcdir@/data/jagged.csv', header 'true', textarray 'true'); +-- basic query tests +SELECT * FROM agg_text WHERE b > 10.0 ORDER BY a; + a | b +-----+-------- + 42 | 324.78 + 100 | 99.097 +(2 rows) + +SELECT * FROM agg_csv ORDER BY a; + a | b +-----+--------- + 0 | 0.09561 + 42 | 324.78 + 100 | 99.097 +(3 rows) + +SELECT * FROM agg_csv c JOIN agg_text t ON (t.a = c.a) ORDER BY c.a; + a | b | a | b +-----+---------+-----+--------- + 0 | 0.09561 | 0 | 0.09561 + 42 | 324.78 | 42 | 324.78 + 100 | 99.097 | 100 | 99.097 +(3 rows) + +-- textarray tests +SELECT * FROM jagged_text; + t +-------------------------------------------- + {"one field"} + {two,fields} + {three,NULL,"fields of which one is null"} + {"next line has no fields"} + {} +(5 rows) + +SELECT t[3] AS a, t[1] AS b, t[99] AS c FROM jagged_text; + a | b | c +-----------------------------+-------------------------+--- + | one field | + | two | + fields of which one is null | three | + | next line has no fields | + | | +(5 rows) + +-- error context report tests +SELECT * FROM agg_bad; -- ERROR +ERROR: invalid input syntax for type real: "aaa" +CONTEXT: relation agg_bad, line 3, column b: "aaa" +-- misc query tests +\t on +EXPLAIN (VERBOSE, COSTS FALSE) SELECT * FROM agg_csv; + Foreign Scan on public.agg_csv + Output: a, b + FDW-Info: file="@abs_srcdir@/data/agg.csv", size=46 + +\t off +PREPARE st(int) AS SELECT * FROM agg_csv WHERE a = $1; +EXECUTE st(100); + a | b +-----+-------- + 100 | 99.097 +(1 row) + +EXECUTE st(100); + a | b +-----+-------- + 100 | 99.097 +(1 row) + +DEALLOCATE st; +-- privilege tests +SET ROLE file_fdw_superuser; +SELECT * FROM agg_text ORDER BY a; + a | b +-----+--------- + 0 | 0.09561 + 42 | 324.78 + 56 | 7.8 + 100 | 99.097 +(4 rows) + +SET ROLE file_fdw_user; +SELECT * FROM agg_text ORDER BY a; + a | b +-----+--------- + 0 | 0.09561 + 42 | 324.78 + 56 | 7.8 + 100 | 99.097 +(4 rows) + +SET ROLE no_priv_user; +SELECT * FROM agg_text ORDER BY a; -- ERROR +ERROR: permission denied for relation agg_text +SET ROLE file_fdw_user; +\t on +EXPLAIN (VERBOSE, COSTS FALSE) SELECT * FROM agg_text; + Foreign Scan on public.agg_text + Output: a, b + FDW-Info: file="@abs_srcdir@/data/agg.data", size=38 + +\t off +-- privilege tests for object +SET ROLE file_fdw_superuser; +ALTER FOREIGN TABLE agg_text OWNER TO file_fdw_user; +ALTER FOREIGN TABLE agg_text OPTIONS (SET format 'text'); +SET ROLE file_fdw_user; +ALTER FOREIGN TABLE agg_text OPTIONS (SET format 'text'); +ERROR: only superuser can change foreign table options +SET ROLE file_fdw_superuser; +-- cleanup +RESET ROLE; +DROP FOREIGN DATA WRAPPER file_fdw CASCADE; +NOTICE: drop cascades to 8 other objects +DETAIL: drop cascades to server file_server +drop cascades to user mapping for file_fdw_user +drop cascades to user mapping for file_fdw_superuser +drop cascades to user mapping for no_priv_user +drop cascades to foreign table agg_text +drop cascades to foreign table agg_csv +drop cascades to foreign table agg_bad +drop cascades to foreign table jagged_text +DROP ROLE IF EXISTS file_fdw_superuser, file_fdw_user, no_priv_user; diff --git a/contrib/file_fdw/sql/.gitignore b/contrib/file_fdw/sql/.gitignore new file mode 100644 index 0000000000..ebf16fed94 --- /dev/null +++ b/contrib/file_fdw/sql/.gitignore @@ -0,0 +1 @@ +/file_fdw.sql diff --git a/contrib/file_fdw/uninstall_file_fdw.sql b/contrib/file_fdw/uninstall_file_fdw.sql new file mode 100644 index 0000000000..b0b58f0fb9 --- /dev/null +++ b/contrib/file_fdw/uninstall_file_fdw.sql @@ -0,0 +1,10 @@ +/* contrib/file_fdw/uninstall_file_fdw.sql */ + +-- Adjust this setting to control where the objects get created. +SET search_path = public; + +-- create wrapper with validator and handler +DROP FOREIGN DATA WRAPPER file_fdw; +DROP FUNCTION file_fdw_handler (); +DROP FUNCTION file_fdw_validator (text[], oid); + diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index eda82c5f34..984843ce78 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -2811,6 +2811,17 @@ + + fdwhandler + oid + pg_proc.oid + + References a handler function that is responsible for + supplying foreign-data wrapper routines. + Zero if no handler is provided. + + + fdwacl aclitem[] diff --git a/doc/src/sgml/contrib.sgml b/doc/src/sgml/contrib.sgml index 75d08d5f69..c781c5608d 100644 --- a/doc/src/sgml/contrib.sgml +++ b/doc/src/sgml/contrib.sgml @@ -93,6 +93,7 @@ psql -d dbname -f SHAREDIR/contrib/module.sql &dict-xsyn; &dummy-seclabel; &earthdistance; + &file-fdw; &fuzzystrmatch; &hstore; &intagg; diff --git a/doc/src/sgml/file-fdw.sgml b/doc/src/sgml/file-fdw.sgml new file mode 100644 index 0000000000..f1a293fba6 --- /dev/null +++ b/doc/src/sgml/file-fdw.sgml @@ -0,0 +1,123 @@ + + + + file_fdw + + + file_fdw + + + + The file_fdw module provides foreign-data wrapper + file_fdw which can be used to access + files on the server-side with format accepted by + COPY FROM. + + + + Functions + + + + + + file_fdw_validator(text[], oid) returns bool + + + + + file_fdw_validator is a validator function to + validate generic options of file_fdw and related objects. + + + The location of data file is specified in filename + generic option of the foreign table. + Options valid for COPY FROM can be specified in + generic options of the foreign table, other than + force_not_null and oids. + The force_not_null must be specified in generic + option of the column of the foreign table with boolean value. + The oids is not supported in file_fdw. + + + All of foreign-data wrapper, server or user mapping accepts no generic + option. + + + + + + + file_fdw_handler() returns fdw_handler + + + + + file_fdw_handler is a foreign-data wrapper + handler function which returns foreign-data wrapper handler for + COPY FROM files in type of fdw_handler. + Since fdw_hanlder is a pseudo type, file_fdw_handler can't be + called from a SQL statement. + + + This handler opens the data file pointed by the generic option + filename, and read records from the file along the + format specified with other generic options. The result tuples + are returned to backend one by one, so the memory usage would not depend + on the size of the file. + + + + + + + + + + Details of file_fdw + + + Table-level options + + Changing table-level options requires superuser privilege for security reason. + Non-superuser can't change any table-level option even if the user is the + owner of the table. This restriction is essentially for 'filename' option, but + currently applied to all table-level options. + + + + + Connection management + + The file_fdw doesn't use any connection, so ConnectServer and + FreeFSConnection is not implemented. + + + + + Cost estimation + + The file_fdw estimates the costs of scanning with the size of the file. + + + + + WHERE clause push-down + + The file_fdw can't push down any WHERE clause, so it always reads all of + the records for each scan. If the plan requires to rescan the table, + file_fdw resets the read-pointer and reads the whole of the file again. + + + + + EXPLAIN VERBOSE + + EXPLAIN VERBOSE on the foreign table which uses file_fdw shows the name and + the size of the data file. + + + + + + diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml index b9d4ea59b1..42149dc234 100644 --- a/doc/src/sgml/filelist.sgml +++ b/doc/src/sgml/filelist.sgml @@ -105,6 +105,7 @@ + diff --git a/doc/src/sgml/ref/alter_foreign_data_wrapper.sgml b/doc/src/sgml/ref/alter_foreign_data_wrapper.sgml index 4e9e8a2e28..7e17ef3e23 100644 --- a/doc/src/sgml/ref/alter_foreign_data_wrapper.sgml +++ b/doc/src/sgml/ref/alter_foreign_data_wrapper.sgml @@ -23,6 +23,7 @@ PostgreSQL documentation ALTER FOREIGN DATA WRAPPER name [ VALIDATOR valfunction | NO VALIDATOR ] + [ HANDLER handler | NO HANDLER ] [ OPTIONS ( [ ADD | SET | DROP ] option ['value'] [, ... ]) ] ALTER FOREIGN DATA WRAPPER name OWNER TO new_owner @@ -85,6 +86,29 @@ ALTER FOREIGN DATA WRAPPER name OWN + + HANDLER handler + + + Specifies a new foreign-data wrapper handler function. + + + + + + NO HANDLER + + + This is used to specify that the foreign-data wrapper should no + longer have a handler function. + + + Note that foreign tables which uses a foreign-data wrapper with no + handler can't be used in a SELECT statement. + + + + OPTIONS ( [ ADD | SET | DROP ] option ['value'] [, ... ] ) @@ -127,8 +151,8 @@ ALTER FOREIGN DATA WRAPPER dbi VALIDATOR bob.myvalidator; ALTER FOREIGN DATA WRAPPER conforms to ISO/IEC 9075-9 (SQL/MED). The standard does not specify the - VALIDATOR and OWNER TO variants of the - command. + VALIDATOR, HANDLER and OWNER TO + variants of the command. diff --git a/doc/src/sgml/ref/create_foreign_data_wrapper.sgml b/doc/src/sgml/ref/create_foreign_data_wrapper.sgml index f626d56665..d70321ba77 100644 --- a/doc/src/sgml/ref/create_foreign_data_wrapper.sgml +++ b/doc/src/sgml/ref/create_foreign_data_wrapper.sgml @@ -23,6 +23,7 @@ PostgreSQL documentation CREATE FOREIGN DATA WRAPPER name [ VALIDATOR valfunction | NO VALIDATOR ] + [ HANDLER handler | NO HANDLER ] [ OPTIONS ( option 'value' [, ... ] ) ] @@ -81,6 +82,19 @@ CREATE FOREIGN DATA WRAPPER name + + HANDLER handler + + + handler is the + name of a previously registered function that will be called to + retrieve a set of functions for foreign tables. + The handler function must take no arguments. + The return type must be fdw_handler. + + + + OPTIONS ( option 'value' [, ... ] ) @@ -151,8 +165,8 @@ CREATE FOREIGN DATA WRAPPER mywrapper CREATE FOREIGN DATA WRAPPER conforms to ISO/IEC - 9075-9 (SQL/MED), with the exception that - the VALIDATOR clause is an extension and the + 9075-9 (SQL/MED), with the exception that the VALIDATOR + and HANDLER clauses are extensions and the clauses LIBRARY and LANGUAGE are not yet implemented in PostgreSQL. diff --git a/doc/src/sgml/ref/explain.sgml b/doc/src/sgml/ref/explain.sgml index 3d7d13c0c1..35f9331f95 100644 --- a/doc/src/sgml/ref/explain.sgml +++ b/doc/src/sgml/ref/explain.sgml @@ -129,8 +129,9 @@ ROLLBACK; Display additional information regarding the plan. Specifically, include the output column list for each node in the plan tree, schema-qualify table and function names, always label variables in expressions with - their range table alias, and always print the name of each trigger for - which statistics are displayed. This parameter defaults to + their range table alias, always print the name of each trigger for + which statistics are displayed, and print FDW-specific information for + each ForeignScan node in the plan tree. This parameter defaults to FALSE. diff --git a/doc/src/sgml/ref/lock.sgml b/doc/src/sgml/ref/lock.sgml index 86cd744ea4..1e571ce80c 100644 --- a/doc/src/sgml/ref/lock.sgml +++ b/doc/src/sgml/ref/lock.sgml @@ -108,7 +108,8 @@ LOCK [ TABLE ] [ ONLY ] name [, ... name - The name (optionally schema-qualified) of an existing table to + The name (optionally schema-qualified) of an existing table + or a existing foreign table to lock. If ONLY is specified, only that table is locked. If ONLY is not specified, the table and all its descendant tables (if any) are locked. diff --git a/doc/src/sgml/ref/select.sgml b/doc/src/sgml/ref/select.sgml index 24f8249713..e5f7c863d0 100644 --- a/doc/src/sgml/ref/select.sgml +++ b/doc/src/sgml/ref/select.sgml @@ -195,6 +195,8 @@ TABLE { [ ONLY ] table_name [ * ] | or FOR SHARE requires UPDATE privilege as well (for at least one column of each table so selected). + So you cannot lock contents of a foreign table because only SELECT + privilege can be granted on foreign tables. diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index 3350ca0b6e..f820799ac3 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -93,13 +93,11 @@ typedef struct CopyStateData FILE *copy_file; /* used if copy_dest == COPY_FILE */ StringInfo fe_msgbuf; /* used for all dests during COPY TO, only for * dest == COPY_NEW_FE in COPY FROM */ - bool fe_copy; /* true for all FE copy dests */ bool fe_eof; /* true if detected end of copy data */ EolType eol_type; /* EOL type of input */ int client_encoding; /* remote side's character encoding */ bool need_transcoding; /* client encoding diff from server? */ bool encoding_embeds_ascii; /* ASCII can be non-first byte? */ - uint64 processed; /* # of tuples processed */ /* parameters from the COPY command */ Relation rel; /* relation to copy to or from */ @@ -119,12 +117,17 @@ typedef struct CopyStateData bool *force_quote_flags; /* per-column CSV FQ flags */ bool *force_notnull_flags; /* per-column CSV FNN flags */ - /* these are just for error messages, see copy_in_error_callback */ + /* these are just for error messages, see CopyFromErrorCallback */ const char *cur_relname; /* table name for error messages */ int cur_lineno; /* line number for error messages */ const char *cur_attname; /* current att for error messages */ const char *cur_attval; /* current att value for error messages */ + /* + * Working state for COPY TO/FROM + */ + MemoryContext copycontext; /* per-copy execution context */ + /* * Working state for COPY TO */ @@ -167,15 +170,28 @@ typedef struct CopyStateData char *raw_buf; int raw_buf_index; /* next byte to process */ int raw_buf_len; /* total # of bytes stored */ -} CopyStateData; -typedef CopyStateData *CopyState; + /* + * The definition of input functions and default expressions are stored + * in these variables. + */ + EState *estate; + AttrNumber num_defaults; + bool file_has_oids; + FmgrInfo oid_in_function; + Oid oid_typioparam; + FmgrInfo *in_functions; /* array of input functions for each attrs */ + Oid *typioparams; /* array of element types for in_functions */ + int *defmap; /* array of default att numbers */ + ExprState **defexprs; /* array of default att expressions */ +} CopyStateData; /* DestReceiver for COPY (SELECT) TO */ typedef struct { DestReceiver pub; /* publicly-known function pointers */ CopyState cstate; /* CopyStateData for the command */ + uint64 processed; /* # of tuples processed */ } DR_copy; @@ -248,11 +264,17 @@ static const char BinarySignature[11] = "PGCOPY\n\377\r\n\0"; /* non-export function prototypes */ -static void DoCopyTo(CopyState cstate); -static void CopyTo(CopyState cstate); +static CopyState BeginCopy(bool is_from, Relation rel, Node *raw_query, + const char *queryString, List *attnamelist, List *options); +static void EndCopy(CopyState cstate); +static CopyState BeginCopyTo(Relation rel, Node *query, const char *queryString, + const char *filename, List *attnamelist, List *options); +static void EndCopyTo(CopyState cstate); +static uint64 DoCopyTo(CopyState cstate); +static uint64 CopyTo(CopyState cstate); static void CopyOneRowTo(CopyState cstate, Oid tupleOid, Datum *values, bool *nulls); -static void CopyFrom(CopyState cstate); +static uint64 CopyFrom(CopyState cstate); static bool CopyReadLine(CopyState cstate); static bool CopyReadLineText(CopyState cstate); static int CopyReadAttributesText(CopyState cstate); @@ -724,22 +746,125 @@ DoCopy(const CopyStmt *stmt, const char *queryString) CopyState cstate; bool is_from = stmt->is_from; bool pipe = (stmt->filename == NULL); - List *attnamelist = stmt->attlist; + Relation rel; + uint64 processed; + + /* Disallow file COPY except to superusers. */ + if (!pipe && !superuser()) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("must be superuser to COPY to or from a file"), + errhint("Anyone can COPY to stdout or from stdin. " + "psql's \\copy command also works for anyone."))); + + if (stmt->relation) + { + TupleDesc tupDesc; + AclMode required_access = (is_from ? ACL_INSERT : ACL_SELECT); + RangeTblEntry *rte; + List *attnums; + ListCell *cur; + + Assert(!stmt->query); + + /* Open and lock the relation, using the appropriate lock type. */ + rel = heap_openrv(stmt->relation, + (is_from ? RowExclusiveLock : AccessShareLock)); + + rte = makeNode(RangeTblEntry); + rte->rtekind = RTE_RELATION; + rte->relid = RelationGetRelid(rel); + rte->requiredPerms = required_access; + + tupDesc = RelationGetDescr(rel); + attnums = CopyGetAttnums(tupDesc, rel, stmt->attlist); + foreach (cur, attnums) + { + int attno = lfirst_int(cur) - + FirstLowInvalidHeapAttributeNumber; + + if (is_from) + rte->modifiedCols = bms_add_member(rte->modifiedCols, attno); + else + rte->selectedCols = bms_add_member(rte->selectedCols, attno); + } + ExecCheckRTPerms(list_make1(rte), true); + } + else + { + Assert(stmt->query); + + rel = NULL; + } + + if (is_from) + { + /* check read-only transaction */ + if (XactReadOnly && rel->rd_backend != MyBackendId) + PreventCommandIfReadOnly("COPY FROM"); + + cstate = BeginCopyFrom(rel, stmt->filename, + stmt->attlist, stmt->options); + processed = CopyFrom(cstate); /* copy from file to database */ + EndCopyFrom(cstate); + } + else + { + cstate = BeginCopyTo(rel, stmt->query, queryString, stmt->filename, + stmt->attlist, stmt->options); + processed = DoCopyTo(cstate); /* copy from database to file */ + EndCopyTo(cstate); + } + + /* + * Close the relation. If reading, we can release the AccessShareLock we got; + * if writing, we should hold the lock until end of transaction to ensure that + * updates will be committed before lock is released. + */ + if (rel != NULL) + heap_close(rel, (is_from ? NoLock : AccessShareLock)); + + return processed; +} + +/* + * Common setup routines used by BeginCopyFrom and BeginCopyTo. + */ +static CopyState +BeginCopy(bool is_from, + Relation rel, + Node *raw_query, + const char *queryString, + List *attnamelist, + List *options) +{ + CopyState cstate; List *force_quote = NIL; List *force_notnull = NIL; bool force_quote_all = false; bool format_specified = false; - AclMode required_access = (is_from ? ACL_INSERT : ACL_SELECT); ListCell *option; TupleDesc tupDesc; int num_phys_attrs; - uint64 processed; + MemoryContext oldcontext; /* Allocate workspace and zero all fields */ cstate = (CopyStateData *) palloc0(sizeof(CopyStateData)); + /* + * We allocate everything used by a cstate in a new memory context. + * This would avoid memory leaks repeated uses of COPY in a query. + */ + cstate->copycontext = AllocSetContextCreate(CurrentMemoryContext, + "COPY", + ALLOCSET_DEFAULT_MINSIZE, + ALLOCSET_DEFAULT_INITSIZE, + ALLOCSET_DEFAULT_MAXSIZE); + + oldcontext = MemoryContextSwitchTo(cstate->copycontext); + /* Extract options from the statement node tree */ - foreach(option, stmt->options) + foreach(option, options) { DefElem *defel = (DefElem *) lfirst(option); @@ -980,51 +1105,14 @@ DoCopy(const CopyStmt *stmt, const char *queryString) (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("CSV quote character must not appear in the NULL specification"))); - /* Disallow file COPY except to superusers. */ - if (!pipe && !superuser()) - ereport(ERROR, - (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("must be superuser to COPY to or from a file"), - errhint("Anyone can COPY to stdout or from stdin. " - "psql's \\copy command also works for anyone."))); - - if (stmt->relation) + if (rel) { - RangeTblEntry *rte; - List *attnums; - ListCell *cur; - - Assert(!stmt->query); - cstate->queryDesc = NULL; + Assert(!raw_query); - /* Open and lock the relation, using the appropriate lock type. */ - cstate->rel = heap_openrv(stmt->relation, - (is_from ? RowExclusiveLock : AccessShareLock)); + cstate->rel = rel; tupDesc = RelationGetDescr(cstate->rel); - /* Check relation permissions. */ - rte = makeNode(RangeTblEntry); - rte->rtekind = RTE_RELATION; - rte->relid = RelationGetRelid(cstate->rel); - rte->requiredPerms = required_access; - - attnums = CopyGetAttnums(tupDesc, cstate->rel, attnamelist); - foreach (cur, attnums) - { - int attno = lfirst_int(cur) - FirstLowInvalidHeapAttributeNumber; - - if (is_from) - rte->modifiedCols = bms_add_member(rte->modifiedCols, attno); - else - rte->selectedCols = bms_add_member(rte->selectedCols, attno); - } - ExecCheckRTPerms(list_make1(rte), true); - - /* check read-only transaction */ - if (XactReadOnly && is_from && cstate->rel->rd_backend != MyBackendId) - PreventCommandIfReadOnly("COPY FROM"); - /* Don't allow COPY w/ OIDs to or from a table without them */ if (cstate->oids && !cstate->rel->rd_rel->relhasoids) ereport(ERROR, @@ -1058,7 +1146,7 @@ DoCopy(const CopyStmt *stmt, const char *queryString) * function and is executed repeatedly. (See also the same hack in * DECLARE CURSOR and PREPARE.) XXX FIXME someday. */ - rewritten = pg_analyze_and_rewrite((Node *) copyObject(stmt->query), + rewritten = pg_analyze_and_rewrite((Node *) copyObject(raw_query), queryString, NULL, 0); /* We don't expect more or less than one result query */ @@ -1160,14 +1248,6 @@ DoCopy(const CopyStmt *stmt, const char *queryString) } } - /* Set up variables to avoid per-attribute overhead. */ - initStringInfo(&cstate->attribute_buf); - initStringInfo(&cstate->line_buf); - cstate->line_buf_converted = false; - cstate->raw_buf = (char *) palloc(RAW_BUF_SIZE + 1); - cstate->raw_buf_index = cstate->raw_buf_len = 0; - cstate->processed = 0; - /* * Set up encoding conversion info. Even if the client and server * encodings are the same, we must apply pg_client_to_server() to validate @@ -1181,84 +1261,75 @@ DoCopy(const CopyStmt *stmt, const char *queryString) cstate->encoding_embeds_ascii = PG_ENCODING_IS_CLIENT_ONLY(cstate->client_encoding); cstate->copy_dest = COPY_FILE; /* default */ - cstate->filename = stmt->filename; - if (is_from) - CopyFrom(cstate); /* copy from file to database */ - else - DoCopyTo(cstate); /* copy from database to file */ + MemoryContextSwitchTo(oldcontext); - /* - * Close the relation or query. If reading, we can release the - * AccessShareLock we got; if writing, we should hold the lock until end - * of transaction to ensure that updates will be committed before lock is - * released. - */ - if (cstate->rel) - heap_close(cstate->rel, (is_from ? NoLock : AccessShareLock)); - else - { - /* Close down the query and free resources. */ - ExecutorEnd(cstate->queryDesc); - FreeQueryDesc(cstate->queryDesc); - PopActiveSnapshot(); - } + return cstate; +} - /* Clean up storage (probably not really necessary) */ - processed = cstate->processed; +/* + * Release resources allocated in a cstate. + */ +static void +EndCopy(CopyState cstate) +{ + if (cstate->filename != NULL && FreeFile(cstate->copy_file)) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not close file \"%s\": %m", + cstate->filename))); - pfree(cstate->attribute_buf.data); - pfree(cstate->line_buf.data); - pfree(cstate->raw_buf); + MemoryContextDelete(cstate->copycontext); pfree(cstate); - - return processed; } - /* - * This intermediate routine exists mainly to localize the effects of setjmp - * so we don't need to plaster a lot of variables with "volatile". + * Setup CopyState to read tuples from a table or a query for COPY TO. */ -static void -DoCopyTo(CopyState cstate) +static CopyState +BeginCopyTo(Relation rel, + Node *query, + const char *queryString, + const char *filename, + List *attnamelist, + List *options) { - bool pipe = (cstate->filename == NULL); + CopyState cstate; + bool pipe = (filename == NULL); + MemoryContext oldcontext; - if (cstate->rel) + if (rel != NULL && rel->rd_rel->relkind != RELKIND_RELATION) { - if (cstate->rel->rd_rel->relkind != RELKIND_RELATION) - { - if (cstate->rel->rd_rel->relkind == RELKIND_VIEW) - ereport(ERROR, - (errcode(ERRCODE_WRONG_OBJECT_TYPE), - errmsg("cannot copy from view \"%s\"", - RelationGetRelationName(cstate->rel)), - errhint("Try the COPY (SELECT ...) TO variant."))); - else if (cstate->rel->rd_rel->relkind == RELKIND_FOREIGN_TABLE) - ereport(ERROR, - (errcode(ERRCODE_WRONG_OBJECT_TYPE), - errmsg("cannot copy from foreign table \"%s\"", - RelationGetRelationName(cstate->rel)), - errhint("Try the COPY (SELECT ...) TO variant."))); - else if (cstate->rel->rd_rel->relkind == RELKIND_SEQUENCE) - ereport(ERROR, - (errcode(ERRCODE_WRONG_OBJECT_TYPE), - errmsg("cannot copy from sequence \"%s\"", - RelationGetRelationName(cstate->rel)))); - else - ereport(ERROR, - (errcode(ERRCODE_WRONG_OBJECT_TYPE), - errmsg("cannot copy from non-table relation \"%s\"", - RelationGetRelationName(cstate->rel)))); - } + if (rel->rd_rel->relkind == RELKIND_VIEW) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot copy from view \"%s\"", + RelationGetRelationName(rel)), + errhint("Try the COPY (SELECT ...) TO variant."))); + else if (rel->rd_rel->relkind == RELKIND_FOREIGN_TABLE) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot copy from foreign table \"%s\"", + RelationGetRelationName(rel)), + errhint("Try the COPY (SELECT ...) TO variant."))); + else if (rel->rd_rel->relkind == RELKIND_SEQUENCE) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot copy from sequence \"%s\"", + RelationGetRelationName(rel)))); + else + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot copy from non-table relation \"%s\"", + RelationGetRelationName(rel)))); } + cstate = BeginCopy(false, rel, query, queryString, attnamelist, options); + oldcontext = MemoryContextSwitchTo(cstate->copycontext); + if (pipe) { - if (whereToSendOutput == DestRemote) - cstate->fe_copy = true; - else + if (whereToSendOutput != DestRemote) cstate->copy_file = stdout; } else @@ -1270,11 +1341,12 @@ DoCopyTo(CopyState cstate) * Prevent write to relative path ... too easy to shoot oneself in the * foot by overwriting a database file ... */ - if (!is_absolute_path(cstate->filename)) + if (!is_absolute_path(filename)) ereport(ERROR, (errcode(ERRCODE_INVALID_NAME), errmsg("relative path not allowed for COPY to file"))); + cstate->filename = pstrdup(filename); oumask = umask(S_IWGRP | S_IWOTH); cstate->copy_file = AllocateFile(cstate->filename, PG_BINARY_W); umask(oumask); @@ -1292,14 +1364,30 @@ DoCopyTo(CopyState cstate) errmsg("\"%s\" is a directory", cstate->filename))); } + MemoryContextSwitchTo(oldcontext); + + return cstate; +} + +/* + * This intermediate routine exists mainly to localize the effects of setjmp + * so we don't need to plaster a lot of variables with "volatile". + */ +static uint64 +DoCopyTo(CopyState cstate) +{ + bool pipe = (cstate->filename == NULL); + bool fe_copy = (pipe && whereToSendOutput == DestRemote); + uint64 processed; + PG_TRY(); { - if (cstate->fe_copy) + if (fe_copy) SendCopyBegin(cstate); - CopyTo(cstate); + processed = CopyTo(cstate); - if (cstate->fe_copy) + if (fe_copy) SendCopyEnd(cstate); } PG_CATCH(); @@ -1314,26 +1402,38 @@ DoCopyTo(CopyState cstate) } PG_END_TRY(); - if (!pipe) + return processed; +} + +/* + * Clean up storage and release resources for COPY TO. + */ +static void +EndCopyTo(CopyState cstate) +{ + if (cstate->queryDesc != NULL) { - if (FreeFile(cstate->copy_file)) - ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not close file \"%s\": %m", - cstate->filename))); + /* Close down the query and free resources. */ + ExecutorEnd(cstate->queryDesc); + FreeQueryDesc(cstate->queryDesc); + PopActiveSnapshot(); } + + /* Clean up storage */ + EndCopy(cstate); } /* * Copy from relation or query TO file. */ -static void +static uint64 CopyTo(CopyState cstate) { TupleDesc tupDesc; int num_phys_attrs; Form_pg_attribute *attr; ListCell *cur; + uint64 processed; if (cstate->rel) tupDesc = RelationGetDescr(cstate->rel); @@ -1439,6 +1539,7 @@ CopyTo(CopyState cstate) scandesc = heap_beginscan(cstate->rel, GetActiveSnapshot(), 0, NULL); + processed = 0; while ((tuple = heap_getnext(scandesc, ForwardScanDirection)) != NULL) { CHECK_FOR_INTERRUPTS(); @@ -1448,14 +1549,19 @@ CopyTo(CopyState cstate) /* Format and send the data */ CopyOneRowTo(cstate, HeapTupleGetOid(tuple), values, nulls); + processed++; } heap_endscan(scandesc); + + pfree(values); + pfree(nulls); } else { /* run the plan --- the dest receiver will send tuples */ ExecutorRun(cstate->queryDesc, ForwardScanDirection, 0L); + processed = ((DR_copy *) cstate->queryDesc->dest)->processed; } if (cstate->binary) @@ -1467,6 +1573,8 @@ CopyTo(CopyState cstate) } MemoryContextDelete(cstate->rowcontext); + + return processed; } /* @@ -1558,16 +1666,16 @@ CopyOneRowTo(CopyState cstate, Oid tupleOid, Datum *values, bool *nulls) CopySendEndOfRow(cstate); MemoryContextSwitchTo(oldcontext); - - cstate->processed++; } /* * error context callback for COPY FROM + * + * The argument for the error context must be CopyState. */ -static void -copy_in_error_callback(void *arg) +void +CopyFromErrorCallback(void *arg) { CopyState cstate = (CopyState) arg; @@ -1575,11 +1683,11 @@ copy_in_error_callback(void *arg) { /* can't usefully display the data */ if (cstate->cur_attname) - errcontext("COPY %s, line %d, column %s", + errcontext("relation %s, line %d, column %s", cstate->cur_relname, cstate->cur_lineno, cstate->cur_attname); else - errcontext("COPY %s, line %d", + errcontext("relation %s, line %d", cstate->cur_relname, cstate->cur_lineno); } else @@ -1590,7 +1698,7 @@ copy_in_error_callback(void *arg) char *attval; attval = limit_printout_length(cstate->cur_attval); - errcontext("COPY %s, line %d, column %s: \"%s\"", + errcontext("relation %s, line %d, column %s: \"%s\"", cstate->cur_relname, cstate->cur_lineno, cstate->cur_attname, attval); pfree(attval); @@ -1598,7 +1706,7 @@ copy_in_error_callback(void *arg) else if (cstate->cur_attname) { /* error is relevant to a particular column, value is NULL */ - errcontext("COPY %s, line %d, column %s: null input", + errcontext("relation %s, line %d, column %s: null input", cstate->cur_relname, cstate->cur_lineno, cstate->cur_attname); } @@ -1610,7 +1718,7 @@ copy_in_error_callback(void *arg) char *lineval; lineval = limit_printout_length(cstate->line_buf.data); - errcontext("COPY %s, line %d: \"%s\"", + errcontext("relation %s, line %d: \"%s\"", cstate->cur_relname, cstate->cur_lineno, lineval); pfree(lineval); } @@ -1624,7 +1732,7 @@ copy_in_error_callback(void *arg) * regurgitate it without conversion. So we have to punt and * just report the line number. */ - errcontext("COPY %s, line %d", + errcontext("relation %s, line %d", cstate->cur_relname, cstate->cur_lineno); } } @@ -1669,41 +1777,22 @@ limit_printout_length(const char *str) /* * Copy FROM file to relation. */ -static void +static uint64 CopyFrom(CopyState cstate) { - bool pipe = (cstate->filename == NULL); HeapTuple tuple; TupleDesc tupDesc; - Form_pg_attribute *attr; - AttrNumber num_phys_attrs, - attr_count, - num_defaults; - FmgrInfo *in_functions; - FmgrInfo oid_in_function; - Oid *typioparams; - Oid oid_typioparam; - int attnum; - int i; - Oid in_func_oid; Datum *values; bool *nulls; - int nfields; - char **field_strings; - bool done = false; - bool isnull; ResultRelInfo *resultRelInfo; - EState *estate = CreateExecutorState(); /* for ExecConstraints() */ + EState *estate = cstate->estate; /* for ExecConstraints() */ TupleTableSlot *slot; - bool file_has_oids; - int *defmap; - ExprState **defexprs; /* array of default att expressions */ - ExprContext *econtext; /* used for ExecEvalExpr for default atts */ MemoryContext oldcontext = CurrentMemoryContext; ErrorContextCallback errcontext; CommandId mycid = GetCurrentCommandId(true); int hi_options = 0; /* start with default heap_insert options */ BulkInsertState bistate; + uint64 processed = 0; Assert(cstate->rel); @@ -1731,6 +1820,8 @@ CopyFrom(CopyState cstate) RelationGetRelationName(cstate->rel)))); } + tupDesc = RelationGetDescr(cstate->rel); + /*---------- * Check to see if we can avoid writing WAL * @@ -1766,38 +1857,6 @@ CopyFrom(CopyState cstate) hi_options |= HEAP_INSERT_SKIP_WAL; } - if (pipe) - { - if (whereToSendOutput == DestRemote) - ReceiveCopyBegin(cstate); - else - cstate->copy_file = stdin; - } - else - { - struct stat st; - - cstate->copy_file = AllocateFile(cstate->filename, PG_BINARY_R); - - if (cstate->copy_file == NULL) - ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not open file \"%s\" for reading: %m", - cstate->filename))); - - fstat(fileno(cstate->copy_file), &st); - if (S_ISDIR(st.st_mode)) - ereport(ERROR, - (errcode(ERRCODE_WRONG_OBJECT_TYPE), - errmsg("\"%s\" is a directory", cstate->filename))); - } - - tupDesc = RelationGetDescr(cstate->rel); - attr = tupDesc->attrs; - num_phys_attrs = tupDesc->natts; - attr_count = list_length(cstate->attnumlist); - num_defaults = 0; - /* * We need a ResultRelInfo so we can use the regular executor's * index-entry-making machinery. (There used to be a huge amount of code @@ -1826,7 +1885,187 @@ CopyFrom(CopyState cstate) slot = ExecInitExtraTupleSlot(estate); ExecSetSlotDescriptor(slot, tupDesc); - econtext = GetPerTupleExprContext(estate); + /* Prepare to catch AFTER triggers. */ + AfterTriggerBeginQuery(); + + /* + * Check BEFORE STATEMENT insertion triggers. It's debateable whether we + * should do this for COPY, since it's not really an "INSERT" statement as + * such. However, executing these triggers maintains consistency with the + * EACH ROW triggers that we already fire on COPY. + */ + ExecBSInsertTriggers(estate, resultRelInfo); + + values = (Datum *) palloc(tupDesc->natts * sizeof(Datum)); + nulls = (bool *) palloc(tupDesc->natts * sizeof(bool)); + + bistate = GetBulkInsertState(); + + /* Set up callback to identify error line number */ + errcontext.callback = CopyFromErrorCallback; + errcontext.arg = (void *) cstate; + errcontext.previous = error_context_stack; + error_context_stack = &errcontext; + + for (;;) + { + bool skip_tuple; + Oid loaded_oid = InvalidOid; + + CHECK_FOR_INTERRUPTS(); + + /* Reset the per-tuple exprcontext */ + ResetPerTupleExprContext(estate); + + /* Switch into its memory context */ + MemoryContextSwitchTo(GetPerTupleMemoryContext(estate)); + + if (!NextCopyFrom(cstate, values, nulls, &loaded_oid)) + break; + + /* And now we can form the input tuple. */ + tuple = heap_form_tuple(tupDesc, values, nulls); + + if (loaded_oid != InvalidOid) + HeapTupleSetOid(tuple, loaded_oid); + + /* Triggers and stuff need to be invoked in query context. */ + MemoryContextSwitchTo(oldcontext); + + skip_tuple = false; + + /* BEFORE ROW INSERT Triggers */ + if (resultRelInfo->ri_TrigDesc && + resultRelInfo->ri_TrigDesc->trig_insert_before_row) + { + HeapTuple newtuple; + + newtuple = ExecBRInsertTriggers(estate, resultRelInfo, tuple); + + if (newtuple == NULL) /* "do nothing" */ + skip_tuple = true; + else if (newtuple != tuple) /* modified by Trigger(s) */ + { + heap_freetuple(tuple); + tuple = newtuple; + } + } + + if (!skip_tuple) + { + List *recheckIndexes = NIL; + + /* Place tuple in tuple slot */ + ExecStoreTuple(tuple, slot, InvalidBuffer, false); + + /* Check the constraints of the tuple */ + if (cstate->rel->rd_att->constr) + ExecConstraints(resultRelInfo, slot, estate); + + /* OK, store the tuple and create index entries for it */ + heap_insert(cstate->rel, tuple, mycid, hi_options, bistate); + + if (resultRelInfo->ri_NumIndices > 0) + recheckIndexes = ExecInsertIndexTuples(slot, &(tuple->t_self), + estate); + + /* AFTER ROW INSERT Triggers */ + ExecARInsertTriggers(estate, resultRelInfo, tuple, + recheckIndexes); + + list_free(recheckIndexes); + + /* + * We count only tuples not suppressed by a BEFORE INSERT trigger; + * this is the same definition used by execMain.c for counting + * tuples inserted by an INSERT command. + */ + processed++; + } + } + + /* Done, clean up */ + error_context_stack = errcontext.previous; + + FreeBulkInsertState(bistate); + + MemoryContextSwitchTo(oldcontext); + + /* Execute AFTER STATEMENT insertion triggers */ + ExecASInsertTriggers(estate, resultRelInfo); + + /* Handle queued AFTER triggers */ + AfterTriggerEndQuery(estate); + + pfree(values); + pfree(nulls); + + ExecResetTupleTable(estate->es_tupleTable, false); + + ExecCloseIndices(resultRelInfo); + + /* + * If we skipped writing WAL, then we need to sync the heap (but not + * indexes since those use WAL anyway) + */ + if (hi_options & HEAP_INSERT_SKIP_WAL) + heap_sync(cstate->rel); + + return processed; +} + +/* + * CopyGetAttnums - build an integer list of attnums to be copied + * + * The input attnamelist is either the user-specified column list, + * or NIL if there was none (in which case we want all the non-dropped + * columns). + * + * rel can be NULL ... it's only used for error reports. + */ +CopyState +BeginCopyFrom(Relation rel, + const char *filename, + List *attnamelist, + List *options) +{ + CopyState cstate; + bool pipe = (filename == NULL); + TupleDesc tupDesc; + Form_pg_attribute *attr; + AttrNumber num_phys_attrs, + num_defaults; + FmgrInfo *in_functions; + Oid *typioparams; + int attnum; + Oid in_func_oid; + EState *estate = CreateExecutorState(); /* for ExecPrepareExpr() */ + int *defmap; + ExprState **defexprs; + MemoryContext oldcontext; + + cstate = BeginCopy(true, rel, NULL, NULL, attnamelist, options); + oldcontext = MemoryContextSwitchTo(cstate->copycontext); + + /* Initialize state variables */ + cstate->fe_eof = false; + cstate->eol_type = EOL_UNKNOWN; + cstate->cur_relname = RelationGetRelationName(cstate->rel); + cstate->cur_lineno = 0; + cstate->cur_attname = NULL; + cstate->cur_attval = NULL; + + /* Set up variables to avoid per-attribute overhead. */ + initStringInfo(&cstate->attribute_buf); + initStringInfo(&cstate->line_buf); + cstate->line_buf_converted = false; + cstate->raw_buf = (char *) palloc(RAW_BUF_SIZE + 1); + cstate->raw_buf_index = cstate->raw_buf_len = 0; + + tupDesc = RelationGetDescr(cstate->rel); + attr = tupDesc->attrs; + num_phys_attrs = tupDesc->natts; + num_defaults = 0; /* * Pick up the required catalog information for each attribute in the @@ -1871,19 +2110,46 @@ CopyFrom(CopyState cstate) } } - /* Prepare to catch AFTER triggers. */ - AfterTriggerBeginQuery(); + /* We keep those variables in cstate. */ + cstate->estate = estate; + cstate->in_functions = in_functions; + cstate->typioparams = typioparams; + cstate->defmap = defmap; + cstate->defexprs = defexprs; + cstate->num_defaults = num_defaults; - /* - * Check BEFORE STATEMENT insertion triggers. It's debateable whether we - * should do this for COPY, since it's not really an "INSERT" statement as - * such. However, executing these triggers maintains consistency with the - * EACH ROW triggers that we already fire on COPY. - */ - ExecBSInsertTriggers(estate, resultRelInfo); + if (pipe) + { + if (whereToSendOutput == DestRemote) + ReceiveCopyBegin(cstate); + else + cstate->copy_file = stdin; + } + else + { + struct stat st; + + cstate->filename = pstrdup(filename); + cstate->copy_file = AllocateFile(cstate->filename, PG_BINARY_R); + + if (cstate->copy_file == NULL) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not open file \"%s\" for reading: %m", + cstate->filename))); + + fstat(fileno(cstate->copy_file), &st); + if (S_ISDIR(st.st_mode)) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("\"%s\" is a directory", cstate->filename))); + } if (!cstate->binary) - file_has_oids = cstate->oids; /* must rely on user to tell us... */ + { + /* must rely on user to tell us... */ + cstate->file_has_oids = cstate->oids; + } else { /* Read and verify binary header */ @@ -1901,7 +2167,7 @@ CopyFrom(CopyState cstate) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), errmsg("invalid COPY file header (missing flags)"))); - file_has_oids = (tmp & (1 << 16)) != 0; + cstate->file_has_oids = (tmp & (1 << 16)) != 0; tmp &= ~(1 << 16); if ((tmp >> 16) != 0) ereport(ERROR, @@ -1923,73 +2189,61 @@ CopyFrom(CopyState cstate) } } - if (file_has_oids && cstate->binary) + if (cstate->file_has_oids && cstate->binary) { getTypeBinaryInputInfo(OIDOID, - &in_func_oid, &oid_typioparam); - fmgr_info(in_func_oid, &oid_in_function); + &in_func_oid, &cstate->oid_typioparam); + fmgr_info(in_func_oid, &cstate->oid_in_function); } - values = (Datum *) palloc(num_phys_attrs * sizeof(Datum)); - nulls = (bool *) palloc(num_phys_attrs * sizeof(bool)); - /* create workspace for CopyReadAttributes results */ - nfields = file_has_oids ? (attr_count + 1) : attr_count; - if (! cstate->binary) + if (!cstate->binary) { + AttrNumber attr_count = list_length(cstate->attnumlist); + int nfields = cstate->file_has_oids ? (attr_count + 1) : attr_count; + cstate->max_fields = nfields; cstate->raw_fields = (char **) palloc(nfields * sizeof(char *)); } - /* Initialize state variables */ - cstate->fe_eof = false; - cstate->eol_type = EOL_UNKNOWN; - cstate->cur_relname = RelationGetRelationName(cstate->rel); - cstate->cur_lineno = 0; - cstate->cur_attname = NULL; - cstate->cur_attval = NULL; + MemoryContextSwitchTo(oldcontext); - bistate = GetBulkInsertState(); + return cstate; +} - /* Set up callback to identify error line number */ - errcontext.callback = copy_in_error_callback; - errcontext.arg = (void *) cstate; - errcontext.previous = error_context_stack; - error_context_stack = &errcontext; +/* + * Read raw fields in the next line for COPY FROM in text or csv mode. + * Return false if no more lines. + * + * An internal buffer is returned via fields, so the caller must not free it. + * Since the function returns all fields in the input file, nfields could be + * smaller or larger than the number of columns. Oid of the tuple is returned + * with tupleOid separately. + */ + bool +NextLineCopyFrom(CopyState cstate, char ***fields, int *nfields, Oid *tupleOid) +{ + char **field_strings; + ListCell *cur; + int fldct; + int fieldno; + char *string; + bool done; + + /* only available for text or csv input */ + Assert(!cstate->binary); /* on input just throw the header line away */ - if (cstate->header_line) + if (cstate->cur_lineno == 0 && cstate->header_line) { cstate->cur_lineno++; - done = CopyReadLine(cstate); + if (CopyReadLine(cstate)) + return false; /* done */ } - while (!done) - { - bool skip_tuple; - Oid loaded_oid = InvalidOid; - - CHECK_FOR_INTERRUPTS(); - + /* XXX: Indentation is not adjusted to keep the patch small. */ cstate->cur_lineno++; - /* Reset the per-tuple exprcontext */ - ResetPerTupleExprContext(estate); - - /* Switch into its memory context */ - MemoryContextSwitchTo(GetPerTupleMemoryContext(estate)); - - /* Initialize all values for row to NULL */ - MemSet(values, 0, num_phys_attrs * sizeof(Datum)); - MemSet(nulls, true, num_phys_attrs * sizeof(bool)); - - if (!cstate->binary) - { - ListCell *cur; - int fldct; - int fieldno; - char *string; - /* Actually read the line into memory here */ done = CopyReadLine(cstate); @@ -1999,7 +2253,7 @@ CopyFrom(CopyState cstate) * EOF, ie, process the line and then exit loop on next iteration. */ if (done && cstate->line_buf.len == 0) - break; + return false; /* Parse the line into de-escaped field values */ if (cstate->csv_mode) @@ -2007,17 +2261,11 @@ CopyFrom(CopyState cstate) else fldct = CopyReadAttributesText(cstate); - /* check for overflowing fields */ - if (nfields > 0 && fldct > nfields) - ereport(ERROR, - (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), - errmsg("extra data after last expected column"))); - fieldno = 0; field_strings = cstate->raw_fields; /* Read the OID field if present */ - if (file_has_oids) + if (cstate->file_has_oids) { if (fieldno >= fldct) ereport(ERROR, @@ -2029,13 +2277,13 @@ CopyFrom(CopyState cstate) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), errmsg("null OID in COPY data"))); - else + else if (cstate->oids && tupleOid != NULL) { cstate->cur_attname = "oid"; cstate->cur_attval = string; - loaded_oid = DatumGetObjectId(DirectFunctionCall1(oidin, - CStringGetDatum(string))); - if (loaded_oid == InvalidOid) + *tupleOid = DatumGetObjectId(DirectFunctionCall1(oidin, + CStringGetDatum(string))); + if (*tupleOid == InvalidOid) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), errmsg("invalid OID in COPY data"))); @@ -2044,6 +2292,9 @@ CopyFrom(CopyState cstate) } } + *fields = &field_strings[fieldno]; + *nfields = fldct - fieldno; + /* Loop to read the user attributes on the line. */ foreach(cur, cstate->attnumlist) { @@ -2051,19 +2302,93 @@ CopyFrom(CopyState cstate) int m = attnum - 1; if (fieldno >= fldct) - ereport(ERROR, - (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), - errmsg("missing data for column \"%s\"", - NameStr(attr[m]->attname)))); - string = field_strings[fieldno++]; + break; - if (cstate->csv_mode && string == NULL && + /* + * FIXME: force_notnull_flags won't work for extra columsn + * because we cannot specify it for unnamed columns... + */ + if (cstate->csv_mode && field_strings[fieldno] == NULL && cstate->force_notnull_flags[m]) { /* Go ahead and read the NULL string */ - string = cstate->null_print; + field_strings[fieldno] = cstate->null_print; } + fieldno++; + } + /* XXX: End of only-indentation changes. */ + + return true; +} + +/* + * Read next tuple from file for COPY FROM. Return false if no more tuples. + * + * values and nulls arrays must be the same length as columns of the + * relation passed to BeginCopyFrom. Oid of the tuple is returned with + * tupleOid separately. + */ +bool +NextCopyFrom(CopyState cstate, Datum *values, bool *nulls, Oid *tupleOid) +{ + TupleDesc tupDesc; + Form_pg_attribute *attr; + AttrNumber num_phys_attrs, + attr_count, + num_defaults = cstate->num_defaults; + FmgrInfo *in_functions = cstate->in_functions; + Oid *typioparams = cstate->typioparams; + int i; + bool isnull; + bool file_has_oids = cstate->file_has_oids; + int *defmap = cstate->defmap; + ExprState **defexprs = cstate->defexprs; + ExprContext *econtext; /* used for ExecEvalExpr for default atts */ + + tupDesc = RelationGetDescr(cstate->rel); + attr = tupDesc->attrs; + num_phys_attrs = tupDesc->natts; + attr_count = list_length(cstate->attnumlist); + + /* XXX: Indentation is not adjusted to keep the patch small. */ + /* Initialize all values for row to NULL */ + MemSet(values, 0, num_phys_attrs * sizeof(Datum)); + MemSet(nulls, true, num_phys_attrs * sizeof(bool)); + + if (!cstate->binary) + { + char **field_strings; + ListCell *cur; + int fldct; + int fieldno; + char *string; + + /* read raw fields in the next line */ + if (!NextLineCopyFrom(cstate, &field_strings, &fldct, tupleOid)) + return false; + + /* check for overflowing fields */ + if (attr_count > 0 && fldct > attr_count) + ereport(ERROR, + (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), + errmsg("extra data after last expected column"))); + + fieldno = 0; + + /* Loop to read the user attributes on the line. */ + foreach(cur, cstate->attnumlist) + { + int attnum = lfirst_int(cur); + int m = attnum - 1; + + if (fieldno >= fldct) + ereport(ERROR, + (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), + errmsg("missing data for column \"%s\"", + NameStr(attr[m]->attname)))); + string = field_strings[fieldno++]; + cstate->cur_attname = NameStr(attr[m]->attname); cstate->cur_attval = string; values[m] = InputFunctionCall(&in_functions[m], @@ -2076,7 +2401,7 @@ CopyFrom(CopyState cstate) cstate->cur_attval = NULL; } - Assert(fieldno == nfields); + Assert(fieldno == attr_count); } else { @@ -2084,11 +2409,12 @@ CopyFrom(CopyState cstate) int16 fld_count; ListCell *cur; + cstate->cur_lineno++; + if (!CopyGetInt16(cstate, &fld_count)) { /* EOF detected (end of file, or protocol-level EOF) */ - done = true; - break; + return false; } if (fld_count == -1) @@ -2112,8 +2438,7 @@ CopyFrom(CopyState cstate) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), errmsg("received copy data after EOF marker"))); - done = true; - break; + return false; } if (fld_count != attr_count) @@ -2124,12 +2449,14 @@ CopyFrom(CopyState cstate) if (file_has_oids) { + Oid loaded_oid; + cstate->cur_attname = "oid"; loaded_oid = DatumGetObjectId(CopyReadBinaryAttribute(cstate, 0, - &oid_in_function, - oid_typioparam, + &cstate->oid_in_function, + cstate->oid_typioparam, -1, &isnull)); if (isnull || loaded_oid == InvalidOid) @@ -2137,6 +2464,8 @@ CopyFrom(CopyState cstate) (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), errmsg("invalid OID in COPY data"))); cstate->cur_attname = NULL; + if (cstate->oids && tupleOid != NULL) + *tupleOid = loaded_oid; } i = 0; @@ -2162,117 +2491,27 @@ CopyFrom(CopyState cstate) * provided by the input data. Anything not processed here or above * will remain NULL. */ + econtext = GetPerTupleExprContext(cstate->estate); for (i = 0; i < num_defaults; i++) { values[defmap[i]] = ExecEvalExpr(defexprs[i], econtext, &nulls[defmap[i]], NULL); } + /* XXX: End of only-indentation changes. */ - /* And now we can form the input tuple. */ - tuple = heap_form_tuple(tupDesc, values, nulls); - - if (cstate->oids && file_has_oids) - HeapTupleSetOid(tuple, loaded_oid); - - /* Triggers and stuff need to be invoked in query context. */ - MemoryContextSwitchTo(oldcontext); - - skip_tuple = false; - - /* BEFORE ROW INSERT Triggers */ - if (resultRelInfo->ri_TrigDesc && - resultRelInfo->ri_TrigDesc->trig_insert_before_row) - { - HeapTuple newtuple; - - newtuple = ExecBRInsertTriggers(estate, resultRelInfo, tuple); - - if (newtuple == NULL) /* "do nothing" */ - skip_tuple = true; - else if (newtuple != tuple) /* modified by Trigger(s) */ - { - heap_freetuple(tuple); - tuple = newtuple; - } - } - - if (!skip_tuple) - { - List *recheckIndexes = NIL; - - /* Place tuple in tuple slot */ - ExecStoreTuple(tuple, slot, InvalidBuffer, false); - - /* Check the constraints of the tuple */ - if (cstate->rel->rd_att->constr) - ExecConstraints(resultRelInfo, slot, estate); - - /* OK, store the tuple and create index entries for it */ - heap_insert(cstate->rel, tuple, mycid, hi_options, bistate); - - if (resultRelInfo->ri_NumIndices > 0) - recheckIndexes = ExecInsertIndexTuples(slot, &(tuple->t_self), - estate); - - /* AFTER ROW INSERT Triggers */ - ExecARInsertTriggers(estate, resultRelInfo, tuple, - recheckIndexes); - - list_free(recheckIndexes); - - /* - * We count only tuples not suppressed by a BEFORE INSERT trigger; - * this is the same definition used by execMain.c for counting - * tuples inserted by an INSERT command. - */ - cstate->processed++; - } - } - - /* Done, clean up */ - error_context_stack = errcontext.previous; - - FreeBulkInsertState(bistate); - - MemoryContextSwitchTo(oldcontext); - - /* Execute AFTER STATEMENT insertion triggers */ - ExecASInsertTriggers(estate, resultRelInfo); - - /* Handle queued AFTER triggers */ - AfterTriggerEndQuery(estate); - - pfree(values); - pfree(nulls); - if (! cstate->binary) - pfree(cstate->raw_fields); - - pfree(in_functions); - pfree(typioparams); - pfree(defmap); - pfree(defexprs); - - ExecResetTupleTable(estate->es_tupleTable, false); - - ExecCloseIndices(resultRelInfo); - - FreeExecutorState(estate); + return true; +} - if (!pipe) - { - if (FreeFile(cstate->copy_file)) - ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not close file \"%s\": %m", - cstate->filename))); - } +/* + * Clean up storage and release resources for COPY FROM. + */ +void +EndCopyFrom(CopyState cstate) +{ + FreeExecutorState(cstate->estate); - /* - * If we skipped writing WAL, then we need to sync the heap (but not - * indexes since those use WAL anyway) - */ - if (hi_options & HEAP_INSERT_SKIP_WAL) - heap_sync(cstate->rel); + /* Clean up storage */ + EndCopy(cstate); } @@ -3537,6 +3776,7 @@ copy_dest_receive(TupleTableSlot *slot, DestReceiver *self) /* And send the data */ CopyOneRowTo(cstate, InvalidOid, slot->tts_values, slot->tts_isnull); + myState->processed++; } /* diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c index 65345907d0..3f003c84c8 100644 --- a/src/backend/commands/explain.c +++ b/src/backend/commands/explain.c @@ -705,6 +705,9 @@ ExplainNode(PlanState *planstate, List *ancestors, case T_WorkTableScan: pname = sname = "WorkTable Scan"; break; + case T_ForeignScan: + pname = sname = "Foreign Scan"; + break; case T_Material: pname = sname = "Materialize"; break; @@ -854,6 +857,7 @@ ExplainNode(PlanState *planstate, List *ancestors, case T_ValuesScan: case T_CteScan: case T_WorkTableScan: + case T_ForeignScan: ExplainScanTarget((Scan *) plan, es); break; case T_BitmapIndexScan: @@ -1033,6 +1037,7 @@ ExplainNode(PlanState *planstate, List *ancestors, case T_ValuesScan: case T_CteScan: case T_WorkTableScan: + case T_ForeignScan: case T_SubqueryScan: show_scan_qual(plan->qual, "Filter", planstate, ancestors, es); break; @@ -1100,6 +1105,14 @@ ExplainNode(PlanState *planstate, List *ancestors, break; } + /* Show FDW specific information, if any */ + if (IsA(plan, ForeignScan)) + { + ForeignScan *scan = (ForeignScan *) plan; + if (scan->fplan->explainInfo) + ExplainPropertyText("FDW-Info", scan->fplan->explainInfo, es); + } + /* Show buffer usage */ if (es->buffers) { @@ -1570,6 +1583,7 @@ ExplainScanTarget(Scan *plan, ExplainState *es) case T_IndexScan: case T_BitmapHeapScan: case T_TidScan: + case T_ForeignScan: /* Assert it's on a real relation */ Assert(rte->rtekind == RTE_RELATION); objectname = get_rel_name(rte->relid); diff --git a/src/backend/commands/foreigncmds.c b/src/backend/commands/foreigncmds.c index 3a0ea9a632..e9248f64c0 100644 --- a/src/backend/commands/foreigncmds.c +++ b/src/backend/commands/foreigncmds.c @@ -317,16 +317,69 @@ AlterForeignServerOwner(const char *name, Oid newOwnerId) * Convert a validator function name passed from the parser to an Oid. */ static Oid -lookup_fdw_validator_func(List *validator) +lookup_fdw_validator_func(DefElem *validator) { Oid funcargtypes[2]; + if (validator == NULL || validator->arg == NULL) + return InvalidOid; + funcargtypes[0] = TEXTARRAYOID; funcargtypes[1] = OIDOID; - return LookupFuncName(validator, 2, funcargtypes, false); + return LookupFuncName((List *) validator->arg, 2, funcargtypes, false); /* return value is ignored, so we don't check the type */ } +static Oid +lookup_fdw_handler_func(DefElem *handler) +{ + Oid handlerOid; + + if (handler == NULL || handler->arg == NULL) + return InvalidOid; + + /* check that handler have correct return type */ + handlerOid = LookupFuncName((List *) handler->arg, 0, NULL, false); + if (get_func_rettype(handlerOid) != FDW_HANDLEROID) + { + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("function %s must return type \"fdw_handler\"", + NameListToString((List *) handler->arg)))); + } + + return handlerOid; +} + +static void +parse_func_options(List *func_options, DefElem **validator, DefElem **handler) +{ + ListCell *cell; + + *validator = NULL; + *handler = NULL; + foreach (cell, func_options) + { + DefElem *def = lfirst(cell); + + if (pg_strcasecmp(def->defname, "validator") == 0) + { + if (*validator) + ereport(ERROR, (errmsg("duplicated VALIDATOR"))); + *validator = def; + } + else if (pg_strcasecmp(def->defname, "handler") == 0) + { + if (*handler) + ereport(ERROR, (errmsg("duplicated HANDLER"))); + *handler = def; + } + else + { + ereport(ERROR, (errmsg("invalid option"))); + } + } +} /* * Create a foreign-data wrapper @@ -339,7 +392,10 @@ CreateForeignDataWrapper(CreateFdwStmt *stmt) bool nulls[Natts_pg_foreign_data_wrapper]; HeapTuple tuple; Oid fdwId; + DefElem *defvalidator; + DefElem *defhandler; Oid fdwvalidator; + Oid fdwhandler; Datum fdwoptions; Oid ownerId; @@ -375,12 +431,13 @@ CreateForeignDataWrapper(CreateFdwStmt *stmt) DirectFunctionCall1(namein, CStringGetDatum(stmt->fdwname)); values[Anum_pg_foreign_data_wrapper_fdwowner - 1] = ObjectIdGetDatum(ownerId); - if (stmt->validator) - fdwvalidator = lookup_fdw_validator_func(stmt->validator); - else - fdwvalidator = InvalidOid; + /* determine which validator to be used (or not used at all) */ + parse_func_options(stmt->func_options, &defvalidator, &defhandler); + fdwvalidator = lookup_fdw_validator_func(defvalidator); + fdwhandler = lookup_fdw_handler_func(defhandler); values[Anum_pg_foreign_data_wrapper_fdwvalidator - 1] = fdwvalidator; + values[Anum_pg_foreign_data_wrapper_fdwhandler - 1] = fdwhandler; nulls[Anum_pg_foreign_data_wrapper_fdwacl - 1] = true; @@ -416,6 +473,21 @@ CreateForeignDataWrapper(CreateFdwStmt *stmt) recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); } + if (fdwhandler != InvalidOid) + { + ObjectAddress myself; + ObjectAddress referenced; + + myself.classId = ForeignDataWrapperRelationId; + myself.objectId = fdwId; + myself.objectSubId = 0; + + referenced.classId = ProcedureRelationId; + referenced.objectId = fdwhandler; + referenced.objectSubId = 0; + recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + } + recordDependencyOnOwner(ForeignDataWrapperRelationId, fdwId, ownerId); /* Post creation hook for new foreign data wrapper */ @@ -440,7 +512,10 @@ AlterForeignDataWrapper(AlterFdwStmt *stmt) Oid fdwId; bool isnull; Datum datum; + DefElem *defvalidator; + DefElem *defhandler; Oid fdwvalidator; + Oid fdwhandler; /* Must be super user */ if (!superuser()) @@ -464,9 +539,11 @@ AlterForeignDataWrapper(AlterFdwStmt *stmt) memset(repl_null, false, sizeof(repl_null)); memset(repl_repl, false, sizeof(repl_repl)); - if (stmt->change_validator) + parse_func_options(stmt->func_options, &defvalidator, &defhandler); + + if (defvalidator) { - fdwvalidator = stmt->validator ? lookup_fdw_validator_func(stmt->validator) : InvalidOid; + fdwvalidator = lookup_fdw_validator_func(defvalidator); repl_val[Anum_pg_foreign_data_wrapper_fdwvalidator - 1] = ObjectIdGetDatum(fdwvalidator); repl_repl[Anum_pg_foreign_data_wrapper_fdwvalidator - 1] = true; @@ -474,7 +551,7 @@ AlterForeignDataWrapper(AlterFdwStmt *stmt) * It could be that the options for the FDW, SERVER and USER MAPPING * are no longer valid with the new validator. Warn about this. */ - if (stmt->validator) + if (defvalidator->arg) ereport(WARNING, (errmsg("changing the foreign-data wrapper validator can cause " "the options for dependent objects to become invalid"))); @@ -492,6 +569,34 @@ AlterForeignDataWrapper(AlterFdwStmt *stmt) fdwvalidator = DatumGetObjectId(datum); } + if (defhandler) + { + fdwhandler = lookup_fdw_handler_func(defhandler); + repl_val[Anum_pg_foreign_data_wrapper_fdwhandler - 1] = ObjectIdGetDatum(fdwhandler); + repl_repl[Anum_pg_foreign_data_wrapper_fdwhandler - 1] = true; + + /* + * It could be that the behavior of accessing foreign table changes + * with the new handler. Warn about this. + */ + if (defhandler->arg) + ereport(WARNING, + (errmsg("changing the foreign-data wrapper handler would change " + "the behavior of accessing foreign tables"))); + } + else + { + /* + * Validator is not changed, but we need it for validating options. + */ + datum = SysCacheGetAttr(FOREIGNDATAWRAPPEROID, + tp, + Anum_pg_foreign_data_wrapper_fdwhandler, + &isnull); + Assert(!isnull); + fdwhandler = DatumGetObjectId(datum); + } + /* * Options specified, validate and update. */ diff --git a/src/backend/executor/Makefile b/src/backend/executor/Makefile index da4fbb440b..a854c9a5dc 100644 --- a/src/backend/executor/Makefile +++ b/src/backend/executor/Makefile @@ -23,6 +23,6 @@ OBJS = execAmi.o execCurrent.o execGrouping.o execJunk.o execMain.o \ nodeSeqscan.o nodeSetOp.o nodeSort.o nodeUnique.o \ nodeValuesscan.o nodeCtescan.o nodeWorktablescan.o \ nodeGroup.o nodeSubplan.o nodeSubqueryscan.o nodeTidscan.o \ - nodeWindowAgg.o tstoreReceiver.o spi.o + nodeForeignscan.o nodeWindowAgg.o tstoreReceiver.o spi.o include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/executor/execAmi.c b/src/backend/executor/execAmi.c index 6bdc60c352..b27de28e03 100644 --- a/src/backend/executor/execAmi.c +++ b/src/backend/executor/execAmi.c @@ -22,6 +22,7 @@ #include "executor/nodeBitmapOr.h" #include "executor/nodeCtescan.h" #include "executor/nodeFunctionscan.h" +#include "executor/nodeForeignscan.h" #include "executor/nodeGroup.h" #include "executor/nodeGroup.h" #include "executor/nodeHash.h" @@ -186,6 +187,10 @@ ExecReScan(PlanState *node) ExecReScanWorkTableScan((WorkTableScanState *) node); break; + case T_ForeignScanState: + ExecForeignReScan((ForeignScanState *) node); + break; + case T_NestLoopState: ExecReScanNestLoop((NestLoopState *) node); break; diff --git a/src/backend/executor/execProcnode.c b/src/backend/executor/execProcnode.c index e8ebec1234..92b83ac2ef 100644 --- a/src/backend/executor/execProcnode.c +++ b/src/backend/executor/execProcnode.c @@ -86,6 +86,7 @@ #include "executor/nodeBitmapOr.h" #include "executor/nodeCtescan.h" #include "executor/nodeFunctionscan.h" +#include "executor/nodeForeignscan.h" #include "executor/nodeGroup.h" #include "executor/nodeHash.h" #include "executor/nodeHashjoin.h" @@ -232,6 +233,11 @@ ExecInitNode(Plan *node, EState *estate, int eflags) estate, eflags); break; + case T_ForeignScan: + result = (PlanState *) ExecInitForeignScan((ForeignScan *) node, + estate, eflags); + break; + /* * join nodes */ @@ -422,6 +428,10 @@ ExecProcNode(PlanState *node) result = ExecWorkTableScan((WorkTableScanState *) node); break; + case T_ForeignScanState: + result = ExecForeignScan((ForeignScanState *) node); + break; + /* * join nodes */ @@ -650,6 +660,10 @@ ExecEndNode(PlanState *node) ExecEndWorkTableScan((WorkTableScanState *) node); break; + case T_ForeignScanState: + ExecEndForeignScan((ForeignScanState *) node); + break; + /* * join nodes */ diff --git a/src/backend/executor/nodeForeignscan.c b/src/backend/executor/nodeForeignscan.c new file mode 100644 index 0000000000..c346eeb4ce --- /dev/null +++ b/src/backend/executor/nodeForeignscan.c @@ -0,0 +1,290 @@ +/*------------------------------------------------------------------------- + * + * nodeForeignscan.c + * Support routines for sequential scans of foreign tables. + * + * Portions Copyright (c) 1996-2011, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/backend/executor/nodeForeignscan.c + * + *------------------------------------------------------------------------- + */ +/* + * INTERFACE ROUTINES + * ExecForeignScan sequentially scans a foreign table. + * ExecForeignNext retrieve next tuple in sequential order. + * ExecInitForeignScan creates and initializes a seqscan node. + * ExecEndForeignScan releases any storage allocated. + * ExecForeignReScan rescans the foreign table + * ExecForeignMarkPos marks scan position + * ExecForeignRestrPos restores scan position + */ +#include "postgres.h" + +#include "executor/executor.h" +#include "executor/nodeForeignscan.h" +#include "foreign/foreign.h" +#include "miscadmin.h" +#include "utils/memutils.h" + +static TupleTableSlot *ForeignNext(ForeignScanState *node); +static bool ForeignRecheck(ForeignScanState *node, TupleTableSlot *slot); + +/* ---------------------------------------------------------------- + * Scan Support + * ---------------------------------------------------------------- + */ + +/* ---------------------------------------------------------------- + * ForeignNext + * + * This is a workhorse for ExecForeignScan + * ---------------------------------------------------------------- + */ +static TupleTableSlot * +ForeignNext(ForeignScanState *node) +{ + TupleTableSlot *slot = node->ss.ss_ScanTupleSlot; + ExprContext *econtext; + MemoryContext oldcontext; + + Assert(node->ss.ps.state->es_direction == ForwardScanDirection); + + /* tupleslot will be filled by Iterate. */ + if (node->routine->Iterate == NULL) + ereport(ERROR, + (errmsg("foreign-data wrapper must support Iterate to scan foreign table"))); + + /* Slot has to be cleared explicitly before resetting per-tuple context. */ + ExecClearTuple(slot); + + /* We call Iterate in per-tuple context, similar to FunctionScan */ + econtext = node->ss.ps.ps_ExprContext; + ResetExprContext(econtext); + oldcontext = MemoryContextSwitchTo(econtext->ecxt_per_tuple_memory); + node->routine->Iterate(node->fstate, slot); + MemoryContextSwitchTo(oldcontext); + + /* Set tableoid if the tuple was valid. */ + if (!TupIsNull(slot)) + { + /* + * If the foreign-data wrapper returned a MinimalTuple, materialize the + * tuple to store system attributes. + */ + if (!TTS_HAS_PHYSICAL_TUPLE(slot)) + ExecMaterializeSlot(slot); + + /* overwrite only tableoid of the tuple */ + slot->tts_tuple->t_tableOid = + RelationGetRelid(node->ss.ss_currentRelation); + } + + return slot; +} + +/* + * ForeignRecheck -- access method routine to recheck a tuple in EvalPlanQual + */ +static bool +ForeignRecheck(ForeignScanState *node, TupleTableSlot *slot) +{ + /* ForeignScan never use keys in ForeignNext. */ + return true; +} + +/* ---------------------------------------------------------------- + * ExecForeignScan(node) + * + * Scans the relation sequentially and returns the next qualifying + * tuple. + * We call the ExecScan() routine and pass it the appropriate + * access method functions. + * ---------------------------------------------------------------- + */ +TupleTableSlot * +ExecForeignScan(ForeignScanState *node) +{ + return ExecScan((ScanState *) node, + (ExecScanAccessMtd) ForeignNext, + (ExecScanRecheckMtd) ForeignRecheck); +} + + +/* ---------------------------------------------------------------- + * ExecInitForeignScan + * ---------------------------------------------------------------- + */ +ForeignScanState * +ExecInitForeignScan(ForeignScan *node, EState *estate, int eflags) +{ + ForeignScanState *scanstate; + Relation currentRelation; + FdwRoutine *routine; + + /* + * foreign scan has no child node. + * but not any more. + */ + Assert(outerPlan(node) == NULL); + Assert(innerPlan(node) == NULL); + + /* + * create state structure + */ + scanstate = makeNode(ForeignScanState); + scanstate->ss.ps.plan = (Plan *) node; + scanstate->ss.ps.state = estate; + + /* + * Miscellaneous initialization + * + * create expression context for node + */ + ExecAssignExprContext(estate, &scanstate->ss.ps); + + /* + * initialize child expressions + */ + scanstate->ss.ps.targetlist = (List *) + ExecInitExpr((Expr *) node->scan.plan.targetlist, + (PlanState *) scanstate); + scanstate->ss.ps.qual = (List *) + ExecInitExpr((Expr *) node->scan.plan.qual, + (PlanState *) scanstate); + + /* + * tuple table initialization + */ + ExecInitResultTupleSlot(estate, &scanstate->ss.ps); + ExecInitScanTupleSlot(estate, &scanstate->ss); + + /* + * initialize scan relation. get the relation object id from the + * relid'th entry in the range table, open that relation and acquire + * appropriate lock on it. + */ + currentRelation = ExecOpenScanRelation(estate, node->scan.scanrelid); + scanstate->ss.ss_currentRelation = currentRelation; + ExecAssignScanType(&scanstate->ss, RelationGetDescr(currentRelation)); + scanstate->ss.ps.ps_TupFromTlist = false; + + /* + * Initialize result tuple type and projection info. + */ + ExecAssignResultTypeFromTL(&scanstate->ss.ps); + ExecAssignScanProjectionInfo(&scanstate->ss); + + /* cache the routine for the table in ForeignScanState */ + routine = GetFdwRoutineByRelId(RelationGetRelid(currentRelation)); + scanstate->routine = routine; + + /* + * If this execution was not for EXPLAIN w/o ANALYZE flag, initiate the + * foreign scan. + */ + if (!(eflags & EXEC_FLAG_EXPLAIN_ONLY)) + { + ForeignScan *scan = (ForeignScan *) scanstate->ss.ps.plan; + if (routine->BeginScan == NULL) + ereport(ERROR, + (errmsg("foreign-data wrapper must support BeginScan to scan foreign table"))); + scanstate->fstate = routine->BeginScan(scan->fplan, + estate->es_param_list_info); + } + + return scanstate; +} + +/* ---------------------------------------------------------------- + * ExecEndForeignScan + * + * frees any storage allocated through C routines. + * ---------------------------------------------------------------- + */ +void +ExecEndForeignScan(ForeignScanState *node) +{ + Relation relation; + + /* close the scan, if it has been opened */ + if (node->fstate != NULL) + { + if (node->routine->EndScan == NULL) + ereport(ERROR, + (errmsg("foreign-data wrapper must support BeginScan to scan foreign table"))); + node->routine->EndScan(node->fstate); + } + + /* get information from node */ + relation = node->ss.ss_currentRelation; + + /* Free the exprcontext */ + ExecFreeExprContext(&node->ss.ps); + + /* clean out the tuple table */ + ExecClearTuple(node->ss.ps.ps_ResultTupleSlot); + ExecClearTuple(node->ss.ss_ScanTupleSlot); + + /* close the relation. */ + ExecCloseScanRelation(relation); +} + +/* ---------------------------------------------------------------- + * Join Support + * ---------------------------------------------------------------- + */ + +/* ---------------------------------------------------------------- + * ExecForeignReScan + * + * Rescans the relation. + * ---------------------------------------------------------------- + */ +void +ExecForeignReScan(ForeignScanState *node) +{ + /* Use EndScan and BeginScan if the FDW doesn't supply ReScan */ + if (node->routine->ReScan == NULL) + { + ForeignScan *scan = (ForeignScan *) node->ss.ps.plan; + EState *estate = node->ss.ps.state; + + Assert(node->routine->BeginScan); + + node->routine->EndScan(node->fstate); + node->fstate = node->routine->BeginScan(scan->fplan, + estate->es_param_list_info); + } + else + node->routine->ReScan(node->fstate); + + ExecScanReScan((ScanState *) node); +} + +/* ---------------------------------------------------------------- + * ExecForeignMarkPos(node) + * + * Marks scan position. + * ---------------------------------------------------------------- + */ +void +ExecForeignMarkPos(ForeignScanState *node) +{ + elog(ERROR, "ForeignScan does not support mark/restore"); +} + +/* ---------------------------------------------------------------- + * ExecForeignRestrPos + * + * Restores scan position. + * ---------------------------------------------------------------- + */ +void +ExecForeignRestrPos(ForeignScanState *node) +{ + elog(ERROR, "ForeignScan does not support mark/restore"); +} diff --git a/src/backend/executor/nodeLockRows.c b/src/backend/executor/nodeLockRows.c index 2e08008807..0d39a4f5a8 100644 --- a/src/backend/executor/nodeLockRows.c +++ b/src/backend/executor/nodeLockRows.c @@ -79,6 +79,11 @@ ExecLockRows(LockRowsState *node) if (node->lr_epqstate.estate != NULL) EvalPlanQualSetTuple(&node->lr_epqstate, erm->rti, NULL); + /* if foreign table, the tuple can't be locked */ + if (erm->relation->rd_rel->relkind == RELKIND_FOREIGN_TABLE) + ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + (errmsg("SELECT FOR UPDATE/SHARE is not allowed with foreign tables")))); + /* if child rel, must check whether it produced this row */ if (erm->rti != erm->prti) { diff --git a/src/backend/foreign/foreign.c b/src/backend/foreign/foreign.c index 9a0f847f93..1fdd9254d5 100644 --- a/src/backend/foreign/foreign.c +++ b/src/backend/foreign/foreign.c @@ -16,6 +16,7 @@ #include "catalog/namespace.h" #include "catalog/pg_foreign_data_wrapper.h" #include "catalog/pg_foreign_server.h" +#include "catalog/pg_foreign_table.h" #include "catalog/pg_type.h" #include "catalog/pg_user_mapping.h" #include "foreign/foreign.h" @@ -59,6 +60,7 @@ GetForeignDataWrapper(Oid fdwid) fdw->owner = fdwform->fdwowner; fdw->fdwname = pstrdup(NameStr(fdwform->fdwname)); fdw->fdwvalidator = fdwform->fdwvalidator; + fdw->fdwhandler = fdwform->fdwhandler; /* Extract the options */ datum = SysCacheGetAttr(FOREIGNDATAWRAPPEROID, @@ -249,6 +251,140 @@ GetUserMapping(Oid userid, Oid serverid) } +/* + * GetForeignTable - look up the foreign table definition by relation oid. + */ +ForeignTable * +GetForeignTable(Oid relid) +{ + Form_pg_foreign_table tableform; + ForeignTable *ft; + HeapTuple tp; + Datum datum; + bool isnull; + + tp = SearchSysCache(FOREIGNTABLEREL, + ObjectIdGetDatum(relid), + 0, 0, 0); + + if (!HeapTupleIsValid(tp)) + elog(ERROR, "cache lookup failed for foreign table %u", relid); + + tableform = (Form_pg_foreign_table) GETSTRUCT(tp); + + ft = palloc(sizeof(ForeignTable)); + ft->relid = relid; + ft->serverid = tableform->ftserver; + + /* Extract the ftoptions */ + datum = SysCacheGetAttr(FOREIGNTABLEREL, + tp, + Anum_pg_foreign_table_ftoptions, + &isnull); + + /* untransformRelOptions does exactly what we want - avoid duplication */ + ft->options = untransformRelOptions(datum); + ReleaseSysCache(tp); + + return ft; +} + + +/* + * GetFdwRoutine - look up the handler of the foreign-data wrapper by OID and + * retrieve FdwRoutine. + */ +FdwRoutine * +GetFdwRoutine(Oid fdwhandler) +{ + FmgrInfo flinfo; + FunctionCallInfoData fcinfo; + Datum result; + FdwRoutine *routine; + + if (fdwhandler == InvalidOid) + elog(ERROR, "foreign-data wrapper has no handler"); + + fmgr_info(fdwhandler, &flinfo); + InitFunctionCallInfoData(fcinfo, &flinfo, 0, NULL, NULL); + result = FunctionCallInvoke(&fcinfo); + + if (fcinfo.isnull || + (routine = (FdwRoutine *) DatumGetPointer(result)) == NULL) + { + elog(ERROR, "function %u returned NULL", flinfo.fn_oid); + routine = NULL; /* keep compiler quiet */ + } + + return routine; +} + + +/* + * GetFdwRoutineByRelId - look up the handler of the foreign-data wrapper by + * OID of the foreign table and retrieve FdwRoutine. + */ +FdwRoutine * +GetFdwRoutineByRelId(Oid relid) +{ + HeapTuple tp; + Form_pg_foreign_data_wrapper fdwform; + Form_pg_foreign_server serverform; + Form_pg_foreign_table tableform; + Oid serverid; + Oid fdwid; + Oid fdwhandler; + + /* Get function OID for the foreign table. */ + tp = SearchSysCache1(FOREIGNTABLEREL, ObjectIdGetDatum(relid)); + if (!HeapTupleIsValid(tp)) + elog(ERROR, "cache lookup failed for foreign table %u", relid); + tableform = (Form_pg_foreign_table) GETSTRUCT(tp); + serverid = tableform->ftserver; + ReleaseSysCache(tp); + + tp = SearchSysCache1(FOREIGNSERVEROID, ObjectIdGetDatum(serverid)); + if (!HeapTupleIsValid(tp)) + elog(ERROR, "cache lookup failed for foreign server %u", serverid); + serverform = (Form_pg_foreign_server) GETSTRUCT(tp); + fdwid = serverform->srvfdw; + ReleaseSysCache(tp); + + tp = SearchSysCache1(FOREIGNDATAWRAPPEROID, ObjectIdGetDatum(fdwid)); + if (!HeapTupleIsValid(tp)) + elog(ERROR, "cache lookup failed for foreign-data wrapper %u", fdwid); + fdwform = (Form_pg_foreign_data_wrapper) GETSTRUCT(tp); + fdwhandler = fdwform->fdwhandler; + ReleaseSysCache(tp); + + return GetFdwRoutine(fdwhandler); +} + + +/* + * Determine the relation is a foreign table. + */ +bool +IsForeignTable(Oid relid) +{ + HeapTuple tuple; + Form_pg_class classForm; + char relkind; + + tuple = SearchSysCache1(RELOID, ObjectIdGetDatum(relid)); + if (!HeapTupleIsValid(tuple)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_TABLE), + errmsg("relation with OID %u does not exist", relid))); + classForm = (Form_pg_class) GETSTRUCT(tuple); + relkind = classForm->relkind; + ReleaseSysCache(tuple); + + return (relkind == RELKIND_FOREIGN_TABLE); +} + + + /* * deflist_to_tuplestore - Helper function to convert DefElem list to * tuplestore usable in SRF. diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index 662916d210..b94905f2f0 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -547,6 +547,42 @@ _copyWorkTableScan(WorkTableScan *from) return newnode; } +/* + * _copyForeignScan + */ +static ForeignScan * +_copyForeignScan(ForeignScan *from) +{ + ForeignScan *newnode = makeNode(ForeignScan); + + /* + * copy node superclass fields + */ + CopyScanFields((Scan *) from, (Scan *) newnode); + COPY_NODE_FIELD(fplan); + + return newnode; +} + +/* + * _copyFdwPlan + */ +static FdwPlan * +_copyFdwPlan(FdwPlan *from) +{ + FdwPlan *newnode = makeNode(FdwPlan); + + /* + * copy node superclass fields + */ + COPY_STRING_FIELD(explainInfo); + COPY_SCALAR_FIELD(startup_cost); + COPY_SCALAR_FIELD(total_cost); + COPY_NODE_FIELD(fdw_private); + + return newnode; +} + /* * CopyJoinFields * @@ -3202,7 +3238,7 @@ _copyCreateFdwStmt(CreateFdwStmt *from) CreateFdwStmt *newnode = makeNode(CreateFdwStmt); COPY_STRING_FIELD(fdwname); - COPY_NODE_FIELD(validator); + COPY_NODE_FIELD(func_options); COPY_NODE_FIELD(options); return newnode; @@ -3214,8 +3250,7 @@ _copyAlterFdwStmt(AlterFdwStmt *from) AlterFdwStmt *newnode = makeNode(AlterFdwStmt); COPY_STRING_FIELD(fdwname); - COPY_NODE_FIELD(validator); - COPY_SCALAR_FIELD(change_validator); + COPY_NODE_FIELD(func_options); COPY_NODE_FIELD(options); return newnode; @@ -3760,6 +3795,12 @@ copyObject(void *from) case T_WorkTableScan: retval = _copyWorkTableScan(from); break; + case T_ForeignScan: + retval = _copyForeignScan(from); + break; + case T_FdwPlan: + retval = _copyFdwPlan(from); + break; case T_Join: retval = _copyJoin(from); break; diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c index b7dc450447..26df9acd6b 100644 --- a/src/backend/nodes/equalfuncs.c +++ b/src/backend/nodes/equalfuncs.c @@ -1629,7 +1629,7 @@ static bool _equalCreateFdwStmt(CreateFdwStmt *a, CreateFdwStmt *b) { COMPARE_STRING_FIELD(fdwname); - COMPARE_NODE_FIELD(validator); + COMPARE_NODE_FIELD(func_options); COMPARE_NODE_FIELD(options); return true; @@ -1639,8 +1639,7 @@ static bool _equalAlterFdwStmt(AlterFdwStmt *a, AlterFdwStmt *b) { COMPARE_STRING_FIELD(fdwname); - COMPARE_NODE_FIELD(validator); - COMPARE_SCALAR_FIELD(change_validator); + COMPARE_NODE_FIELD(func_options); COMPARE_NODE_FIELD(options); return true; diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index c8eccce5a7..6fc6825af7 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -532,6 +532,25 @@ _outWorkTableScan(StringInfo str, WorkTableScan *node) WRITE_INT_FIELD(wtParam); } +static void +_outForeignScan(StringInfo str, ForeignScan *node) +{ + WRITE_NODE_TYPE("FOREIGNSCAN"); + + _outScanInfo(str, (Scan *) node); +} + +static void +_outFdwPlan(StringInfo str, FdwPlan *node) +{ + WRITE_NODE_TYPE("FDWPLAN"); + + WRITE_STRING_FIELD(explainInfo); + WRITE_FLOAT_FIELD(startup_cost, "%.2f"); + WRITE_FLOAT_FIELD(total_cost, "%.2f"); + WRITE_NODE_FIELD(fdw_private); +} + static void _outJoin(StringInfo str, Join *node) { @@ -1475,6 +1494,14 @@ _outTidPath(StringInfo str, TidPath *node) WRITE_NODE_FIELD(tidquals); } +static void +_outForeignPath(StringInfo str, ForeignPath *node) +{ + WRITE_NODE_TYPE("FOREIGNPATH"); + + _outPathInfo(str, (Path *) node); +} + static void _outAppendPath(StringInfo str, AppendPath *node) { @@ -2618,6 +2645,12 @@ _outNode(StringInfo str, void *obj) case T_WorkTableScan: _outWorkTableScan(str, obj); break; + case T_ForeignScan: + _outForeignScan(str, obj); + break; + case T_FdwPlan: + _outFdwPlan(str, obj); + break; case T_Join: _outJoin(str, obj); break; @@ -2820,6 +2853,9 @@ _outNode(StringInfo str, void *obj) case T_TidPath: _outTidPath(str, obj); break; + case T_ForeignPath: + _outForeignPath(str, obj); + break; case T_AppendPath: _outAppendPath(str, obj); break; diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c index 37d83323d6..ae2e13417d 100644 --- a/src/backend/optimizer/path/allpaths.c +++ b/src/backend/optimizer/path/allpaths.c @@ -17,6 +17,7 @@ #include +#include "foreign/foreign.h" #include "nodes/nodeFuncs.h" #ifdef OPTIMIZER_DEBUG #include "nodes/print.h" @@ -255,14 +256,22 @@ set_plain_rel_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte) * least one dimension of cost or sortedness. */ - /* Consider sequential scan */ - add_path(rel, create_seqscan_path(root, rel)); + if (IsForeignTable(rte->relid)) + { + /* only foreign scan path is applyable to foreign table */ + add_path(rel, create_foreignscan_path(root, rel)); + } + else + { + /* Consider sequential scan */ + add_path(rel, create_seqscan_path(root, rel)); - /* Consider index scans */ - create_index_paths(root, rel); + /* Consider index scans */ + create_index_paths(root, rel); - /* Consider TID scans */ - create_tidscan_paths(root, rel); + /* Consider TID scans */ + create_tidscan_paths(root, rel); + } /* Now find the cheapest of the paths for this rel */ set_cheapest(rel); @@ -1503,6 +1512,9 @@ print_path(PlannerInfo *root, Path *path, int indent) case T_TidPath: ptype = "TidScan"; break; + case T_ForeignPath: + ptype = "ForeignScan"; + break; case T_AppendPath: ptype = "Append"; break; diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c index c74125f1f7..938b8af847 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -20,6 +20,7 @@ #include #include "access/skey.h" +#include "catalog/pg_class.h" #include "miscadmin.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" @@ -71,6 +72,8 @@ static CteScan *create_ctescan_plan(PlannerInfo *root, Path *best_path, List *tlist, List *scan_clauses); static WorkTableScan *create_worktablescan_plan(PlannerInfo *root, Path *best_path, List *tlist, List *scan_clauses); +static ForeignScan *create_foreignscan_plan(PlannerInfo *root, Path *best_path, + List *tlist, List *scan_clauses); static NestLoop *create_nestloop_plan(PlannerInfo *root, NestPath *best_path, Plan *outer_plan, Plan *inner_plan); static MergeJoin *create_mergejoin_plan(PlannerInfo *root, MergePath *best_path, @@ -112,6 +115,8 @@ static CteScan *make_ctescan(List *qptlist, List *qpqual, Index scanrelid, int ctePlanId, int cteParam); static WorkTableScan *make_worktablescan(List *qptlist, List *qpqual, Index scanrelid, int wtParam); +static ForeignScan *make_foreignscan(List *qptlist, RangeTblEntry *rte, + List *qpqual, Index scanrelid, FdwPlan *fplan); static BitmapAnd *make_bitmap_and(List *bitmapplans); static BitmapOr *make_bitmap_or(List *bitmapplans); static NestLoop *make_nestloop(List *tlist, @@ -204,6 +209,7 @@ create_plan_recurse(PlannerInfo *root, Path *best_path) case T_ValuesScan: case T_CteScan: case T_WorkTableScan: + case T_ForeignScan: plan = create_scan_plan(root, best_path); break; case T_HashJoin: @@ -344,6 +350,13 @@ create_scan_plan(PlannerInfo *root, Path *best_path) scan_clauses); break; + case T_ForeignScan: + plan = (Plan *) create_foreignscan_plan(root, + best_path, + tlist, + scan_clauses); + break; + default: elog(ERROR, "unrecognized node type: %d", (int) best_path->pathtype); @@ -466,6 +479,7 @@ disuse_physical_tlist(Plan *plan, Path *path) case T_ValuesScan: case T_CteScan: case T_WorkTableScan: + case T_ForeignScan: plan->targetlist = build_relation_tlist(path->parent); break; default: @@ -1747,6 +1761,43 @@ create_worktablescan_plan(PlannerInfo *root, Path *best_path, return scan_plan; } +/* + * create_foreignscan_plan + * Returns a foreignscan plan for the base relation scanned by 'best_path' + * with restriction clauses 'scan_clauses' and targetlist 'tlist'. + */ +static ForeignScan * +create_foreignscan_plan(PlannerInfo *root, Path *best_path, + List *tlist, List *scan_clauses) +{ + ForeignPath *fpath = (ForeignPath *) best_path; + ForeignScan *scan_plan; + Index scan_relid = best_path->parent->relid; + RangeTblEntry *rte; + + /* it should be a base rel... */ + Assert(scan_relid > 0); + Assert(best_path->parent->rtekind == RTE_RELATION); + rte = planner_rt_fetch(scan_relid, root); + Assert(rte->rtekind == RTE_RELATION); + + /* Sort clauses into best execution order */ + scan_clauses = order_qual_clauses(root, scan_clauses); + + /* Reduce RestrictInfo list to bare expressions; ignore pseudoconstants */ + scan_clauses = extract_actual_clauses(scan_clauses, false); + + scan_plan = make_foreignscan(tlist, + rte, + scan_clauses, + scan_relid, + fpath->fplan); + + copy_path_costsize(&scan_plan->scan.plan, best_path); + + return scan_plan; +} + /***************************************************************************** * @@ -2962,6 +3013,27 @@ make_worktablescan(List *qptlist, return node; } +static ForeignScan * +make_foreignscan(List *qptlist, + RangeTblEntry *rte, + List *qpqual, + Index scanrelid, + FdwPlan *fplan) +{ + ForeignScan *node = makeNode(ForeignScan); + Plan *plan = &node->scan.plan; + + /* cost should be inserted by caller */ + plan->targetlist = qptlist; + plan->qual = qpqual; + plan->lefttree = NULL; + plan->righttree = NULL; + node->scan.scanrelid = scanrelid; + node->fplan = fplan; + + return node; +} + Append * make_append(List *appendplans, List *tlist) { diff --git a/src/backend/optimizer/plan/setrefs.c b/src/backend/optimizer/plan/setrefs.c index 02f5cabd25..ef2833aa2c 100644 --- a/src/backend/optimizer/plan/setrefs.c +++ b/src/backend/optimizer/plan/setrefs.c @@ -400,6 +400,17 @@ set_plan_refs(PlannerGlobal *glob, Plan *plan, int rtoffset) fix_scan_list(glob, splan->scan.plan.qual, rtoffset); } break; + case T_ForeignScan: + { + ForeignScan *splan = (ForeignScan *) plan; + + splan->scan.scanrelid += rtoffset; + splan->scan.plan.targetlist = + fix_scan_list(glob, splan->scan.plan.targetlist, rtoffset); + splan->scan.plan.qual = + fix_scan_list(glob, splan->scan.plan.qual, rtoffset); + } + break; case T_NestLoop: case T_MergeJoin: case T_HashJoin: diff --git a/src/backend/optimizer/plan/subselect.c b/src/backend/optimizer/plan/subselect.c index febec1e15f..54846daa1f 100644 --- a/src/backend/optimizer/plan/subselect.c +++ b/src/backend/optimizer/plan/subselect.c @@ -2044,6 +2044,10 @@ finalize_plan(PlannerInfo *root, Plan *plan, Bitmapset *valid_params, context.paramids = bms_add_members(context.paramids, scan_params); break; + case T_ForeignScan: + context.paramids = bms_add_members(context.paramids, scan_params); + break; + case T_ModifyTable: { ModifyTable *mtplan = (ModifyTable *) plan; diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c index d1010af662..134ec7ef20 100644 --- a/src/backend/optimizer/util/pathnode.c +++ b/src/backend/optimizer/util/pathnode.c @@ -17,6 +17,7 @@ #include #include "catalog/pg_operator.h" +#include "foreign/foreign.h" #include "miscadmin.h" #include "nodes/nodeFuncs.h" #include "optimizer/clauses.h" @@ -1419,6 +1420,36 @@ create_worktablescan_path(PlannerInfo *root, RelOptInfo *rel) return pathnode; } +/* + * create_foreignscan_path + * Creates a path corresponding to a scan of a foreign table, + * returning the pathnode. + */ +Path * +create_foreignscan_path(PlannerInfo *root, RelOptInfo *rel) +{ + RangeTblEntry *rte; + FdwRoutine *routine; + ForeignPath *pathnode = makeNode(ForeignPath); + + pathnode->path.pathtype = T_ForeignScan; + pathnode->path.parent = rel; + pathnode->path.pathkeys = NIL; /* result is always unordered */ + + rte = planner_rt_fetch(rel->relid, root); + routine = GetFdwRoutineByRelId(rte->relid); + if (routine->PlanRelScan == NULL) + ereport(ERROR, + (errmsg("foreign-data wrapper must support PlanRelScan to scan foreign table"))); + pathnode->fplan = routine->PlanRelScan(rte->relid, root, rel); + + /* use costs estimated by FDW */ + pathnode->path.startup_cost = pathnode->fplan->startup_cost; + pathnode->path.total_cost = pathnode->fplan->total_cost; + + return (Path *) pathnode; +} + /* * create_nestloop_path * Creates a pathnode corresponding to a nestloop join between two diff --git a/src/backend/optimizer/util/plancat.c b/src/backend/optimizer/util/plancat.c index adbe45caec..1d60af8b4f 100644 --- a/src/backend/optimizer/util/plancat.c +++ b/src/backend/optimizer/util/plancat.c @@ -90,12 +90,6 @@ get_relation_info(PlannerInfo *root, Oid relationObjectId, bool inhparent, */ relation = heap_open(relationObjectId, NoLock); - /* Foreign table scans are not implemented yet. */ - if (relation->rd_rel->relkind == RELKIND_FOREIGN_TABLE) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("foreign table scans are not yet supported"))); - rel->min_attr = FirstLowInvalidHeapAttributeNumber + 1; rel->max_attr = RelationGetNumberOfAttributes(relation); rel->reltablespace = RelationGetForm(relation)->reltablespace; @@ -460,6 +454,11 @@ estimate_rel_size(Relation rel, int32 *attr_widths, *pages = 1; *tuples = 1; break; + case RELKIND_FOREIGN_TABLE: + /* foreign tables has no storage, trust statistics */ + *pages = rel->rd_rel->relpages; + *tuples = rel->rd_rel->reltuples; + break; default: /* else it has no disk storage; probably shouldn't get here? */ *pages = 0; @@ -759,7 +758,8 @@ relation_excluded_by_constraints(PlannerInfo *root, * * We also support building a "physical" tlist for subqueries, functions, * values lists, and CTEs, since the same optimization can occur in - * SubqueryScan, FunctionScan, ValuesScan, CteScan, and WorkTableScan nodes. + * SubqueryScan, FunctionScan, ValuesScan, CteScan, WorkTableScan and + * ForeignScan nodes. */ List * build_physical_tlist(PlannerInfo *root, RelOptInfo *rel) diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 456db5c50e..52c1f74db1 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -306,6 +306,9 @@ static RangeVar *makeRangeVarFromAnyName(List *names, int position, core_yyscan_ create_generic_options alter_generic_options relation_expr_list dostmt_opt_list +%type opt_func_options func_options +%type fdw_option + %type OptTempTableName %type into_clause create_as_target @@ -3196,16 +3199,33 @@ DropTableSpaceStmt: DROP TABLESPACE name * *****************************************************************************/ -CreateFdwStmt: CREATE FOREIGN DATA_P WRAPPER name opt_validator create_generic_options +CreateFdwStmt: CREATE FOREIGN DATA_P WRAPPER name opt_func_options create_generic_options { CreateFdwStmt *n = makeNode(CreateFdwStmt); n->fdwname = $5; - n->validator = $6; + n->func_options = $6; n->options = $7; $$ = (Node *) n; } ; +fdw_option: + VALIDATOR handler_name { $$ = makeDefElem("validator", (Node *)$2); } + | NO VALIDATOR { $$ = makeDefElem("validator", NULL); } + | HANDLER handler_name { $$ = makeDefElem("handler", (Node *)$2); } + | NO HANDLER { $$ = makeDefElem("handler", NULL); } + ; + +func_options: + fdw_option { $$ = list_make1($1); } + | func_options fdw_option { $$ = lappend($1, $2); } + ; + +opt_func_options: + func_options { $$ = $1; } + | /*EMPTY*/ { $$ = NIL; } + ; + /***************************************************************************** * * QUERY : @@ -3238,28 +3258,20 @@ DropFdwStmt: DROP FOREIGN DATA_P WRAPPER name opt_drop_behavior * ****************************************************************************/ -AlterFdwStmt: ALTER FOREIGN DATA_P WRAPPER name validator_clause alter_generic_options +AlterFdwStmt: ALTER FOREIGN DATA_P WRAPPER name opt_func_options alter_generic_options { AlterFdwStmt *n = makeNode(AlterFdwStmt); n->fdwname = $5; - n->validator = $6; - n->change_validator = true; + n->func_options = $6; n->options = $7; $$ = (Node *) n; } - | ALTER FOREIGN DATA_P WRAPPER name validator_clause + | ALTER FOREIGN DATA_P WRAPPER name func_options { AlterFdwStmt *n = makeNode(AlterFdwStmt); n->fdwname = $5; - n->validator = $6; - n->change_validator = true; - $$ = (Node *) n; - } - | ALTER FOREIGN DATA_P WRAPPER name alter_generic_options - { - AlterFdwStmt *n = makeNode(AlterFdwStmt); - n->fdwname = $5; - n->options = $6; + n->func_options = $6; + n->options = NIL; $$ = (Node *) n; } ; diff --git a/src/backend/utils/adt/pseudotypes.c b/src/backend/utils/adt/pseudotypes.c index 2be0696f60..779844d161 100644 --- a/src/backend/utils/adt/pseudotypes.c +++ b/src/backend/utils/adt/pseudotypes.c @@ -453,3 +453,29 @@ pg_node_tree_send(PG_FUNCTION_ARGS) { return textsend(fcinfo); } + +/* + * fdw_handler_in - input routine for pseudo-type FDW_HANDLER. + */ +Datum +fdw_handler_in(PG_FUNCTION_ARGS) +{ + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot accept a value of type fdw_handler"))); + + PG_RETURN_VOID(); /* keep compiler quiet */ +} + +/* + * fdw_handler_out - output routine for pseudo-type FDW_HANDLER. + */ +Datum +fdw_handler_out(PG_FUNCTION_ARGS) +{ + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot display a value of type fdw_handler"))); + + PG_RETURN_VOID(); /* keep compiler quiet */ +} diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index e844b5b062..721f4df2ee 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -5972,6 +5972,7 @@ getForeignDataWrappers(int *numForeignDataWrappers) int i_fdwname; int i_rolname; int i_fdwvalidator; + int i_fdwhandler; int i_fdwacl; int i_fdwoptions; @@ -5985,13 +5986,27 @@ getForeignDataWrappers(int *numForeignDataWrappers) /* Make sure we are in proper schema */ selectSourceSchema("pg_catalog"); - appendPQExpBuffer(query, "SELECT tableoid, oid, fdwname, " - "(%s fdwowner) AS rolname, fdwvalidator::pg_catalog.regproc, fdwacl," - "array_to_string(ARRAY(" - " SELECT option_name || ' ' || quote_literal(option_value) " - " FROM pg_options_to_table(fdwoptions)), ', ') AS fdwoptions " - "FROM pg_foreign_data_wrapper", - username_subquery); + if (g_fout->remoteVersion >= 90100) + { + appendPQExpBuffer(query, "SELECT tableoid, oid, fdwname, " + "(%s fdwowner) AS rolname, fdwvalidator::pg_catalog.regproc, " + "fdwhandler::pg_catalog.regproc, fdwacl," + "array_to_string(ARRAY(" + " SELECT option_name || ' ' || quote_literal(option_value) " + " FROM pg_options_to_table(fdwoptions)), ', ') AS fdwoptions " + "FROM pg_foreign_data_wrapper", + username_subquery); + } + else + { + appendPQExpBuffer(query, "SELECT tableoid, oid, fdwname, " + "(%s fdwowner) AS rolname, fdwvalidator::pg_catalog.regproc, fdwacl," + "array_to_string(ARRAY(" + " SELECT option_name || ' ' || quote_literal(option_value) " + " FROM pg_options_to_table(fdwoptions)), ', ') AS fdwoptions " + "FROM pg_foreign_data_wrapper", + username_subquery); + } res = PQexec(g_conn, query->data); check_sql_result(res, g_conn, query->data, PGRES_TUPLES_OK); @@ -6006,6 +6021,8 @@ getForeignDataWrappers(int *numForeignDataWrappers) i_fdwname = PQfnumber(res, "fdwname"); i_rolname = PQfnumber(res, "rolname"); i_fdwvalidator = PQfnumber(res, "fdwvalidator"); + if (g_fout->remoteVersion >= 90100) + i_fdwhandler = PQfnumber(res, "fdwhandler"); i_fdwacl = PQfnumber(res, "fdwacl"); i_fdwoptions = PQfnumber(res, "fdwoptions"); @@ -6019,6 +6036,10 @@ getForeignDataWrappers(int *numForeignDataWrappers) fdwinfo[i].dobj.namespace = NULL; fdwinfo[i].rolname = strdup(PQgetvalue(res, i, i_rolname)); fdwinfo[i].fdwvalidator = strdup(PQgetvalue(res, i, i_fdwvalidator)); + if (g_fout->remoteVersion >= 90100) + fdwinfo[i].fdwhandler = strdup(PQgetvalue(res, i, i_fdwhandler)); + else + fdwinfo[i].fdwhandler = NULL; fdwinfo[i].fdwoptions = strdup(PQgetvalue(res, i, i_fdwoptions)); fdwinfo[i].fdwacl = strdup(PQgetvalue(res, i, i_fdwacl)); @@ -10324,6 +10345,11 @@ dumpForeignDataWrapper(Archive *fout, FdwInfo *fdwinfo) appendPQExpBuffer(q, " VALIDATOR %s", fdwinfo->fdwvalidator); + if (g_fout->remoteVersion >= 90100 && fdwinfo->fdwhandler && + strcmp(fdwinfo->fdwhandler, "-") != 0) + appendPQExpBuffer(q, " HANDLER %s", + fdwinfo->fdwhandler); + if (fdwinfo->fdwoptions && strlen(fdwinfo->fdwoptions) > 0) appendPQExpBuffer(q, " OPTIONS (%s)", fdwinfo->fdwoptions); diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h index 43fd1ade27..c630a1d06f 100644 --- a/src/bin/pg_dump/pg_dump.h +++ b/src/bin/pg_dump/pg_dump.h @@ -421,6 +421,7 @@ typedef struct _fdwInfo DumpableObject dobj; char *rolname; char *fdwvalidator; + char *fdwhandler; char *fdwoptions; char *fdwacl; } FdwInfo; diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index ada04de451..57718eb9b7 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -3480,9 +3480,11 @@ listForeignDataWrappers(const char *pattern, bool verbose) printfPQExpBuffer(&buf, "SELECT fdwname AS \"%s\",\n" " pg_catalog.pg_get_userbyid(fdwowner) AS \"%s\",\n" + " fdwhandler::pg_catalog.regproc AS \"%s\",\n" " fdwvalidator::pg_catalog.regproc AS \"%s\"", gettext_noop("Name"), gettext_noop("Owner"), + gettext_noop("Handler"), gettext_noop("Validator")); if (verbose) diff --git a/src/include/catalog/pg_foreign_data_wrapper.h b/src/include/catalog/pg_foreign_data_wrapper.h index a83556df15..971326cdae 100644 --- a/src/include/catalog/pg_foreign_data_wrapper.h +++ b/src/include/catalog/pg_foreign_data_wrapper.h @@ -33,6 +33,7 @@ CATALOG(pg_foreign_data_wrapper,2328) NameData fdwname; /* foreign-data wrapper name */ Oid fdwowner; /* FDW owner */ Oid fdwvalidator; /* optional validation function */ + Oid fdwhandler; /* foreign-data routines function */ /* VARIABLE LENGTH FIELDS start here. */ @@ -52,11 +53,12 @@ typedef FormData_pg_foreign_data_wrapper *Form_pg_foreign_data_wrapper; * ---------------- */ -#define Natts_pg_foreign_data_wrapper 5 +#define Natts_pg_foreign_data_wrapper 6 #define Anum_pg_foreign_data_wrapper_fdwname 1 #define Anum_pg_foreign_data_wrapper_fdwowner 2 #define Anum_pg_foreign_data_wrapper_fdwvalidator 3 -#define Anum_pg_foreign_data_wrapper_fdwacl 4 -#define Anum_pg_foreign_data_wrapper_fdwoptions 5 +#define Anum_pg_foreign_data_wrapper_fdwhandler 4 +#define Anum_pg_foreign_data_wrapper_fdwacl 5 +#define Anum_pg_foreign_data_wrapper_fdwoptions 6 #endif /* PG_FOREIGN_DATA_WRAPPER_H */ diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h index f8b5d4da3d..05fc57c633 100644 --- a/src/include/catalog/pg_proc.h +++ b/src/include/catalog/pg_proc.h @@ -3899,6 +3899,10 @@ DATA(insert OID = 2777 ( anynonarray_in PGNSP PGUID 12 1 0 0 f f f t f i 1 0 27 DESCR("I/O"); DATA(insert OID = 2778 ( anynonarray_out PGNSP PGUID 12 1 0 0 f f f t f i 1 0 2275 "2776" _null_ _null_ _null_ _null_ anynonarray_out _null_ _null_ _null_ )); DESCR("I/O"); +DATA(insert OID = 3116 ( fdw_handler_in PGNSP PGUID 12 1 0 0 f f f f f i 1 0 3115 "2275" _null_ _null_ _null_ _null_ fdw_handler_in _null_ _null_ _null_ )); +DESCR("I/O"); +DATA(insert OID = 3117 ( fdw_handler_out PGNSP PGUID 12 1 0 0 f f f t f i 1 0 2275 "3115" _null_ _null_ _null_ _null_ fdw_handler_out _null_ _null_ _null_ )); +DESCR("I/O"); /* cryptographic */ DATA(insert OID = 2311 ( md5 PGNSP PGUID 12 1 0 0 f f f t f i 1 0 25 "25" _null_ _null_ _null_ _null_ md5_text _null_ _null_ _null_ )); diff --git a/src/include/catalog/pg_type.h b/src/include/catalog/pg_type.h index 620f7b4612..ee40705160 100644 --- a/src/include/catalog/pg_type.h +++ b/src/include/catalog/pg_type.h @@ -623,6 +623,8 @@ DATA(insert OID = 2776 ( anynonarray PGNSP PGUID 4 t p P f t \054 0 0 0 anynona #define ANYNONARRAYOID 2776 DATA(insert OID = 3500 ( anyenum PGNSP PGUID 4 t p P f t \054 0 0 0 anyenum_in anyenum_out - - - - - i p f 0 -1 0 _null_ _null_ )); #define ANYENUMOID 3500 +DATA(insert OID = 3115 ( fdw_handler PGNSP PGUID 4 t p P f t \054 0 0 0 fdw_handler_in fdw_handler_out - - - - - i p f 0 -1 0 _null_ _null_ )); +#define FDW_HANDLEROID 3115 /* diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h index 9e2bbe8d8e..ba5f69b377 100644 --- a/src/include/commands/copy.h +++ b/src/include/commands/copy.h @@ -14,12 +14,24 @@ #ifndef COPY_H #define COPY_H +#include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "tcop/dest.h" +typedef struct CopyStateData *CopyState; + extern uint64 DoCopy(const CopyStmt *stmt, const char *queryString); +extern CopyState BeginCopyFrom(Relation rel, const char *filename, + List *attnamelist, List *options); +extern void EndCopyFrom(CopyState cstate); +extern bool NextCopyFrom(CopyState cstate, + Datum *values, bool *nulls, Oid *tupleOid); +extern bool NextLineCopyFrom(CopyState cstate, + char ***fields, int *nfields, Oid *tupleOid); +extern void CopyFromErrorCallback(void *arg); + extern DestReceiver *CreateCopyDestReceiver(void); #endif /* COPY_H */ diff --git a/src/include/executor/nodeForeignscan.h b/src/include/executor/nodeForeignscan.h new file mode 100644 index 0000000000..2aaaf2221e --- /dev/null +++ b/src/include/executor/nodeForeignscan.h @@ -0,0 +1,26 @@ +/*------------------------------------------------------------------------- + * + * nodeForeignscan.h + * + * + * + * Portions Copyright (c) 1996-2011, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/executor/nodeForeignscan.h + * + *------------------------------------------------------------------------- + */ +#ifndef NODEFOREIGNSCAN_H +#define NODEFOREIGNSCAN_H + +#include "nodes/execnodes.h" + +extern ForeignScanState *ExecInitForeignScan(ForeignScan *node, EState *estate, int eflags); +extern TupleTableSlot *ExecForeignScan(ForeignScanState *node); +extern void ExecEndForeignScan(ForeignScanState *node); +extern void ExecForeignMarkPos(ForeignScanState *node); +extern void ExecForeignRestrPos(ForeignScanState *node); +extern void ExecForeignReScan(ForeignScanState *node); + +#endif /* NODEFOREIGNSCAN_H */ diff --git a/src/include/foreign/fdwapi.h b/src/include/foreign/fdwapi.h new file mode 100644 index 0000000000..d6284bad56 --- /dev/null +++ b/src/include/foreign/fdwapi.h @@ -0,0 +1,151 @@ +/*------------------------------------------------------------------------- + * + * fdwapi.h + * API for foreign-data wrappers + * + * Portions Copyright (c) 1996-2011, PostgreSQL Global Development Group + * + * src/include/foreign/fdwapi.h + * + *------------------------------------------------------------------------- + */ +#ifndef FDWAPI_H +#define FDWAPI_H + +#include "executor/tuptable.h" +#include "nodes/pg_list.h" +#include "nodes/relation.h" + +/* + * When a plan is going to be cached, the plan node is copied into another + * context with copyObject. It means that FdwPlan, a part of ForeignScan plan + * node, and its contents must have copyObject support too. + */ +struct FdwPlan +{ + NodeTag type; + + /* + * Free-form text shown in EXPLAIN. The SQL to be sent to the remote + * server is typically shown here. + */ + char *explainInfo; + + /* + * Cost estimation info. The startup_cost should include the cost of + * connecting to the remote host and sending over the query, as well as + * the cost of starting up the query so that it returns the first result + * row. + */ + double startup_cost; + double total_cost; +#ifdef NOT_USED + /* XXX: Should FDWs estimate rows and width? */ + double rows; + int width; +#endif + + /* + * FDW-private data. FDW must guarantee that every elements in this list + * have copyObject support. If FDW needs to store arbitrary data such as + * non-Node structure, Const of bytea would be able to use as a container. + */ + List *fdw_private; +}; +typedef struct FdwPlan FdwPlan; + +struct FdwExecutionState +{ + /* FDW-private data */ + void *fdw_private; +}; +typedef struct FdwExecutionState FdwExecutionState; + +/* + * Common interface routines of FDW, inspired by the FDW API in the SQL/MED + * standard, but adapted to the PostgreSQL world. + * + * A foreign-data wrapper implements these routines. At a minimum, it must + * implement PlanRelScan, BeginScan, Iterate, ReScan and EndScan. + * + * The PlanXXX functions return an FdwPlan struct that can later be executed + * with BeginScan. The implementation should fill in the cost estimates in + * FdwPlan, and may store private information. + */ +struct FdwRoutine +{ + /* + * Plan a scan on a foreign table. 'foreigntableid' identifies the foreign + * table. + * + * 'root' and 'baserel' contain context information that the + * implementation can use to restrict the rows that are fetched. + * baserel->baserestrictinfo is particularly interseting, as it contains + * quals (WHERE clauses) that can be used to filter the rows in the remote + * server. 'root' and 'baserel' can be safely ignored, the planner will + * re-check the quals on every fetched row anyway. + */ + FdwPlan *(*PlanRelScan)(Oid foreigntableid, PlannerInfo *root, + RelOptInfo *baserel); + + /* + * Begin execution of a foreign scan. This function is called when an + * actual scan is needed, so EXPLAIN without ANALYZE option doesn't call + * BeginScan(). + */ + FdwExecutionState *(*BeginScan)(FdwPlan *plan, ParamListInfo params); + + /* + * Fetch the next record and store it into slot. + * FDW can return result as either a physical tuple or a virtual tuple. + * If FDW returns virtual tuple, executor will materialize the virtual + * tuple and store tableoid in it. + * + * When the end of the external data is found, FDW should clear the slot + * with ExecClearTuple(slot) to tell executor to finish the scan. + * + * Note that Iterate is called in per-tuple memory context which has been + * reset before each call. + */ + void (*Iterate)(FdwExecutionState *state, TupleTableSlot *slot); + + /* + * Reset the read pointer to the head of the scan. + * This function will be called when the new outer tuple was acquired in a + * nested loop. + */ + void (*ReScan)(FdwExecutionState *state); + + /* + * End the foreign scan and do clean up. + */ + void (*EndScan)(FdwExecutionState *state); + +#ifdef NOT_USED + /* + * Plan a query of arbitrary native SQL (or other query language supported + * by the foreign server). This is used for SQL/MED passthrough mode, or + * e.g contrib/dblink. + */ + FdwPlan *(*PlanNative)(Oid serverid, char *query); + + /* + * Plan a whole subquery. This is used for example to execute an aggregate + * query remotely without pulling all the rows to the local server. + * + * The implementation can return NULL if it cannot satisfy the whole + * subquery, in which case the planner will break down the query into + * smaller parts and call PlanRelScan for the foreign tables involved. + * + * The implementation must be careful to only accept queries it fully + * understands! For example, if it ignores windowClauses, and returns + * a non-NULL results for a query that contains one, the windowClause + * would be lost and the query would return incorrect results. + */ + FdwPlan *(*PlanQuery)(PlannerInfo *root, Query query); +#endif +}; +typedef struct FdwRoutine FdwRoutine; + +#endif /* FDWAPI_H */ + diff --git a/src/include/foreign/foreign.h b/src/include/foreign/foreign.h index 2d1495cfe1..fe299b38c4 100644 --- a/src/include/foreign/foreign.h +++ b/src/include/foreign/foreign.h @@ -13,6 +13,7 @@ #ifndef FOREIGN_H #define FOREIGN_H +#include "foreign/fdwapi.h" #include "nodes/parsenodes.h" @@ -38,6 +39,7 @@ typedef struct ForeignDataWrapper Oid owner; /* FDW owner user Oid */ char *fdwname; /* Name of the FDW */ Oid fdwvalidator; + Oid fdwhandler; List *options; /* fdwoptions as DefElem list */ } ForeignDataWrapper; @@ -59,6 +61,12 @@ typedef struct UserMapping List *options; /* useoptions as DefElem list */ } UserMapping; +typedef struct ForeignTable +{ + Oid relid; /* relation Oid */ + Oid serverid; /* server Oid */ + List *options; /* ftoptions as DefElem list */ +} ForeignTable; extern ForeignServer *GetForeignServer(Oid serverid); extern ForeignServer *GetForeignServerByName(const char *name, bool missing_ok); @@ -68,5 +76,9 @@ extern ForeignDataWrapper *GetForeignDataWrapper(Oid fdwid); extern ForeignDataWrapper *GetForeignDataWrapperByName(const char *name, bool missing_ok); extern Oid GetForeignDataWrapperOidByName(const char *name, bool missing_ok); +extern ForeignTable *GetForeignTable(Oid relid); +extern FdwRoutine *GetFdwRoutineByRelId(Oid relid); +extern FdwRoutine *GetFdwRoutine(Oid fdwhandler); +extern bool IsForeignTable(Oid relid); #endif /* FOREIGN_H */ diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 546b581b6d..7d1c681e06 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -17,8 +17,10 @@ #include "access/genam.h" #include "access/heapam.h" #include "access/skey.h" +#include "foreign/fdwapi.h" #include "nodes/params.h" #include "nodes/plannodes.h" +#include "nodes/relation.h" #include "nodes/tidbitmap.h" #include "utils/hsearch.h" #include "utils/rel.h" @@ -1402,6 +1404,20 @@ typedef struct WorkTableScanState RecursiveUnionState *rustate; } WorkTableScanState; +/* ---------------- + * ForeignScanState information + * + * ForeignScan nodes are used to scan the foreign table managed by + * a foreign server. + * ---------------- + */ +typedef struct ForeignScanState +{ + ScanState ss; /* its first field is NodeTag */ + FdwRoutine *routine; + FdwExecutionState *fstate; /* private data for each data wrapper */ +} ForeignScanState; + /* ---------------------------------------------------------------- * Join State Information * ---------------------------------------------------------------- diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index 456e8e216a..5b065b517c 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -60,6 +60,8 @@ typedef enum NodeTag T_ValuesScan, T_CteScan, T_WorkTableScan, + T_ForeignScan, + T_FdwPlan, T_Join, T_NestLoop, T_MergeJoin, @@ -103,6 +105,7 @@ typedef enum NodeTag T_ValuesScanState, T_CteScanState, T_WorkTableScanState, + T_ForeignScanState, T_JoinState, T_NestLoopState, T_MergeJoinState, @@ -216,6 +219,7 @@ typedef enum NodeTag T_MergePath, T_HashPath, T_TidPath, + T_ForeignPath, T_AppendPath, T_MergeAppendPath, T_ResultPath, diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 483f22591e..2db3c48a72 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -1531,7 +1531,7 @@ typedef struct CreateFdwStmt { NodeTag type; char *fdwname; /* foreign-data wrapper name */ - List *validator; /* optional validator function (qual. name) */ + List *func_options; /* VALIDATOR/HANDLER conbination */ List *options; /* generic options to FDW */ } CreateFdwStmt; @@ -1539,8 +1539,7 @@ typedef struct AlterFdwStmt { NodeTag type; char *fdwname; /* foreign-data wrapper name */ - List *validator; /* optional validator function (qual. name) */ - bool change_validator; + List *func_options; /* VALIDATOR/HANDLER conbination */ List *options; /* generic options to FDW */ } AlterFdwStmt; diff --git a/src/include/nodes/plannodes.h b/src/include/nodes/plannodes.h index 011c6869f2..fcfd20320b 100644 --- a/src/include/nodes/plannodes.h +++ b/src/include/nodes/plannodes.h @@ -15,6 +15,7 @@ #define PLANNODES_H #include "access/sdir.h" +#include "foreign/fdwapi.h" #include "nodes/bitmapset.h" #include "nodes/primnodes.h" #include "storage/itemptr.h" @@ -434,6 +435,16 @@ typedef struct WorkTableScan int wtParam; /* ID of Param representing work table */ } WorkTableScan; +/* ---------------- + * ForeignScan node + * ---------------- + */ +typedef struct ForeignScan +{ + Scan scan; + FdwPlan *fplan; +} ForeignScan; + /* * ========== diff --git a/src/include/nodes/relation.h b/src/include/nodes/relation.h index 8b14838625..133a2bd10e 100644 --- a/src/include/nodes/relation.h +++ b/src/include/nodes/relation.h @@ -744,6 +744,15 @@ typedef struct TidPath List *tidquals; /* qual(s) involving CTID = something */ } TidPath; +/* + * ForeignPath represents a scan on a foreign table + */ +typedef struct ForeignPath +{ + Path path; + struct FdwPlan *fplan; +} ForeignPath; + /* * AppendPath represents an Append plan, ie, successive execution of * several member plans. diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathnode.h index ff220e32a7..fe746006b2 100644 --- a/src/include/optimizer/pathnode.h +++ b/src/include/optimizer/pathnode.h @@ -61,6 +61,7 @@ extern Path *create_functionscan_path(PlannerInfo *root, RelOptInfo *rel); extern Path *create_valuesscan_path(PlannerInfo *root, RelOptInfo *rel); extern Path *create_ctescan_path(PlannerInfo *root, RelOptInfo *rel); extern Path *create_worktablescan_path(PlannerInfo *root, RelOptInfo *rel); +extern Path *create_foreignscan_path(PlannerInfo *root, RelOptInfo *rel); extern NestPath *create_nestloop_path(PlannerInfo *root, RelOptInfo *joinrel, diff --git a/src/include/utils/builtins.h b/src/include/utils/builtins.h index 4cb6c5c17a..eaaf7d0651 100644 --- a/src/include/utils/builtins.h +++ b/src/include/utils/builtins.h @@ -518,6 +518,8 @@ extern Datum pg_node_tree_in(PG_FUNCTION_ARGS); extern Datum pg_node_tree_out(PG_FUNCTION_ARGS); extern Datum pg_node_tree_recv(PG_FUNCTION_ARGS); extern Datum pg_node_tree_send(PG_FUNCTION_ARGS); +extern Datum fdw_handler_in(PG_FUNCTION_ARGS); +extern Datum fdw_handler_out(PG_FUNCTION_ARGS); /* regexp.c */ extern Datum nameregexeq(PG_FUNCTION_ARGS); diff --git a/src/test/regress/expected/alter_table.out b/src/test/regress/expected/alter_table.out index 3280065f75..eea0b0953a 100644 --- a/src/test/regress/expected/alter_table.out +++ b/src/test/regress/expected/alter_table.out @@ -1002,7 +1002,7 @@ copy test("........pg.dropped.1........") to stdout; ERROR: column "........pg.dropped.1........" of relation "test" does not exist copy test from stdin; ERROR: extra data after last expected column -CONTEXT: COPY test, line 1: "10 11 12" +CONTEXT: relation test, line 1: "10 11 12" select * from test; b | c ---+--- diff --git a/src/test/regress/expected/copy2.out b/src/test/regress/expected/copy2.out index 15cbe02977..d9630f845d 100644 --- a/src/test/regress/expected/copy2.out +++ b/src/test/regress/expected/copy2.out @@ -35,17 +35,17 @@ ERROR: column "d" specified more than once -- missing data: should fail COPY x from stdin; ERROR: invalid input syntax for integer: "" -CONTEXT: COPY x, line 1, column a: "" +CONTEXT: relation x, line 1, column a: "" COPY x from stdin; ERROR: missing data for column "e" -CONTEXT: COPY x, line 1: "2000 230 23 23" +CONTEXT: relation x, line 1: "2000 230 23 23" COPY x from stdin; ERROR: missing data for column "e" -CONTEXT: COPY x, line 1: "2001 231 \N \N" +CONTEXT: relation x, line 1: "2001 231 \N \N" -- extra data: should fail COPY x from stdin; ERROR: extra data after last expected column -CONTEXT: COPY x, line 1: "2002 232 40 50 60 70 80" +CONTEXT: relation x, line 1: "2002 232 40 50 60 70 80" -- various COPY options: delimiters, oids, NULL string COPY x (b, c, d, e) from stdin with oids delimiter ',' null 'x'; COPY x from stdin WITH DELIMITER AS ';' NULL AS ''; diff --git a/src/test/regress/expected/domain.out b/src/test/regress/expected/domain.out index 7d72791e5e..daadcef616 100644 --- a/src/test/regress/expected/domain.out +++ b/src/test/regress/expected/domain.out @@ -49,7 +49,7 @@ INSERT INTO basictest values ('88', 'haha', 'short', '123.1212'); -- Truncate -- Test copy COPY basictest (testvarchar) FROM stdin; -- fail ERROR: value too long for type character varying(5) -CONTEXT: COPY basictest, line 1, column testvarchar: "notsoshorttext" +CONTEXT: relation basictest, line 1, column testvarchar: "notsoshorttext" COPY basictest (testvarchar) FROM stdin; select * from basictest; testint4 | testtext | testvarchar | testnumeric @@ -130,7 +130,7 @@ select testint4arr[1], testchar4arr[2:2] from domarrtest; COPY domarrtest FROM stdin; COPY domarrtest FROM stdin; -- fail ERROR: value too long for type character varying(4) -CONTEXT: COPY domarrtest, line 1, column testchar4arr: "{qwerty,w,e}" +CONTEXT: relation domarrtest, line 1, column testchar4arr: "{qwerty,w,e}" select * from domarrtest; testint4arr | testchar4arr ---------------+--------------------- @@ -173,14 +173,14 @@ INSERT INTO nulltest values ('a', 'b', 'c', NULL, 'd'); -- Good -- Test copy COPY nulltest FROM stdin; --fail ERROR: null value in column "col3" violates not-null constraint -CONTEXT: COPY nulltest, line 1: "a b \N d d" +CONTEXT: relation nulltest, line 1: "a b \N d d" COPY nulltest FROM stdin; --fail ERROR: domain dcheck does not allow null values -CONTEXT: COPY nulltest, line 1, column col5: null input +CONTEXT: relation nulltest, line 1, column col5: null input -- Last row is bad COPY nulltest FROM stdin; ERROR: new row for relation "nulltest" violates check constraint "nulltest_col5_check" -CONTEXT: COPY nulltest, line 3: "a b c \N a" +CONTEXT: relation nulltest, line 3: "a b c \N a" select * from nulltest; col1 | col2 | col3 | col4 | col5 ------+------+------+------+------ diff --git a/src/test/regress/expected/foreign_data.out b/src/test/regress/expected/foreign_data.out index d6c650be14..8f7eac5333 100644 --- a/src/test/regress/expected/foreign_data.out +++ b/src/test/regress/expected/foreign_data.out @@ -16,11 +16,11 @@ CREATE ROLE unprivileged_role; CREATE FOREIGN DATA WRAPPER dummy; CREATE FOREIGN DATA WRAPPER postgresql VALIDATOR postgresql_fdw_validator; -- At this point we should have 2 built-in wrappers and no servers. -SELECT fdwname, fdwvalidator::regproc, fdwoptions FROM pg_foreign_data_wrapper ORDER BY 1, 2, 3; - fdwname | fdwvalidator | fdwoptions -------------+--------------------------+------------ - dummy | - | - postgresql | postgresql_fdw_validator | +SELECT fdwname, fdwvalidator::regproc, fdwhandler::regproc, fdwoptions FROM pg_foreign_data_wrapper ORDER BY 1, 2, 3; + fdwname | fdwvalidator | fdwhandler | fdwoptions +------------+--------------------------+------------+------------ + dummy | - | - | + postgresql | postgresql_fdw_validator | - | (2 rows) SELECT srvname, srvoptions FROM pg_foreign_server; @@ -38,12 +38,12 @@ CREATE FOREIGN DATA WRAPPER foo VALIDATOR bar; -- ERROR ERROR: function bar(text[], oid) does not exist CREATE FOREIGN DATA WRAPPER foo; \dew - List of foreign-data wrappers - Name | Owner | Validator -------------+-------------------+-------------------------- - dummy | foreign_data_user | - - foo | foreign_data_user | - - postgresql | foreign_data_user | postgresql_fdw_validator + List of foreign-data wrappers + Name | Owner | Handler | Validator +------------+-------------------+---------+-------------------------- + dummy | foreign_data_user | - | - + foo | foreign_data_user | - | - + postgresql | foreign_data_user | - | postgresql_fdw_validator (3 rows) CREATE FOREIGN DATA WRAPPER foo; -- duplicate @@ -51,12 +51,12 @@ ERROR: foreign-data wrapper "foo" already exists DROP FOREIGN DATA WRAPPER foo; CREATE FOREIGN DATA WRAPPER foo OPTIONS (testing '1'); \dew+ - List of foreign-data wrappers - Name | Owner | Validator | Access privileges | Options -------------+-------------------+--------------------------+-------------------+------------- - dummy | foreign_data_user | - | | - foo | foreign_data_user | - | | {testing=1} - postgresql | foreign_data_user | postgresql_fdw_validator | | + List of foreign-data wrappers + Name | Owner | Handler | Validator | Access privileges | Options +------------+-------------------+---------+--------------------------+-------------------+------------- + dummy | foreign_data_user | - | - | | + foo | foreign_data_user | - | - | | {testing=1} + postgresql | foreign_data_user | - | postgresql_fdw_validator | | (3 rows) DROP FOREIGN DATA WRAPPER foo; @@ -64,12 +64,12 @@ CREATE FOREIGN DATA WRAPPER foo OPTIONS (testing '1', testing '2'); -- ERROR ERROR: option "testing" provided more than once CREATE FOREIGN DATA WRAPPER foo OPTIONS (testing '1', another '2'); \dew+ - List of foreign-data wrappers - Name | Owner | Validator | Access privileges | Options -------------+-------------------+--------------------------+-------------------+----------------------- - dummy | foreign_data_user | - | | - foo | foreign_data_user | - | | {testing=1,another=2} - postgresql | foreign_data_user | postgresql_fdw_validator | | + List of foreign-data wrappers + Name | Owner | Handler | Validator | Access privileges | Options +------------+-------------------+---------+--------------------------+-------------------+----------------------- + dummy | foreign_data_user | - | - | | + foo | foreign_data_user | - | - | | {testing=1,another=2} + postgresql | foreign_data_user | - | postgresql_fdw_validator | | (3 rows) DROP FOREIGN DATA WRAPPER foo; @@ -80,12 +80,12 @@ HINT: Must be superuser to create a foreign-data wrapper. RESET ROLE; CREATE FOREIGN DATA WRAPPER foo VALIDATOR postgresql_fdw_validator; \dew+ - List of foreign-data wrappers - Name | Owner | Validator | Access privileges | Options -------------+-------------------+--------------------------+-------------------+--------- - dummy | foreign_data_user | - | | - foo | foreign_data_user | postgresql_fdw_validator | | - postgresql | foreign_data_user | postgresql_fdw_validator | | + List of foreign-data wrappers + Name | Owner | Handler | Validator | Access privileges | Options +------------+-------------------+---------+--------------------------+-------------------+--------- + dummy | foreign_data_user | - | - | | + foo | foreign_data_user | - | postgresql_fdw_validator | | + postgresql | foreign_data_user | - | postgresql_fdw_validator | | (3 rows) -- ALTER FOREIGN DATA WRAPPER @@ -97,12 +97,12 @@ ALTER FOREIGN DATA WRAPPER foo VALIDATOR bar; -- ERROR ERROR: function bar(text[], oid) does not exist ALTER FOREIGN DATA WRAPPER foo NO VALIDATOR; \dew+ - List of foreign-data wrappers - Name | Owner | Validator | Access privileges | Options -------------+-------------------+--------------------------+-------------------+--------- - dummy | foreign_data_user | - | | - foo | foreign_data_user | - | | - postgresql | foreign_data_user | postgresql_fdw_validator | | + List of foreign-data wrappers + Name | Owner | Handler | Validator | Access privileges | Options +------------+-------------------+---------+--------------------------+-------------------+--------- + dummy | foreign_data_user | - | - | | + foo | foreign_data_user | - | - | | + postgresql | foreign_data_user | - | postgresql_fdw_validator | | (3 rows) ALTER FOREIGN DATA WRAPPER foo OPTIONS (a '1', b '2'); @@ -112,34 +112,34 @@ ALTER FOREIGN DATA WRAPPER foo OPTIONS (DROP c); -- ERROR ERROR: option "c" not found ALTER FOREIGN DATA WRAPPER foo OPTIONS (ADD x '1', DROP x); \dew+ - List of foreign-data wrappers - Name | Owner | Validator | Access privileges | Options -------------+-------------------+--------------------------+-------------------+----------- - dummy | foreign_data_user | - | | - foo | foreign_data_user | - | | {a=1,b=2} - postgresql | foreign_data_user | postgresql_fdw_validator | | + List of foreign-data wrappers + Name | Owner | Handler | Validator | Access privileges | Options +------------+-------------------+---------+--------------------------+-------------------+----------- + dummy | foreign_data_user | - | - | | + foo | foreign_data_user | - | - | | {a=1,b=2} + postgresql | foreign_data_user | - | postgresql_fdw_validator | | (3 rows) ALTER FOREIGN DATA WRAPPER foo OPTIONS (DROP a, SET b '3', ADD c '4'); \dew+ - List of foreign-data wrappers - Name | Owner | Validator | Access privileges | Options -------------+-------------------+--------------------------+-------------------+----------- - dummy | foreign_data_user | - | | - foo | foreign_data_user | - | | {b=3,c=4} - postgresql | foreign_data_user | postgresql_fdw_validator | | + List of foreign-data wrappers + Name | Owner | Handler | Validator | Access privileges | Options +------------+-------------------+---------+--------------------------+-------------------+----------- + dummy | foreign_data_user | - | - | | + foo | foreign_data_user | - | - | | {b=3,c=4} + postgresql | foreign_data_user | - | postgresql_fdw_validator | | (3 rows) ALTER FOREIGN DATA WRAPPER foo OPTIONS (a '2'); ALTER FOREIGN DATA WRAPPER foo OPTIONS (b '4'); -- ERROR ERROR: option "b" provided more than once \dew+ - List of foreign-data wrappers - Name | Owner | Validator | Access privileges | Options -------------+-------------------+--------------------------+-------------------+--------------- - dummy | foreign_data_user | - | | - foo | foreign_data_user | - | | {b=3,c=4,a=2} - postgresql | foreign_data_user | postgresql_fdw_validator | | + List of foreign-data wrappers + Name | Owner | Handler | Validator | Access privileges | Options +------------+-------------------+---------+--------------------------+-------------------+--------------- + dummy | foreign_data_user | - | - | | + foo | foreign_data_user | - | - | | {b=3,c=4,a=2} + postgresql | foreign_data_user | - | postgresql_fdw_validator | | (3 rows) SET ROLE regress_test_role; @@ -149,12 +149,12 @@ HINT: Must be superuser to alter a foreign-data wrapper. SET ROLE regress_test_role_super; ALTER FOREIGN DATA WRAPPER foo OPTIONS (ADD d '5'); \dew+ - List of foreign-data wrappers - Name | Owner | Validator | Access privileges | Options -------------+-------------------+--------------------------+-------------------+------------------- - dummy | foreign_data_user | - | | - foo | foreign_data_user | - | | {b=3,c=4,a=2,d=5} - postgresql | foreign_data_user | postgresql_fdw_validator | | + List of foreign-data wrappers + Name | Owner | Handler | Validator | Access privileges | Options +------------+-------------------+---------+--------------------------+-------------------+------------------- + dummy | foreign_data_user | - | - | | + foo | foreign_data_user | - | - | | {b=3,c=4,a=2,d=5} + postgresql | foreign_data_user | - | postgresql_fdw_validator | | (3 rows) ALTER FOREIGN DATA WRAPPER foo OWNER TO regress_test_role; -- ERROR @@ -168,12 +168,12 @@ ERROR: permission denied to alter foreign-data wrapper "foo" HINT: Must be superuser to alter a foreign-data wrapper. RESET ROLE; \dew+ - List of foreign-data wrappers - Name | Owner | Validator | Access privileges | Options -------------+-------------------------+--------------------------+-------------------+------------------- - dummy | foreign_data_user | - | | - foo | regress_test_role_super | - | | {b=3,c=4,a=2,d=5} - postgresql | foreign_data_user | postgresql_fdw_validator | | + List of foreign-data wrappers + Name | Owner | Handler | Validator | Access privileges | Options +------------+-------------------------+---------+--------------------------+-------------------+------------------- + dummy | foreign_data_user | - | - | | + foo | regress_test_role_super | - | - | | {b=3,c=4,a=2,d=5} + postgresql | foreign_data_user | - | postgresql_fdw_validator | | (3 rows) -- DROP FOREIGN DATA WRAPPER @@ -182,12 +182,12 @@ ERROR: foreign-data wrapper "nonexistent" does not exist DROP FOREIGN DATA WRAPPER IF EXISTS nonexistent; NOTICE: foreign-data wrapper "nonexistent" does not exist, skipping \dew+ - List of foreign-data wrappers - Name | Owner | Validator | Access privileges | Options -------------+-------------------------+--------------------------+-------------------+------------------- - dummy | foreign_data_user | - | | - foo | regress_test_role_super | - | | {b=3,c=4,a=2,d=5} - postgresql | foreign_data_user | postgresql_fdw_validator | | + List of foreign-data wrappers + Name | Owner | Handler | Validator | Access privileges | Options +------------+-------------------------+---------+--------------------------+-------------------+------------------- + dummy | foreign_data_user | - | - | | + foo | regress_test_role_super | - | - | | {b=3,c=4,a=2,d=5} + postgresql | foreign_data_user | - | postgresql_fdw_validator | | (3 rows) DROP ROLE regress_test_role_super; -- ERROR @@ -202,23 +202,23 @@ ALTER ROLE regress_test_role_super SUPERUSER; DROP FOREIGN DATA WRAPPER foo; DROP ROLE regress_test_role_super; \dew+ - List of foreign-data wrappers - Name | Owner | Validator | Access privileges | Options -------------+-------------------+--------------------------+-------------------+--------- - dummy | foreign_data_user | - | | - postgresql | foreign_data_user | postgresql_fdw_validator | | + List of foreign-data wrappers + Name | Owner | Handler | Validator | Access privileges | Options +------------+-------------------+---------+--------------------------+-------------------+--------- + dummy | foreign_data_user | - | - | | + postgresql | foreign_data_user | - | postgresql_fdw_validator | | (2 rows) CREATE FOREIGN DATA WRAPPER foo; CREATE SERVER s1 FOREIGN DATA WRAPPER foo; CREATE USER MAPPING FOR current_user SERVER s1; \dew+ - List of foreign-data wrappers - Name | Owner | Validator | Access privileges | Options -------------+-------------------+--------------------------+-------------------+--------- - dummy | foreign_data_user | - | | - foo | foreign_data_user | - | | - postgresql | foreign_data_user | postgresql_fdw_validator | | + List of foreign-data wrappers + Name | Owner | Handler | Validator | Access privileges | Options +------------+-------------------+---------+--------------------------+-------------------+--------- + dummy | foreign_data_user | - | - | | + foo | foreign_data_user | - | - | | + postgresql | foreign_data_user | - | postgresql_fdw_validator | | (3 rows) \des+ @@ -250,11 +250,11 @@ NOTICE: drop cascades to 2 other objects DETAIL: drop cascades to server s1 drop cascades to user mapping for foreign_data_user \dew+ - List of foreign-data wrappers - Name | Owner | Validator | Access privileges | Options -------------+-------------------+--------------------------+-------------------+--------- - dummy | foreign_data_user | - | | - postgresql | foreign_data_user | postgresql_fdw_validator | | + List of foreign-data wrappers + Name | Owner | Handler | Validator | Access privileges | Options +------------+-------------------+---------+--------------------------+-------------------+--------- + dummy | foreign_data_user | - | - | | + postgresql | foreign_data_user | - | postgresql_fdw_validator | | (2 rows) \des+ @@ -669,6 +669,10 @@ Has OIDs: no CREATE INDEX id_ft1_c2 ON ft1 (c2); -- ERROR ERROR: "ft1" is not a table +SELECT * FROM ft1; -- ERROR +ERROR: foreign-data wrapper has no handler +EXPLAIN SELECT * FROM ft1; -- ERROR +ERROR: foreign-data wrapper has no handler -- ALTER FOREIGN TABLE COMMENT ON FOREIGN TABLE ft1 IS 'foreign table'; COMMENT ON FOREIGN TABLE ft1 IS NULL; @@ -1105,9 +1109,9 @@ NOTICE: drop cascades to server sc \c DROP ROLE foreign_data_user; -- At this point we should have no wrappers, no servers, and no mappings. -SELECT fdwname, fdwvalidator, fdwoptions FROM pg_foreign_data_wrapper; - fdwname | fdwvalidator | fdwoptions ----------+--------------+------------ +SELECT fdwname, fdwvalidator, fdwhandler, fdwoptions FROM pg_foreign_data_wrapper; + fdwname | fdwvalidator | fdwhandler | fdwoptions +---------+--------------+------------+------------ (0 rows) SELECT srvname, srvoptions FROM pg_foreign_server; diff --git a/src/test/regress/output/constraints.source b/src/test/regress/output/constraints.source index d164b90af7..64735d87ad 100644 --- a/src/test/regress/output/constraints.source +++ b/src/test/regress/output/constraints.source @@ -278,7 +278,7 @@ SELECT '' AS two, * FROM COPY_TBL; COPY COPY_TBL FROM '@abs_srcdir@/data/constrf.data'; ERROR: new row for relation "copy_tbl" violates check constraint "copy_con" -CONTEXT: COPY copy_tbl, line 2: "7 check failed 6" +CONTEXT: relation copy_tbl, line 2: "7 check failed 6" SELECT * FROM COPY_TBL; x | y | z ---+---------------+--- diff --git a/src/test/regress/sql/foreign_data.sql b/src/test/regress/sql/foreign_data.sql index 86b698a1b6..2e0fa9a234 100644 --- a/src/test/regress/sql/foreign_data.sql +++ b/src/test/regress/sql/foreign_data.sql @@ -24,7 +24,7 @@ CREATE FOREIGN DATA WRAPPER dummy; CREATE FOREIGN DATA WRAPPER postgresql VALIDATOR postgresql_fdw_validator; -- At this point we should have 2 built-in wrappers and no servers. -SELECT fdwname, fdwvalidator::regproc, fdwoptions FROM pg_foreign_data_wrapper ORDER BY 1, 2, 3; +SELECT fdwname, fdwvalidator::regproc, fdwhandler::regproc, fdwoptions FROM pg_foreign_data_wrapper ORDER BY 1, 2, 3; SELECT srvname, srvoptions FROM pg_foreign_server; SELECT * FROM pg_user_mapping; @@ -271,6 +271,8 @@ COMMENT ON COLUMN ft1.c1 IS 'ft1.c1'; \d+ ft1 \det+ CREATE INDEX id_ft1_c2 ON ft1 (c2); -- ERROR +SELECT * FROM ft1; -- ERROR +EXPLAIN SELECT * FROM ft1; -- ERROR -- ALTER FOREIGN TABLE COMMENT ON FOREIGN TABLE ft1 IS 'foreign table'; @@ -453,6 +455,6 @@ DROP FOREIGN DATA WRAPPER dummy CASCADE; DROP ROLE foreign_data_user; -- At this point we should have no wrappers, no servers, and no mappings. -SELECT fdwname, fdwvalidator, fdwoptions FROM pg_foreign_data_wrapper; +SELECT fdwname, fdwvalidator, fdwhandler, fdwoptions FROM pg_foreign_data_wrapper; SELECT srvname, srvoptions FROM pg_foreign_server; SELECT * FROM pg_user_mapping;