Merge pull request #3 from tinybirdco/support-extract

support extract

.github/workflows/ci.yml

@@ -11,7 +11,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        pg: [14, 13, 12, 11]
+        pg: [14]
         check_code: ["false", "clang"]
 
     name: Test clickhouse_fdw

DEVELOPMENT.md

@@ -0,0 +1,23 @@
+Use this command to run tests locally:
+
+```sh
+docker-compose -f docker-compose.yml up -d && docker-compose -f docker-compose-extra.yml  build --build-arg CHECK_CODE=false --build-arg PG_VERSION=14 tests && docker-compose -f docker-compose-extra.yml run tests
+```
+
+To debug changes in the code use `elog` in the code, they'll be printed in the output of the tests.
+
+To get the results of the tests locally change `docker-compose-extra.yml` to connect a volume
+
+```
+volumes:
+    - /tmp/pg:/tmp
+```
+
+Use `/tmp` as the outputdir when running the tests by changing this line in `CMakeLists.txt`:
+
+```sh
+add_custom_target(installcheck
+	COMMAND ${PG_REGRESS} --inputdir=${PROJECT_SOURCE_DIR}/tests --outputdir=/tmp/tests \${EXTRA_REGRESS_OPTS} ${REGRESS_TESTS})
+```
+
+Now you can get the results file from `/tmp/pg/tests/results` and override the `.out` files with the actual output expected.

docker-compose.yml

@@ -1,10 +1,11 @@
 version: '2'
 services:
   clickhouse:
-    image: yandex/clickhouse-server
+    image: clickhouse/clickhouse-server:latest
+    network_mode: "host"
     ports:
-      - "8123:8123"
-      - "9000:9000"
+      - 8123:8123
+      - 9000:9000
     volumes:
       - ./docker/clickhouse-config.xml:/etc/clickhouse-server/config.xml
-      - ./docker/clickhouse-users.xml:/etc/clickhouse-server/users.xml
+      - ./docker/clickhouse-users.xml:/etc/clickhouse-server/users.xml

src/custom_types.c

@@ -27,6 +27,8 @@
 #define F_TIMESTAMP_ZONE 2069
 #define F_TIMESTAMPTZ_ZONE 1159
 #define F_TIMESTAMPTZ_PART 1171
+#define F_TIMESTAMP_EXTRACT 6202
+#define F_TIMESTAMPTZ_EXTRACT 6203
 #define F_TO_TIMESTAMP 1778
 #define F_ARRAY_POSITION 3277
 #define F_STRPOS 868
@@ -99,6 +101,8 @@ CustomObjectDef *chfdw_check_for_custom_function(Oid funcid)
 		{
 			case F_TIMESTAMP_TRUNC:
 			case F_TIMESTAMPTZ_TRUNC:
+			case F_TIMESTAMP_EXTRACT:
+			case F_TIMESTAMPTZ_EXTRACT:
 			case F_TIMESTAMP_ZONE:
 			case F_TIMESTAMPTZ_ZONE:
 			case F_TIMESTAMP_PART:
@@ -145,6 +149,13 @@ CustomObjectDef *chfdw_check_for_custom_function(Oid funcid)
 				entry->custom_name[0] = '\1';
 				break;
 			}
+			case F_TIMESTAMPTZ_EXTRACT:
+			case F_TIMESTAMP_EXTRACT:
+			{
+				entry->cf_type = CF_EXTRACT;
+				entry->custom_name[0] = '\1';
+				break;
+			}
 			case F_TIMESTAMP_ZONE:
 			case F_TIMESTAMPTZ_ZONE:
 			{

src/deparse.c

@@ -2468,6 +2468,50 @@ deparseFuncExpr(FuncExpr *node, deparse_expr_cxt *context)
 		appendStringInfoChar(buf, ')');
 		return;
 	}
+	else if (cdef && cdef->cf_type == CF_EXTRACT)
+	{
+		if (list_length(node->args) != 2)
+        	elog(ERROR, "extract requires exactly two arguments");
+
+		Const *arg = (Const *) linitial(node->args);
+		char *parttype = TextDatumGetCString(arg->constvalue);
+		CSTRING_TOLOWER(parttype);
+
+		Expr *source_expr = (Expr *) lsecond(node->args);
+
+		if (strcmp(parttype, "day") == 0)
+			appendStringInfoString(buf, "toDayOfMonth");
+		else if (strcmp(parttype, "doy") == 0)
+			appendStringInfoString(buf, "toDayOfYear");
+		else if (strcmp(parttype, "dow") == 0)
+			appendStringInfoString(buf, "toDayOfWeek");
+		else if (strcmp(parttype, "year") == 0)
+			appendStringInfoString(buf, "toYear");
+		else if (strcmp(parttype, "month") == 0)
+			appendStringInfoString(buf, "toMonth");
+		else if (strcmp(parttype, "hour") == 0)
+			appendStringInfoString(buf, "toHour");
+		else if (strcmp(parttype, "minute") == 0)
+			appendStringInfoString(buf, "toMinute");
+		else if (strcmp(parttype, "second") == 0)
+			appendStringInfoString(buf, "toSecond");
+		else if (strcmp(parttype, "quarter") == 0)
+			appendStringInfoString(buf, "toQuarter");
+		else if (strcmp(parttype, "isoyear") == 0)
+			appendStringInfoString(buf, "toISOYear");
+		else if (strcmp(parttype, "week") == 0)
+			appendStringInfoString(buf, "toISOWeek");
+		else if (strcmp(parttype, "epoch") == 0)
+			appendStringInfoString(buf, "toUnixTimestamp");
+		else
+			elog(ERROR, "extract cannot be exported for: %s", parttype);
+
+		pfree(parttype);
+		appendStringInfoChar(buf, '(');
+		deparseExpr(source_expr, context);
+		appendStringInfoChar(buf, ')');
+		return;
+	}
 	else if (cdef && cdef->cf_type == CF_ISTORE_SEED)
 	{
 		if (!context->func)

src/http.c

@@ -100,6 +100,9 @@ ch_http_connection_t *ch_http_connection(char *host, int port, char *username, c
 
 	conn->base_url = connstring;
 	conn->base_url_len = strlen(conn->base_url);
+
+	if (database == NULL)
+		database = "default";  // Set default value
 	conn->database = malloc(strlen(database));
 	strcpy(conn->database, database);
 

src/include/clickhousedb_fdw.h

@@ -270,6 +270,7 @@ typedef enum {
 	CF_AJTIME_TO_TIMESTAMP,	/* ajtime to timestamp */
 	CF_DATE_TRUNC,		/* date_trunc function */
 	CF_DATE_PART,		/* date_part function */
+	CF_EXTRACT,		/* extract function */
 	CF_TIMESTAMPTZ_PL_INTERVAL,	/* timestamptz + interval */
 	CF_TIMEZONE,		/* timezone */
 	CF_TO_TIMESTAMP,

tests/expected/binary_queries.out

@@ -143,10 +143,10 @@ SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1;  -- should work
 
 ALTER SERVER loopback OPTIONS (SET dbname 'no such database');
 SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1;  -- should fail
-ERROR:  clickhouse_fdw: DB::Exception: Database `no such database` doesn't exist
+ERROR:  clickhouse_fdw: DB::Exception: Database `no such database` does not exist
 ALTER USER MAPPING FOR CURRENT_USER SERVER loopback OPTIONS (ADD user 'no such user');
 SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1;  -- should fail
-ERROR:  clickhouse_fdw: connection error: DB::Exception: no such user: Authentication failed: password is incorrect or there is no user with such name
+ERROR:  clickhouse_fdw: connection error: DB::Exception: no such user: Authentication failed: password is incorrect, or there is no user with such name.
 ALTER SERVER loopback OPTIONS (SET dbname 'regression');
 ALTER USER MAPPING FOR CURRENT_USER SERVER loopback OPTIONS (DROP user);
 SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1;  -- should work again

tests/expected/binary_queries_1.out

@@ -143,10 +143,10 @@ SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1;  -- should work
 
 ALTER SERVER loopback OPTIONS (SET dbname 'no such database');
 SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1;  -- should fail
-ERROR:  clickhouse_fdw: DB::Exception: Database `no such database` doesn't exist
+ERROR:  clickhouse_fdw: DB::Exception: Database `no such database` does not exist
 ALTER USER MAPPING FOR CURRENT_USER SERVER loopback OPTIONS (ADD user 'no such user');
 SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1;  -- should fail
-ERROR:  clickhouse_fdw: connection error: DB::Exception: no such user: Authentication failed: password is incorrect or there is no user with such name
+ERROR:  clickhouse_fdw: connection error: DB::Exception: no such user: Authentication failed: password is incorrect, or there is no user with such name.
 ALTER SERVER loopback OPTIONS (SET dbname 'regression');
 ALTER USER MAPPING FOR CURRENT_USER SERVER loopback OPTIONS (DROP user);
 SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1;  -- should work again

tests/expected/functions.out

@@ -234,17 +234,17 @@ SELECT uniq_exact(a) FILTER(WHERE b>1) FROM t1;
 (1 row)
 
 EXPLAIN (VERBOSE, COSTS OFF) SELECT c as d1 FROM t1 WHERE c >= to_timestamp('2019-01-01 00:00:00.000000', 'YYYY-MM-DD HH24:MI:SS.US') GROUP BY d1 ORDER BY d1;
-                                                                                QUERY PLAN                                                                                
---------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+                                                                   QUERY PLAN                                                                   
+------------------------------------------------------------------------------------------------------------------------------------------------
  Foreign Scan
    Output: c
    Relations: Aggregate on (t1)
    Remote SQL: SELECT c FROM regression.t1 WHERE ((c >= parseDateTimeBestEffortOrNull('2019-01-01 00:00:00.000000'))) GROUP BY c ORDER BY c ASC
 (4 rows)
 
 SELECT c as d1 FROM t1 WHERE c >= to_timestamp('2019-01-01 00:00:00.000000', 'YYYY-MM-DD HH24:MI:SS.US') GROUP BY d1 ORDER BY d1;
-           d1           
-------------------------
+         d1          
+---------------------
  2019-01-01 10:00:00
  2019-01-02 10:00:00
 (2 rows)
@@ -253,7 +253,7 @@ EXPLAIN (VERBOSE, COSTS OFF) SELECT date_trunc('dAy', c at time zone 'UTC') as d
                                                                                 QUERY PLAN                                                                                
 --------------------------------------------------------------------------------------------------------------------------------------------------------------------------
  Foreign Scan
-   Output: (date_trunc('dAy'::text, timezone('UTC'::text, c)))
+   Output: (date_trunc('dAy'::text, (c AT TIME ZONE 'UTC'::text)))
    Relations: Aggregate on (t1)
    Remote SQL: SELECT toStartOfDay(toTimeZone(c, 'UTC')) FROM regression.t1 GROUP BY (toStartOfDay(toTimeZone(c, 'UTC'))) ORDER BY toStartOfDay(toTimeZone(c, 'UTC')) ASC
 (4 rows)
@@ -269,7 +269,7 @@ EXPLAIN (VERBOSE, COSTS OFF) SELECT date_trunc('day', c at time zone 'UTC') as d
                                                                                 QUERY PLAN                                                                                
 --------------------------------------------------------------------------------------------------------------------------------------------------------------------------
  Foreign Scan
-   Output: (date_trunc('day'::text, timezone('UTC'::text, c)))
+   Output: (date_trunc('day'::text, (c AT TIME ZONE 'UTC'::text)))
    Relations: Aggregate on (t2)
    Remote SQL: SELECT toStartOfDay(toTimeZone(c, 'UTC')) FROM regression.t1 GROUP BY (toStartOfDay(toTimeZone(c, 'UTC'))) ORDER BY toStartOfDay(toTimeZone(c, 'UTC')) ASC
 (4 rows)
@@ -361,12 +361,12 @@ SELECT date_part('minuTe'::text, timezone('UTC'::text, c)) as d1 FROM t2 GROUP B
 (1 row)
 
 EXPLAIN (VERBOSE, COSTS OFF) SELECT date_trunc('SeCond', c at time zone 'UTC') as d1 FROM t1 GROUP BY d1 ORDER BY d1;
-                                                                                                              QUERY PLAN                                                                                                              
---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+                                                                                                                        QUERY PLAN                                                                                                                         
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
  Foreign Scan
-   Output: (date_trunc('SeCond'::text, timezone('UTC'::text, c)))
+   Output: (date_trunc('SeCond'::text, (c AT TIME ZONE 'UTC'::text)))
    Relations: Aggregate on (t1)
-   Remote SQL: SELECT toStartOfSecond(toDateTime64(toTimeZone(c, 'UTC'), 1)) FROM regression.t1 GROUP BY (toStartOfSecond(toDateTime64(toTimeZone(c, 'UTC'), 1))) ORDER BY toStartOfSecond(toDateTime64(toTimeZone(c, 'UTC'), 1)) ASC
+   Remote SQL: SELECT toStartOfSecond(toDateTime64(toTimeZone(c, 'UTC'), 1, 'UTC')) FROM regression.t1 GROUP BY (toStartOfSecond(toDateTime64(toTimeZone(c, 'UTC'), 1, 'UTC'))) ORDER BY toStartOfSecond(toDateTime64(toTimeZone(c, 'UTC'), 1, 'UTC')) ASC
 (4 rows)
 
 SELECT date_trunc('SeCond', c at time zone 'UTC') as d1 FROM t1 GROUP BY d1 ORDER BY d1;

tests/expected/functions_1.out

@@ -233,11 +233,27 @@ SELECT uniq_exact(a) FILTER(WHERE b>1) FROM t1;
           1
 (1 row)
 
+EXPLAIN (VERBOSE, COSTS OFF) SELECT c as d1 FROM t1 WHERE c >= to_timestamp('2019-01-01 00:00:00.000000', 'YYYY-MM-DD HH24:MI:SS.US') GROUP BY d1 ORDER BY d1;
+                                                                   QUERY PLAN
+------------------------------------------------------------------------------------------------------------------------------------------------
+ Foreign Scan
+   Output: c
+   Relations: Aggregate on (t1)
+   Remote SQL: SELECT c FROM regression.t1 WHERE ((c >= parseDateTimeBestEffortOrNull('2019-01-01 00:00:00.000000'))) GROUP BY c ORDER BY c ASC
+(4 rows)
+
+SELECT c as d1 FROM t1 WHERE c >= to_timestamp('2019-01-01 00:00:00.000000', 'YYYY-MM-DD HH24:MI:SS.US') GROUP BY d1 ORDER BY d1;
+         d1
+---------------------
+ 2019-01-01 10:00:00
+ 2019-01-02 10:00:00
+(2 rows)
+
 EXPLAIN (VERBOSE, COSTS OFF) SELECT date_trunc('dAy', c at time zone 'UTC') as d1 FROM t1 GROUP BY d1 ORDER BY d1;
                                                                                 QUERY PLAN                                                                                
 --------------------------------------------------------------------------------------------------------------------------------------------------------------------------
  Foreign Scan
-   Output: (date_trunc('dAy'::text, (c AT TIME ZONE 'UTC'::text)))
+   Output: (date_trunc('dAy'::text, timezone('UTC'::text, c)))
    Relations: Aggregate on (t1)
    Remote SQL: SELECT toStartOfDay(toTimeZone(c, 'UTC')) FROM regression.t1 GROUP BY (toStartOfDay(toTimeZone(c, 'UTC'))) ORDER BY toStartOfDay(toTimeZone(c, 'UTC')) ASC
 (4 rows)
@@ -253,7 +269,7 @@ EXPLAIN (VERBOSE, COSTS OFF) SELECT date_trunc('day', c at time zone 'UTC') as d
                                                                                 QUERY PLAN                                                                                
 --------------------------------------------------------------------------------------------------------------------------------------------------------------------------
  Foreign Scan
-   Output: (date_trunc('day'::text, (c AT TIME ZONE 'UTC'::text)))
+   Output: (date_trunc('day'::text, timezone('UTC'::text, c)))
    Relations: Aggregate on (t2)
    Remote SQL: SELECT toStartOfDay(toTimeZone(c, 'UTC')) FROM regression.t1 GROUP BY (toStartOfDay(toTimeZone(c, 'UTC'))) ORDER BY toStartOfDay(toTimeZone(c, 'UTC')) ASC
 (4 rows)
@@ -345,12 +361,12 @@ SELECT date_part('minuTe'::text, timezone('UTC'::text, c)) as d1 FROM t2 GROUP B
 (1 row)
 
 EXPLAIN (VERBOSE, COSTS OFF) SELECT date_trunc('SeCond', c at time zone 'UTC') as d1 FROM t1 GROUP BY d1 ORDER BY d1;
-                                                                                                              QUERY PLAN                                                                                                              
---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+                                                                                                                        QUERY PLAN
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
  Foreign Scan
-   Output: (date_trunc('SeCond'::text, (c AT TIME ZONE 'UTC'::text)))
+   Output: (date_trunc('SeCond'::text, timezone('UTC'::text, c)))
    Relations: Aggregate on (t1)
-   Remote SQL: SELECT toStartOfSecond(toDateTime64(toTimeZone(c, 'UTC'), 1)) FROM regression.t1 GROUP BY (toStartOfSecond(toDateTime64(toTimeZone(c, 'UTC'), 1))) ORDER BY toStartOfSecond(toDateTime64(toTimeZone(c, 'UTC'), 1)) ASC
+   Remote SQL: SELECT toStartOfSecond(toDateTime64(toTimeZone(c, 'UTC'), 1, 'UTC')) FROM regression.t1 GROUP BY (toStartOfSecond(toDateTime64(toTimeZone(c, 'UTC'), 1, 'UTC'))) ORDER BY toStartOfSecond(toDateTime64(toTimeZone(c, 'UTC'), 1, 'UTC')) ASC
 (4 rows)
 
 SELECT date_trunc('SeCond', c at time zone 'UTC') as d1 FROM t1 GROUP BY d1 ORDER BY d1;

tests/expected/http_1.out

@@ -166,12 +166,10 @@ SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1;  -- should work
 
 ALTER SERVER loopback OPTIONS (SET dbname 'no such database');
 SELECT c3, c4 FROM ft1 ORDER BY c3, c1;  -- should fail
-ERROR:  clickhouse_fdw:Code: 81. DB::Exception: Database `no such database` doesn't exist. (UNKNOWN_DATABASE)
-QUERY:SELECT c1, c3, c4 FROM "no such database".t1 ORDER BY c3 ASC, c1 ASC
+ERROR:  clickhouse_fdw: communication error: URL rejected: Malformed input to a URL function
 ALTER USER MAPPING FOR CURRENT_USER SERVER loopback OPTIONS (ADD user 'no such user');
 SELECT c3, c4 FROM ft1 ORDER BY c3, c1;  -- should fail
-ERROR:  clickhouse_fdw:Code: 516. DB::Exception: no such user: Authentication failed: password is incorrect or there is no user with such name. (AUTHENTICATION_FAILED)
-QUERY:SELECT c1, c3, c4 FROM "no such database".t1 ORDER BY c3 ASC, c1 ASC
+ERROR:  clickhouse_fdw: communication error: URL rejected: Malformed input to a URL function
 ALTER SERVER loopback OPTIONS (SET dbname 'regression');
 ALTER USER MAPPING FOR CURRENT_USER SERVER loopback OPTIONS (DROP user);
 SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1;  -- should work again