overview

basic full stack example of CDC with debezium, tested working with hypertables

debugging docker compose

create a connector for the db, table

curl -X POST -H "Content-Type: application/json" http://localhost:8083/connectors --data '{
  "name": "inventory-connector",
  "config": {
    "connector.class": "io.debezium.connector.postgresql.PostgresConnector",
    "topic.prefix": "dbserver",
    "database.hostname": "postgres",
    "database.port": "5432",
    "database.user": "debezium",
    "database.password": "dbzpass",
    "database.dbname": "inventory",
    "table.include.list": "public.customers",
    "plugin.name": "pgoutput"
  }
}'

enter psql container and create a table, add a row

docker exec -it <psql-container-name> psql -U debezium inventory
> CREATE TABLE customers(id SERIAL PRIMARY KEY, name VARCHAR(50));
>INSERT INTO customers(name) VALUES ('test-user');

check the topic for cdc updates

docker exec <redpanda-container-name> rpk topic consume dbserver.public.customers

timeseries testing

testing hypertable

-- Make sure extension on
CREATE EXTENSION IF NOT EXISTS timescaledb;"

-- Create normal table
CREATE TABLE sensor_data (
    time TIMESTAMPTZ NOT NULL,
    sensor_id INTEGER,
    temperature DOUBLE PRECISION,
    humidity DOUBLE PRECISION
);

-- Convert to hypertable
SELECT create_hypertable('sensor_data', 'time');

-- Create publication
CREATE PUBLICATION dbz_publication FOR ALL TABLES;

create the connector

curl -X POST -H "Content-Type: application/json" http://localhost:8083/connectors --data '{
  "name": "timescaledb-connector",
  "config": {
    "connector.class": "io.debezium.connector.postgresql.PostgresConnector",
    "topic.prefix": "tsdb-server",
    "database.hostname": "postgres",
    "database.port": "5432",
    "database.user": "debezium",
    "database.password": "dbzpass",
    "database.dbname": "inventory",
    "schema.include.list": "_timescaledb_internal",
    "publication.name": "dbz_publication",
    "plugin.name": "pgoutput",
    "transforms": "timescaledb",
    "transforms.timescaledb.type": "io.debezium.connector.postgresql.transforms.timescaledb.TimescaleDb",
    "transforms.timescaledb.database.hostname": "postgres",
    "transforms.timescaledb.database.port": "5432",
    "transforms.timescaledb.database.user": "debezium",
    "transforms.timescaledb.database.password": "dbzpass",
    "transforms.timescaledb.database.dbname": "inventory"
  }
}'