最近搞了一个项目,需求是将客户导入的csv文件里的数据存入数据库,并且根据id和packed_on去重。
表结构:

1 按月分表
– 给分表加上id packed_on唯一性约束
drop trigger if exists tri_ins_tb_package on tb_package;
drop function if exists func_tri_tb_package();
create or replace function func_tri_tb_package() returns trigger as
$$
declare my_tbname varchar(64);
declare my_start_time varchar(64);
declare my_end_time varchar(64);
declare sql_str text;
begin
my_tbname = TG_TABLE_NAME || ‘_’ || to_char(NEW.packed_on,‘YYYYMM’);
sql_str = 'INSERT INTO ‘||my_tbname ||’ SELECT $1.* ’ || ‘on CONFLICT(id,packed_on) do nothing’;
EXECUTE sql_str USING NEW;
return null;
exception when undefined_table then
begin
my_start_time=to_char(NEW.packed_on,'YYYY-MM')||'-01 00:00:00';
my_end_time=(my_start_time::timestamp(0) + interval '1 month')::text;
execute 'create table ' || my_tbname || '(check (packed_on >= ' || '''' || my_start_time || '''' || ' and packed_on < ' || '''' || my_end_time || '''' || ')) INHERITS (' || TG_TABLE_NAME || ')';
execute 'alter table ' || my_tbname || ' add constraint pk_' || my_tbname || ' primary key (package_id)';
execute 'alter table ' || my_tbname || ' add constraint unique_' || my_tbname || ' unique (id,packed_on)';
execute 'create index idx_' || my_tbname ||'_sttm_elid on ' || my_tbname || ' (id,packed_on)';
EXECUTE sql_str USING NEW;
return null;
exception when others then
execute sql_str using NEW;
return null;
end;
end;
$$ language plpgsql;
create trigger tri_ins_tb_package BEFORE insert on tb_package for each row EXECUTE PROCEDURE func_tri_tb_package();
2 删除70天以前的记录,防止总数据量太大。使用语句级别触发器,在导入数据的时候执行一次
CREATE OR REPLACE FUNCTION auto_del_package ()
RETURNS TRIGGER AS resresres
BEGIN
delete from tb_package where packed_on < (now() - interval ‘70 day’);
return old;
END;
resresres LANGUAGE plpgsql;
CREATE TRIGGER auto_del_package_trigger AFTER INSERT ON tb_package
FOR statement EXECUTE PROCEDURE auto_del_package();
本文介绍了在pgsql中如何实现分表并结合唯一性约束进行数据去重。通过按月创建分表,并利用触发器在插入时动态插入到对应分表,并在冲突时采取DO NOTHING策略。此外,还设置了语句级触发器,当导入数据后自动删除70天前的记录,以保持数据量的合理范围。
1338

被折叠的 条评论
为什么被折叠?



