Skip to content

Commit

Permalink
Merge pull request #155 from haubourg/master
Browse files Browse the repository at this point in the history 
Add History audit new generation for views and fixes bug in conformity checker with mutti columns Prmiary Keys
  • Loading branch information
Régis Haubourg authored Mar 21, 2017
2 parents ba680c0 + 48fd6a1 commit e910256
Show file tree
Hide file tree
Showing 9 changed files with 581 additions and 97 deletions.
1 change: 1 addition & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ This repository contains the definition of the data model used by [QWAT](https:/


# Model changelog
- v1.2.2 : Integrates the new audit history system. Fixes an issue with multiple primary keys in conformity check procedure
- v1.2.1 : Allow installation type change (done in trigger function generated by submodule meta-project generator).
- v1.2.0 : Simplification of the trigger on views, ie there are no more triggers in cascade generated by the inheritance model. That modification does not affect the data-model code, but the change deserves a change in QWAT version number.
- v1.1.1 : Adds the ability to use post delta files to check auto generated triggers in model
Expand Down
5 changes: 3 additions & 2 deletions init_qwat.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -245,8 +245,9 @@ SRID=$SRID ${DIR}/ordinary_data/views/insert_views.sh


# Finalize System
psql -v ON_ERROR_STOP=1 -f ${DIR}/system/audit_tables.sql
psql -v ON_ERROR_STOP=1 -f ${DIR}/system/update_sequences.sql
psql -v ON_ERROR_STOP=1 -f system/audit_tables.sql
psql -v ON_ERROR_STOP=1 -f system/audit_views.sql
psql -v ON_ERROR_STOP=1 -f system/update_sequences.sql

# Demo data
if [[ "$DEMO" -eq 1 ]]; then
Expand Down
220 changes: 157 additions & 63 deletions system/audit.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,17 @@ COMMENT ON COLUMN qwat_sys.logged_actions.statement_only IS '''t'' if audit even
CREATE INDEX logged_actions_relid_idx ON qwat_sys.logged_actions(relid);
CREATE INDEX logged_actions_action_tstamp_tx_stm_idx ON qwat_sys.logged_actions(action_tstamp_stm);
CREATE INDEX logged_actions_action_idx ON qwat_sys.logged_actions(action);


CREATE TABLE qwat_sys.logged_relations (
relation_name text not null,
uid_column text not null,
PRIMARY KEY (relation_name, uid_column)
);

COMMENT ON TABLE qwat_sys.logged_relations IS 'Table used to store unique identifier columns for table or views, so that events can be replayed';
COMMENT ON COLUMN qwat_sys.logged_relations.relation_name IS 'Relation (table or view) name (with schema if needed)';
COMMENT ON COLUMN qwat_sys.logged_relations.uid_column IS 'Name of a column that is used to uniquely identify a row in the relation';

CREATE OR REPLACE FUNCTION qwat_sys.if_modified_func() RETURNS TRIGGER AS $body$
DECLARE
audit_row qwat_sys.logged_actions;
Expand All @@ -72,10 +82,11 @@ DECLARE
h_new hstore;
excluded_cols text[] = ARRAY[]::text[];
BEGIN
IF TG_WHEN <> 'AFTER' THEN

IF NOT (TG_WHEN IN ('AFTER' , 'INSTEAD OF')) THEN
RAISE EXCEPTION 'qwat_sys.if_modified_func() may only run as an AFTER trigger';
END IF;

audit_row = ROW(
NEXTVAL('qwat_sys.logged_actions_event_id_seq'), -- event_id
TG_TABLE_SCHEMA::text, -- schema_name
Expand All @@ -94,75 +105,91 @@ BEGIN
NULL, NULL, -- row_data, changed_fields
'f' -- statement_only
);

IF NOT TG_ARGV[0]::BOOLEAN IS DISTINCT FROM 'f'::BOOLEAN THEN
audit_row.client_query = NULL;

END IF;

IF TG_ARGV[1] IS NOT NULL THEN
excluded_cols = TG_ARGV[1]::text[];
END IF;

IF (TG_OP = 'UPDATE' AND TG_LEVEL = 'ROW') THEN
audit_row.row_data = hstore(OLD.*);
audit_row.changed_fields = (hstore(NEW.*) - audit_row.row_data) - excluded_cols;
h_old = hstore(OLD.*) - excluded_cols;
audit_row.row_data = h_old;
h_new = hstore(NEW.*)- excluded_cols;
audit_row.changed_fields = h_new - h_old;

IF audit_row.changed_fields = hstore('') THEN
-- All changed fields are ignored. Skip this update.
RAISE WARNING '[qwat_sys.if_modified_func] - Trigger detected NULL hstore. ending';
RETURN NULL;
END IF;
INSERT INTO qwat_sys.logged_actions VALUES (audit_row.*);
RETURN NEW;

ELSIF (TG_OP = 'DELETE' AND TG_LEVEL = 'ROW') THEN
audit_row.row_data = hstore(OLD.*) - excluded_cols;
INSERT INTO qwat_sys.logged_actions VALUES (audit_row.*);
RETURN OLD;

ELSIF (TG_OP = 'INSERT' AND TG_LEVEL = 'ROW') THEN
audit_row.row_data = hstore(NEW.*) - excluded_cols;
INSERT INTO qwat_sys.logged_actions VALUES (audit_row.*);
RETURN NEW;

ELSIF (TG_LEVEL = 'STATEMENT' AND TG_OP IN ('INSERT','UPDATE','DELETE','TRUNCATE')) THEN
audit_row.statement_only = 't';
INSERT INTO qwat_sys.logged_actions VALUES (audit_row.*);
RETURN NULL;

ELSE
RAISE EXCEPTION '[qwat_sys.if_modified_func] - Trigger func added as trigger for unhandled case: %, %',TG_OP, TG_LEVEL;
RETURN NULL;
RAISE EXCEPTION USING MESSAGE = '[qwat_sys.if_modified_func] - Trigger func added as trigger for unhandled case: '||TG_OP||', '||TG_LEVEL;
RETURN NEW;
END IF;
INSERT INTO qwat_sys.logged_actions VALUES (audit_row.*);
RETURN NULL;


END;
$body$
LANGUAGE plpgsql
SECURITY DEFINER
SET search_path = pg_catalog, public
;


SET search_path = pg_catalog, public;


COMMENT ON FUNCTION qwat_sys.if_modified_func() IS $body$
Track changes TO a TABLE at the statement AND/OR row level.

Optional parameters TO TRIGGER IN CREATE TRIGGER call:

param 0: BOOLEAN, whether TO log the query text. default 't'.

param 1: text[], COLUMNS TO IGNORE IN updates. default [].

Updates TO ignored cols are omitted FROM changed_fields.

Updates WITH only ignored cols changed are NOT inserted
INTO the audit log.
Almost ALL the processing work IS still done FOR updates
that ignored. IF you need TO save the LOAD, you need TO USE
WHEN clause ON the TRIGGER instead.
No warning OR error IS issued IF ignored_cols contains COLUMNS
that do NOT exist IN the target TABLE. This lets you specify
a standard SET of ignored COLUMNS.
There IS no parameter TO disable logging of VALUES. ADD this TRIGGER AS
a 'FOR EACH STATEMENT' rather than 'FOR EACH ROW' TRIGGER IF you do NOT
want TO log row VALUES.
Note that the user name logged IS the login role FOR the session. The audit TRIGGER
cannot obtain the active role because it IS reset BY the SECURITY DEFINER invocation
of the audit TRIGGER its self.

Almost ALL the processing work IS still done for updates
that ignored. If you need to save the load, you need to use
WHEN clause on the trigger instead.

No warning or error is issued if ignored_cols contains columns
that do not exist in the target table. This lets you specify
a standard set of ignored columns.

There is no parameter to disable logging of values. Add this trigger as
a 'FOR EACH STATEMENT' rather than 'FOR EACH ROW' trigger if you do not
want to log row values.

Note that the user name logged is the login role for the session. The audit trigger
cannot obtain the active role because it is reset by the SECURITY DEFINER invocation
of the audit trigger its self.
$body$;



CREATE OR REPLACE FUNCTION qwat_sys.audit_table(target_table regclass, audit_rows BOOLEAN, audit_query_text BOOLEAN, ignored_cols text[]) RETURNS void AS $body$
DECLARE
stm_targets text = 'INSERT OR UPDATE OR DELETE OR TRUNCATE';
Expand All @@ -187,12 +214,21 @@ BEGIN
END IF;

_q_txt = 'CREATE TRIGGER audit_trigger_stm AFTER ' || stm_targets || ' ON ' ||
target_table::text ||
target_table ||
' FOR EACH STATEMENT EXECUTE PROCEDURE qwat_sys.if_modified_func('||
quote_literal(audit_query_text) || ');';
RAISE NOTICE '%',_q_txt;
EXECUTE _q_txt;


-- store primary key names
insert into qwat_sys.logged_relations (relation_name, uid_column)
select target_table, a.attname
from pg_index i
join pg_attribute a on a.attrelid = i.indrelid
and a.attnum = any(i.indkey)
where i.indrelid = target_table::regclass
and i.indisprimary
;
END;
$body$
LANGUAGE plpgsql;
Expand All @@ -215,34 +251,48 @@ $body$ LANGUAGE SQL;
-- And provide a convenience call wrapper for the simplest case
-- of row-level logging with no excluded cols and query logging enabled.
--
CREATE OR REPLACE FUNCTION qwat_sys.audit_table(target_table regclass) RETURNS void AS $$
CREATE OR REPLACE FUNCTION qwat_sys.audit_table(target_table regclass) RETURNS void AS $body$
SELECT qwat_sys.audit_table($1, BOOLEAN 't', BOOLEAN 't');
$$ LANGUAGE 'sql';
$body$ LANGUAGE 'sql';

COMMENT ON FUNCTION qwat_sys.audit_table(regclass) IS $body$
Add auditing support to the given table. Row-level changes will be logged with full client query text. No cols are ignored.
$body$;


CREATE OR REPLACE FUNCTION qwat_sys.replay_event(pevent_id int) RETURNS void AS $body$
DECLARE
query text;
BEGIN
select into query
case
when action = 'I' then
'INSERT INTO ' || schema_name || '.' || table_name ||
' ('||(select string_agg(key, ',') from each(row_data))||') VALUES ' ||
'('||(select string_agg(case when value is null then 'null' else '''' || value || '''' end, ',') from each(row_data))||')'
when action = 'D' then
'DELETE FROM ' || schema_name || '.' || table_name ||
' WHERE id=' || (row_data->'id')::text
when action = 'U' then
'UPDATE ' || schema_name || '.' || table_name ||
' SET ' || (select string_agg(key || '=' || case when value is null then 'null' else ''''||value||'''' end, ',') from each(changed_fields)) ||
' WHERE id=' || (row_data->'id')::text
end
from qwat_sys.logged_actions where event_id=pevent_id;

--raise notice '%', query;

execute query;
with
event as (
select * from qwat_sys.logged_actions where event_id = pevent_id
)
-- get primary key names
, where_pks as (
select array_to_string(array_agg(uid_column || '=' || quote_literal(row_data->uid_column)), ' AND ') as where_clause
from qwat_sys.logged_relations r
join event on relation_name = (schema_name || '.' || table_name)
)
select into query
case
when action = 'I' then
'INSERT INTO ' || schema_name || '.' || table_name ||
' ('||(select string_agg(key, ',') from each(row_data))||') VALUES ' ||
'('||(select string_agg(case when value is null then 'null' else quote_literal(value) end, ',') from each(row_data))||')'
when action = 'D' then
'DELETE FROM ' || schema_name || '.' || table_name ||
' WHERE ' || where_clause
when action = 'U' then
'UPDATE ' || schema_name || '.' || table_name ||
' SET ' || (select string_agg(key || '=' || case when value is null then 'null' else quote_literal(value) end, ',') from each(changed_fields)) ||
' WHERE ' || where_clause
end
from
event, where_pks
;

execute query;
END;
$body$
LANGUAGE plpgsql;
Expand All @@ -253,3 +303,47 @@ Replay a logged event.
Arguments:
pevent_id: The event_id of the event in qwat_sys.logged_actions to replay
$body$;

CREATE OR REPLACE FUNCTION qwat_sys.audit_view(target_view regclass, audit_query_text BOOLEAN, ignored_cols text[], uid_cols text[]) RETURNS void AS $body$
DECLARE
stm_targets text = 'INSERT OR UPDATE OR DELETE';
_q_txt text;
_ignored_cols_snip text = '';

BEGIN
EXECUTE 'DROP TRIGGER IF EXISTS audit_trigger_row ON ' || target_view::text;
EXECUTE 'DROP TRIGGER IF EXISTS audit_trigger_stm ON ' || target_view::text;

IF array_length(ignored_cols,1) > 0 THEN
_ignored_cols_snip = ', ' || quote_literal(ignored_cols);
END IF;
_q_txt = 'CREATE TRIGGER audit_trigger_row INSTEAD OF INSERT OR UPDATE OR DELETE ON ' ||
target_view::TEXT ||
' FOR EACH ROW EXECUTE PROCEDURE qwat_sys.if_modified_func(' ||
quote_literal(audit_query_text) || _ignored_cols_snip || ');';
RAISE NOTICE '%',_q_txt;
EXECUTE _q_txt;

-- store uid columns if not already present
IF (select count(*) from qwat_sys.logged_relations where relation_name = (select target_view)::text AND uid_column= (select unnest(uid_cols))::text) = 0 THEN
insert into qwat_sys.logged_relations (relation_name, uid_column)
select target_view, unnest(uid_cols);
END IF;

END;
$body$
LANGUAGE plpgsql;

COMMENT ON FUNCTION qwat_sys.audit_view(regclass, BOOLEAN, text[], text[]) IS $body$
ADD auditing support TO a VIEW.

Arguments:
target_view: TABLE name, schema qualified IF NOT ON search_path
audit_query_text: Record the text of the client query that triggered the audit event?
ignored_cols: COLUMNS TO exclude FROM UPDATE diffs, IGNORE updates that CHANGE only ignored cols.
uid_cols: MANDATORY COLUMNS to use to uniquely identify a row from the view (in order to replay UPDATE and DELETE)

Example:
SELECT qwat_sys.audit_view('qwat_od.vw_element_installation', 'true'::BOOLEAN, '{field_to_ignore}'::text[], '{key_field1, keyfield2}'::text[])
$body$;

40 changes: 20 additions & 20 deletions system/audit_tables.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,26 +7,26 @@ PERFORM qwat_sys.audit_table('qwat_dr.constructionpoint');
PERFORM qwat_sys.audit_table('qwat_dr.dimension_distance');
PERFORM qwat_sys.audit_table('qwat_dr.dimension_orientation');

PERFORM qwat_sys.audit_table('qwat_od.distributor');
PERFORM qwat_sys.audit_table('qwat_od.district');
PERFORM qwat_sys.audit_table('qwat_od.hydrant');
PERFORM qwat_sys.audit_table('qwat_od.installation');
PERFORM qwat_sys.audit_table('qwat_od.pressurecontrol');
PERFORM qwat_sys.audit_table('qwat_od.pump');
PERFORM qwat_sys.audit_table('qwat_od.source');
PERFORM qwat_sys.audit_table('qwat_od.treatment');
PERFORM qwat_sys.audit_table('qwat_od.tank');
PERFORM qwat_sys.audit_table('qwat_od.chamber');
PERFORM qwat_sys.audit_table('qwat_od.leak');
PERFORM qwat_sys.audit_table('qwat_od.meter');
PERFORM qwat_sys.audit_table('qwat_od.pipe');
PERFORM qwat_sys.audit_table('qwat_od.pressurezone');
PERFORM qwat_sys.audit_table('qwat_od.printmap');
PERFORM qwat_sys.audit_table('qwat_od.protectionzone');
PERFORM qwat_sys.audit_table('qwat_od.samplingpoint');
PERFORM qwat_sys.audit_table('qwat_od.subscriber');
PERFORM qwat_sys.audit_table('qwat_od.subscriber_reference');
PERFORM qwat_sys.audit_table('qwat_od.surveypoint');
-- PERFORM qwat_sys.audit_table('qwat_od.distributor');
-- PERFORM qwat_sys.audit_table('qwat_od.district');
-- PERFORM qwat_sys.audit_table('qwat_od.hydrant');
-- PERFORM qwat_sys.audit_table('qwat_od.installation');
-- PERFORM qwat_sys.audit_table('qwat_od.pressurecontrol');
-- PERFORM qwat_sys.audit_table('qwat_od.pump');
-- PERFORM qwat_sys.audit_table('qwat_od.source');
-- PERFORM qwat_sys.audit_table('qwat_od.treatment');
-- PERFORM qwat_sys.audit_table('qwat_od.tank');
-- PERFORM qwat_sys.audit_table('qwat_od.chamber');
-- PERFORM qwat_sys.audit_table('qwat_od.leak');
-- PERFORM qwat_sys.audit_table('qwat_od.meter');
-- PERFORM qwat_sys.audit_table('qwat_od.pipe');
-- PERFORM qwat_sys.audit_table('qwat_od.pressurezone');
-- PERFORM qwat_sys.audit_table('qwat_od.printmap');
-- PERFORM qwat_sys.audit_table('qwat_od.protectionzone');
-- PERFORM qwat_sys.audit_table('qwat_od.samplingpoint');
-- PERFORM qwat_sys.audit_table('qwat_od.subscriber');
-- PERFORM qwat_sys.audit_table('qwat_od.subscriber_reference');
-- PERFORM qwat_sys.audit_table('qwat_od.surveypoint');
PERFORM qwat_sys.audit_table('qwat_od.valve');

PERFORM qwat_sys.audit_table('qwat_vl.cistern');
Expand Down
27 changes: 27 additions & 0 deletions system/audit_views.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
/* Audit views only for those having explicit triggers handling data editing*/
DO $$
BEGIN
PERFORM qwat_sys.audit_view('qwat_od.vw_consumptionzone', 'true'::boolean, '{}'::text[], '{id}'::text[]);
PERFORM qwat_sys.audit_view('qwat_od.vw_element_hydrant', 'true'::boolean, '{}'::text[], '{id}'::text[]);
PERFORM qwat_sys.audit_view('qwat_od.vw_element_installation', 'true'::boolean, '{}'::text[], '{id}'::text[]);
PERFORM qwat_sys.audit_view('qwat_od.vw_element_meter', 'true'::boolean, '{}'::text[], '{id}'::text[]);
PERFORM qwat_sys.audit_view('qwat_od.vw_element_part', 'true'::boolean, '{}'::text[], '{id}'::text[]);
PERFORM qwat_sys.audit_view('qwat_od.vw_element_samplingpoint', 'true'::boolean, '{}'::text[], '{id}'::text[]);
PERFORM qwat_sys.audit_view('qwat_od.vw_element_subscriber', 'true'::boolean, '{}'::text[], '{id}'::text[]);
-- PERFORM qwat_sys.audit_view('qwat_od.vw_node_element', 'true'::boolean, '{}'::text[], '{id}'::text[]);
-- PERFORM qwat_sys.audit_view('qwat_od.vw_pipe_child_parent', 'true'::boolean, '{}'::text[], '{id}'::text[]);
-- PERFORM qwat_sys.audit_view('qwat_od.vw_pipe_schema_merged', 'true'::boolean, '{}'::text[], '{id}'::text[]);
-- PERFORM qwat_sys.audit_view('qwat_od.vw_pipe_schema', 'true'::boolean, '{}'::text[], '{id}'::text[]);
-- PERFORM qwat_sys.audit_view('qwat_od.vw_pipe_schema_error', 'true'::boolean, '{}'::text[], '{id}'::text[]);
-- PERFORM qwat_sys.audit_view('qwat_od.vw_pipe_schema_visibleitems', 'true'::boolean, '{}'::text[], '{id}'::text[]);
-- PERFORM qwat_sys.audit_view('qwat_od.vw_printmap', 'true'::boolean, '{}'::text[], '{id}'::text[]);
PERFORM qwat_sys.audit_view('qwat_od.vw_protectionzone', 'true'::boolean, '{}'::text[], '{id}'::text[]);
PERFORM qwat_sys.audit_view('qwat_od.vw_qwat_installation', 'true'::boolean, '{}'::text[], '{id}'::text[]);
PERFORM qwat_sys.audit_view('qwat_od.vw_qwat_network_element', 'true'::boolean, '{}'::text[], '{id}'::text[]);
PERFORM qwat_sys.audit_view('qwat_od.vw_qwat_node', 'true'::boolean, '{}'::text[], '{id}'::text[]);
PERFORM qwat_sys.audit_view('qwat_od.vw_remote', 'true'::boolean, '{}'::text[], '{id}'::text[]);
-- PERFORM qwat_sys.audit_view('qwat_od.vw_search_view', 'true'::boolean, '{}'::text[], '{id}'::text[]);
-- PERFORM qwat_sys.audit_view('qwat_od.vw_subscriber_pipe_relation', 'true'::boolean, '{}'::text[], '{id}'::text[]);
-- PERFORM qwat_sys.audit_view('qwat_od.vw_valve_lines', 'true'::boolean, '{}'::text[], '{id}'::text[]);
END
$$;
Loading

1 comment on commit e910256

@3nids
Copy link
Member

@3nids 3nids commented on e910256 Mar 27, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the release is missing after this model version increment

Please sign in to comment.