I've created something like this:
begin;
create table test (
    id integer
);
insert into test(id) select generate_series(1,100);
create or replace function trg_check_max_4_updated_records() 
returns trigger as $$
declare
    counter_ integer := 0;
    tablename_ text := 'temptable';
begin
    raise notice 'trigger fired';
    select count(42) into counter_ 
        from pg_catalog.pg_tables where tablename = tablename_;
    if counter_ = 0 then
        raise notice 'Creating table %', tablename_;
        execute 'create temporary table ' || tablename_ || ' (counter integer) on commit drop';
        execute 'insert into ' || tablename_ || ' (counter) values(1)';
        execute 'select counter from ' || tablename_ into counter_;
        raise notice 'Actual value for counter= [%]', counter_;
    else
        execute 'select counter from ' || tablename_ into counter_;
        execute 'update ' || tablename_ || ' set counter = counter + 1';
        raise notice 'updating';
        execute 'select counter from ' || tablename_ into counter_;
        raise notice 'Actual value for counter= [%]', counter_;
        if counter_ > 4 then
            raise exception 'Cannot change more than 4 rows in one trancation';
        end if;
    end if;
    return new;
end; $$ language plpgsql;
create trigger trg_bu_test before 
  update on test 
  for each row
  execute procedure trg_check_max_4_updated_records();
update test set id = 10 where id <= 1;
update test set id = 10 where id <= 2;
update test set id = 10 where id <= 3;
update test set id = 10 where id <= 4;
update test set id = 10 where id <= 5;
rollback;
The main idea is to have a trigger on 'before update for each row' that creates (if necessary) a temporary table (that is dropped at the end of transaction). In this table there is just one row with one value, that is the number of updated rows in current transaction. For each update the value is incremented. If the value is bigger than 4, the transaction is stopped.
But I think that this is a wrong solution for your problem. What's a problem to run such wrong query that you've written about, twice, so you'll have 8 rows changed. What about deletion rows or truncating them?