#!/bin/sh

# wget http://white-barn.com/tmp/data_locks_keys.zip

time unzip -p data_locks_keys.zip \
  | psql -c "
    drop table if exists data_locks_keys;
    create table data_locks_keys(key text, source text);
    copy data_locks_keys from stdin with (
      format csv,
      delimiter E'|',
      header FALSE
    );
"
#  main table data_locks_keys now has 9,197,129 rows

echo "
create table data_locks_keys_distinct
as
    select distinct on (key) key, source
    from data_locks_keys
;
" | psql

# derived table data_locks_keys_distinct has 3,692,089 rows (with column 'key' now unique)