in reply to Multiple queries on DBI corresponding to multiple csv files?

I have included some features and still a bit redundant on the condition "if id exists", it is ok for now and open to suggestions.

This is the result:

#!/usr/bin/perl -W

# This script will create multiple csv files based on 
# passed arguments "id" and custom "where" clause. Also,
# in the future it will read and display it as
# floating boxes in an HTML page (D3JS does the trick)

use DBI;
use Text::CSV_XS;


# Constant:
# my $htmlpage = "htmlfile_$id"; #TODO: it will build the html file


# Here are the temporary params (TODO: it will use @ARGV)
# This is an important param because it will run bunch of queries
# just for the provided id, if no id was provided then
# it will run the queries for all ids from the table
my $id = "";


# DBI CONNECTION
my $dbh = get_dbh();


# SUBROUTINES
sub run_query 
{
    my ($sql, $csvfile) = @_;
    print "Running $sql\n";
    
    open my $fh, '>', $csvfile or die "Could not open $csvfile: $!";

    my $csv = Text::CSV_XS->new ({ binary => 1, eol => "\n" });
    my $sth = $dbh->prepare ($sql); 
    $sth->execute;
    $csv->print($fh, $sth->{NAME_lc});
    my $count = 1;
    while (my $row = $sth->fetchrow_arrayref)
    {
        $csv->print($fh, $row);
        ++$count;
    }
    close $fh; 
    
    print "ID: $id, $count lines dumped to $csvfile\n";
}

sub get_dbh
{
    my $dbname = "jtech";
    my $user = "jtech";
    
    my($dbh) = DBI->connect("dbi:IngresII:$dbname","$user","")
    or die "Could not connect to database $dbname\n";
    
    return $dbh;
}



# MAIN PROGRAM

# If the user type an ID it will generate 4 csv files
# that corresponds to each custom SQL query
if ($id)
{

    # In this case it generates reports for the typed "id" only 
    $clause = "id=$id AND";
    
    my @query = (
        ["SELECT id FROM mytable WHERE $clause salary >= 1","csvfile1_
+$id.csv"],
        ["SELECT name FROM mytable WHERE $clause salary >= 1","csvfile
+2_$id.csv"],
        ["SELECT salary FROM mytable WHERE $clause salary >= 1","csvfi
+le3_$id.csv"],
        ["SELECT dept FROM mytable WHERE $clause salary >= 1","csvfile
+4_$id.csv"],
    );
    
    for (@query)
    {
        run_query(@$_);
    }
}
# If there is NO ID provided than it will generate csv files
# corresponding to all ids found with a special clause where 
# active ='no' (all IDs will get 4 csv file)
else
{
    # Returns a list of all existents "id" (TO BE UPDATED TO USE DBI)
    my $cmd = `echo 'SELECT DISTINCT id from mytable\\g\\q' | sql jtec
+h -jtech -S | sed -e 's/ //g'`;
    
    foreach $cmd (split /[\n\r]+/, $cmd)
    {
        $id = $cmd;
        
        $clause = "active ='no' AND";
        
        my @query = (
            ["SELECT id FROM mytable WHERE $clause salary >= 1","csvfi
+le1_$id.csv"],
            ["SELECT name FROM mytable WHERE $clause salary >= 1","csv
+file2_$id.csv"],
            ["SELECT salary FROM mytable WHERE $clause salary >= 1","c
+svfile3_$id.csv"],
            ["SELECT dept FROM mytable WHERE $clause salary >= 1","csv
+file4_$id.csv"],
        );
        
        for (@query)
        {
            run_query(@$_);
        }
    }
}

[download]

Replies are listed 'Best First'.
Re^2: Multiple queries on DBI corresponding to multiple csv files?
by poj (Abbot) on Feb 19, 2019 at 12:42 UTC

    Always best to use placeholders in your queries if possible

    #!/usr/bin/perl
use strict;
use warnings;
use DBI;
use Text::CSV_XS;

# DBI CONNECTION
my $dbh = get_dbh();

my @id = @ARGV;
unless (@id){
  # no ids in ARGV
  my $sql = "SELECT DISTINCT id from mytable
             WHERE active = ?";
  @id = map{ $_->[0] } $dbh->selectall_array($sql,undef,'no');
}

# MAIN
for my $id (@id){
  my @query = (
    ["SELECT id FROM mytable     WHERE id=? AND salary >= 1","csvfile1
+_$id.csv"],
    ["SELECT name FROM mytable   WHERE id=? AND salary >= 1","csvfile2
+_$id.csv"],
    ["SELECT salary FROM mytable WHERE id=? AND salary >= 1","csvfile3
+_$id.csv"],
    ["SELECT dept FROM mytable   WHERE id=? AND salary >= 1","csvfile4
+_$id.csv"],
  );

  for (@query){
    run_query(@$_,$id);
  }
}

# SUBROUTINES
sub run_query 
{
    my ($sql, $csvfile,$id) = @_;
    print "Running $sql for $id\n";
    
    open my $fh, '>', $csvfile or die "Could not open $csvfile: $!";

    my $csv = Text::CSV_XS->new ({ binary => 1, eol => "\n" });
    my $sth = $dbh->prepare ($sql); 
    $sth->execute($id);
    $csv->print($fh, $sth->{NAME_lc});
    my $count = 1;
    while (my $row = $sth->fetchrow_arrayref)
    {
        $csv->print($fh, $row);
        ++$count;
    }
    close $fh;   
    print "ID: $id, $count lines dumped to $csvfile\n";
}

sub get_dbh
{
    my $dbname = "jtech";  my $user = "jtech";
    my $dbh = DBI->connect("dbi:IngresII:$dbname",$user,'')
      or die "Could not connect to database $dbname\n";  
    return $dbh;
}

    [download]
    poj
[reply]
[d/l]

      Good stuff!

      I have created the Menu and it is accepting params so, it works fine when the id is typed, but it doesn't work when it is left blank ""

      I am struggling with this error message: Can't locate object method "selectall_array" via package "DBI::db" at ./test.pl line 60.

      Well, that is the line 60: @id = map{ $_->[0] } $dbh->selectall_array($sql,undef,'no');

      what am I doing wrong here?

      #!/usr/bin/perl -W

use DBI;
use Text::CSV_XS;

# GLOBALS
@id = "";
$dbname = "";
$user = "";

# MAIN MENU
# Don't allow duplicated arguments and blank/undef arguments
# except for id (in this case blank/undef id means all ids)
foreach my $arg (@ARGV)
{
    if ($arg eq '-h' || $arg eq '--help')
    {
        die "Script usage manual here\n";
        exit;
    }
    
    if ($arg =~ /^(\w+)$/)
    {
        die "Error: Multiple databases: $dbname and $arg." if $dbname;
        
        $dbname = $1 ; next;
    }
    
    if ($arg =~ /^--?u(\w+)$/)
    {
        die "Error: Multiple users $user and $arg." if $user;
        
        $user = $1 ; next;
    }
    
    if ($arg =~ /^--?id(\d+)$/)
    {
        die "Error: Multiple ids specified: -id$id[0] and $arg.\n" if 
+$id[0];

        $id[0] = $1 ; next;
    }
    elsif ($arg =~ /^--?id(\S+|)$/)
    {
        die "Error id format, use -id<integer>\n";
    }    
}

die "No database, type <dbname>\n" unless $dbname;
die "No username, use -u<username>\n" unless $user;


# DBI CONNECTION
my $dbh = get_dbh();

# EXEC
if ($id[0] eq "")
{   
    my $sql = "SELECT DISTINCT id from mytable
             WHERE active = ?";
    @id = map{ $_->[0] } $dbh->selectall_array($sql,undef,'no');
}

for my $id (@id)
{
  my @query =
  (
    ["SELECT id FROM mytable     WHERE id=? AND salary >= 1","csvfile1
+_$id.csv"],
    ["SELECT name FROM mytable   WHERE id=? AND salary >= 1","csvfile2
+_$id.csv"],
    ["SELECT salary FROM mytable WHERE id=? AND salary >= 1","csvfile3
+_$id.csv"],
    ["SELECT dept FROM mytable   WHERE id=? AND salary >= 1","csvfile4
+_$id.csv"],
  );

  for (@query)
  {
    run_query(@$_,$id);
  }
}

# SUBROUTINES
sub get_dbh
{
    my($dbh) = DBI->connect("dbi:IngresII:$dbname","$user","")
        or die "Could not connect to database $dbname\n";
    
    return $dbh;
}

sub run_query 
{
    my ($sql, $csvfile, $id) = @_;
    print "Running $sql for $id\n";
    
    open my $fh, '>', $csvfile or die "Could not open $csvfile: $!";

    my $csv = Text::CSV_XS->new ({ binary => 1, eol => "\n" });
    my $sth = $dbh->prepare ($sql); 
    $sth->execute($id);
    $csv->print($fh, $sth->{NAME_lc});
    my $count = 1;
    
    while (my $row = $sth->fetchrow_arrayref)
    {
        $csv->print($fh, $row);
        ++$count;
    }
    
    close $fh;
    
    print "ID: $id, $count lines dumped to $csvfile\n";
}

      [download]
[reply]
[d/l]
[select]

        Which version of DBI are you running? select_array was added in v1.635, 24/04/2016.

        Update: from the command line, get DBI version: perl -MDBI -e 'print $DBI::VERSION'

[reply]
[d/l]
[select]

        try the selectall_arrayref method

          #@id = map{ $_->[0] } $dbh->selectall_array($sql,undef,'no')
  my $ref =  $dbh->selectall_arrayref($sql,undef,'no');          
  @id = map{ $_->[0] } @$ref;

        [download]
        poj
[reply]
[d/l]
[select]

      New question :)

      Any way to create a temporary table in the array @query and make it contents available to the other queries?

      Something like...

      my @query =
(
    [DECLARE GLOBAL TEMPORARY TABLE SESSION.temp SELECT id FROM mytabl
+e ON COMMIT PRESERVE ROWS WITH NORECOVERY, "/dev/null"],
    ["SELECT name FROM SESSION.temp   WHERE id=? AND salary >= 1","csv
+file2_$id.csv"],
}

      [download]
[reply]
[d/l]

        I suggest you add another 'type' field to run the non-select query with $dbh->do($sql). However you probably only need run that query once not for each id.

        for my $id (@id){
  my @query = (
    [1,"DECLARE GLOBAL TEMPORARY TABLE SESSION.temp 
        SELECT name,id FROM mytable 
        ON COMMIT PRESERVE ROWS WITH NORECOVERY"],
    [2,"SELECT name FROM SESSION.temp
        WHERE id=? AND salary >= 1","csvfile2_$id.csv"], 
    [2,"SELECT id FROM mytable
        WHERE id=? AND salary >= 1","csvfile1_$id.csv"],
  );

  for (@query){
    my ($type,$sql,$filename) = @$_;
    if ($type == 1){
      $dbh->do($sql);
    } elsif ($type == 2){
      run_query($sql,$filename,$id);
    }
  }
}

        [download]
        poj
[reply]
[d/l]
[select]

In Section Seekers of Perl Wisdom