Converting code using hash to array - arrays

Question

I am charged with making a Perl module more efficient. It takes log entries which are currently uniquely identified by date and server name and inserts them into our mysql database for later processing.
I've posted the original file and my attempt below. My code doesn't insert into the database. I'm sure its a simple array handling problem but not sure.
Original file
package UlsaSpectrumAnalyser;
use strict;
use warnings;
use Data::Dumper;
use EnmServiceGroup;
use StatsDB;
use DBI;
use StatsTime;
sub new
{
my $klass = shift;
my $self = bless {}, $klass;
return $self;
}
sub init($$$$)
{
my ($self,$r_cliArgs,$r_incr,$dbh) = #_;
$self->{'site'} = $r_cliArgs->{'site'};
$self->{'siteId'} = $r_cliArgs->{'siteId'};
$self->{'date'} = $r_cliArgs->{'date'};
if ( exists $r_incr->{'UlsaSpectrumAnalyser'} )
{
$self->{'r_UlsaSpectrumAnalyser'} = $r_incr->{'UlsaSpectrumAnalyser'}->{'r_UlsaSpectrumAnalyser'};
}
else
{
$self->{'r_UlsaSpectrumAnalyser'} = {};
}
my #subscriptions = ();
$self->{'serverMap'} = {};
foreach my $service( "pmservice", "saservice" ) {
my $r_serverMap = enmGetServiceGroupInstances($self->{'site'}, $self->{'date'},$service);
while ( my ($server,$serverId) = each %{$r_serverMap} ) {
push ( #subscriptions, {'server' => $server, 'prog' => 'JBOSS'} );
$self->{'serverMap'}->{$server} = $serverId;
}
}
return \#subscriptions;
}
sub handle($$$$$$$)
{
my ($self,$timestamp,$host,$program,$severity,$message,$messageSize) = #_;
if ( $::DEBUG > 9 ) { print "UlsaSpectrumAnalyser::handle got message from $host $program : $message\n"; }
# Skip any warnings/errors
if ( $severity ne 'info' ) {
return;
}
my ($time)=$timestamp=~/(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}).*/;
my ($epochtime) = getDateTimeInMilliSeconds($timestamp);
if ( $::DEBUG > 3 ) { print "UlsaSpectrumAnalyser::handle got message from $time $host $program : $message\n"; }
#Sample Log
#2017-09-08 14:55:36,028 INFO [com.X.Y.itpf.EVENT_LOGGER] (ajp-executor-threads - 18) [administrator, Spectrum Analyzer, DETAILED, #ULSA_COMPONENT_FFT, MeContext=lienb4003, Samples=65510; FileParsingTime(ms)=50; FastFourierTime(ms)=370; PostProcessingTime(ms)=3; #ChartScalingTime(ms)=1; TotalTime(ms)=424]
if( $message =~ /.*ULSA_COMPONENT_FFT,\s+(\S+),\s+Samples=(\d+);\s+FileParsingTime\S+=(\d+);\s+FastFourierTime\S+=(\d+);\s+PostProcessingTime\S+=(\d+);\s+ChartScalingTime\S+=(\d+);\s+TotalTime\S+=(\d+)]/ ) {
my $activity = $epochtime . '##' . $host; #TODO Handle array instead of hash. Working on separately
my $serverid = $self->{'serverMap'}->{$host};
$self->{'r_UlsaSpectrumAnalyser'}->{$activity} = {
'time' => $time,
'epochtime' => $epochtime,
'serverid' => $serverid,
'source' => $1,
'sample' => $2,
'file_parsing_time' => $3,
'fast_fourier_time' => $4,
'post_processing_time' => $5,
'chart_scaling_time' => $6,
'total_time' => $7};
}
}
sub handleExceeded($$$)
{
my ($self, $host, $program) = #_;
}
sub done($$$)
{
my ($self,$dbh,$r_incr) = #_;
my $tmpDir = '/data/tmp';
my $date=$self->{'date'};
if (exists $ENV{'TMP_DIR'})
{
$tmpDir = $ENV{'TMP_DIR'};
}
my $bcpFileUlsaAnalyserLogs = "$tmpDir/ulsa_spectrum_analyser_logs.bcp";
open (BCP, "> $bcpFileUlsaAnalyserLogs") or die "Failed to open $bcpFileUlsaAnalyserLogs";
foreach my $activity (sort keys %{$self->{'r_UlsaSpectrumAnalyser'}}) {
print BCP $self->{'siteId'} . "\t" .
$self->{'r_UlsaSpectrumAnalyser'}->{$activity}->{'serverid'} . "\t" .
$self->{'r_UlsaSpectrumAnalyser'}->{$activity}->{'time'} . "\t" .
$self->{'r_UlsaSpiUectrumAnalyser'}->{$activity}->{'epochtime'} . "\t" .
$self->{'r_UlsaSpectrumAnalyser'}->{$activity}->{'source'} . "\t" .
$self->{'r_UlsaSpectrumAnalyser'}->{$activity}->{'sample'} . "\t" .
$self->{'r_UlsaSpectrumAnalyser'}->{$activity}->{'file_parsing_time'} . "\t" .
$self->{'r_UlsaSpectrumAnalyser'}->{$activity}->{'fast_fourier_time'} . "\t" .
$self->{'r_UlsaSpectrumAnalyser'}->{$activity}->{'post_processing_time'} . "\t" .
$self->{'r_UlsaSpectrumAnalyser'}->{$activity}->{'chart_scaling_time'} . "\t" .
$self->{'r_UlsaSpectrumAnalyser'}->{$activity}->{'total_time'} . "\n";
}
close BCP;
dbDo( $dbh, "DELETE FROM enm_ulsa_spectrum_analyser_logs WHERE siteid = $self->{'siteId'} AND time BETWEEN '$date 00:00:00' AND '$date 23:59:59'" )
or die "Failed to delete from enm_ulsa_spectrum_analyser_logs" . $dbh->errstr;
dbDo( $dbh, "LOAD DATA INFILE '$bcpFileUlsaAnalyserLogs' INTO TABLE enm_ulsa_spectrum_analyser_logs" )
or die "Failed to load new data from '$bcpFileUlsaAnalyserLogs' file to 'enm_ulsa_spectrum_analyser_logs' table" . $dbh->errstr;
unlink($bcpFileUlsaAnalyserLogs);
$r_incr->{'UlsaSpectrumAnalyser'} = {
'r_UlsaSpectrumAnalyser' => $self->{'r_UlsaSpectrumAnalyser'}
};
}
1;
My attempt
package UlsaSpectrumAnalyser;
use strict;
use warnings;
use Data::Dumper;
use StatsDB;
use DBI;
use StatsTime;
sub new
{
my $klass = shift;
my $self = bless {}, $klass;
return $self;
}
sub init($$$$)
{
my ($self,$r_cliArgs,$r_incr,$dbh) = #_;
$self->{'site'} = $r_cliArgs->{'site'};
$self->{'siteId'} = $r_cliArgs->{'siteId'};
$self->{'date'} = $r_cliArgs->{'date'};
$self->{'r_instrDataEvent'}->{'ulsaStats'} = [];
my #subscriptions = ();
$self->{'serverMap'} = {};
foreach my $service( "pmservice", "saservice" ) {
my $r_serverMap = enmGetServiceGroupInstances($self->{'site'}, $self->{'date'},$service);
while ( my ($server,$serverId) = each %{$r_serverMap} ) {
push ( #subscriptions, {'server' => $server, 'prog' => 'JBOSS'} );
$self->{'serverMap'}->{$server} = $serverId;
}
}
return \#subscriptions;
}
sub handle($$$$$$$)
{
my ($self,$timestamp,$host,$program,$severity,$message,$messageSize) = #_;
if ( $::DEBUG > 9 ) { print "UlsaSpectrumAnalyser::handle got message from $host $program : $message\n"; }
# Skip any warnings/errors
if ( $severity ne 'info' ) {
return;
}
my ($time)=$timestamp=~/(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}).*/;
my ($epochtime) = getDateTimeInMilliSeconds($timestamp);
if ( $::DEBUG > 3 ) { print "UlsaSpectrumAnalyser::handle got message from $time $host $program : $message\n"; }
#Sample Log
#2017-09-08 14:55:36,028 INFO [com.X.y.itpf.EVENT_LOGGER] (ajp-executor-threads - 18) [administrator, Spectrum Analyzer, DETAILED, #ULSA_COMPONENT_FFT, MeContext=lienb4003, Samples=65510; FileParsingTime(ms)=50; FastFourierTime(ms)=370; PostProcessingTime(ms)=3; #ChartScalingTime(ms)=1; TotalTime(ms)=424]
if( $message =~ /.*ULSA_COMPONENT_FFT,\s+(\S+),\s+Samples=(\d+);\s+FileParsingTime\S+=(\d+);\s+FastFourierTime\S+=(\d+);\s+PostProcessingTime\S+=(\d+);\s+ChartScalingTime\S+=(\d+);\s+TotalTime\S+=(\d+)]/ ) {
my $serverid = $self->{'r_instrDataEvent'}->{'serverid'};
my %event = (
'time' => $time,
'epochtime' => $epochtime,
'serverid' => $serverid,
'source' => $1,
'sample' => $2,
'file_parsing_time' => $3,
'fast_fourier_time' => $4,
'post_processing_time' => $5,
'chart_scaling_time' => $6,
'total_time' => $7
);
push #{$self->{'r_instrDataEvent'}->{'ulsaStats'}}, \%event;
}
}
sub handleExceeded($$$)
{
my ($self, $host, $program) = #_;
}
sub done($$$)
{
my ($self,$dbh,$r_incr) = #_;
my $bcpFileUlsaAnalyserLogs = getBcpFilename("enm_ulsa_spectrum_analyser_logs ");
open (BCP, "> $bcpFileUlsaAnalyserLogs") or die "Failed to open $bcpFileUlsaAnalyserLogs";
foreach my $activity (#{$self->{'r_instrDataEvent'}->{'ulsaStats'}}) {
print BCP $self->{'siteId'} . "\t" .
$activity->{'serverid'} . "\t" .
$activity->{'time'} . "\t" .
$activity->{'epochtime'} . "\t" .
$activity->{'source'} . "\t" .
$activity->{'sample'} . "\t" .
$activity->{'file_parsing_time'} . "\t" .
$activity->{'fast_fourier_time'} . "\t" .
$activity->{'post_processing_time'} . "\t" .
$activity->{'chart_scaling_time'} . "\t" .
$activity->{'total_time'} . "\n";
}
close BCP;
dbDo( $dbh, sprintf("DELETE FROM enm_ulsa_spectrum_analyser_logs WHERE siteid = %d AND time BETWEEN '%s' AND '%s'",
$self->{'siteId'}, $self->{'r_instrDataEvent'}->{'ulsaStats'}->[0]->{'epochtime'},$self->{'r_instrDataEvent'}->{'ulsaStats'}->[$#{$self->{'r_instrDataEvent'}->{'ulsaStats'}}]->{'epochtime'}))
or die "Failed to delete from enm_ulsa_spectrum_analyser_logs" . $dbh->errstr;#'
dbDo( $dbh, "LOAD DATA INFILE '$bcpFileUlsaAnalyserLogs' INTO TABLE enm_ulsa_spectrum_analyser_logs" )
or die "Failed to load new data from '$bcpFileUlsaAnalyserLogs' file to 'enm_ulsa_spectrum_analyser_logs' table" . $dbh->errstr;
}
1;

Answer 1

I would say that the problem is your
my $bcpFileUlsaAnalyserLogs = getBcpFilename( "enm_ulsa_spectrum_analyser_logs " )
I doubt if the file name really has a space at the end!
I'm surprised that your dbDo or at least your
or die "Failed to load new data ... "
didn't catch this and report it. I don't like wrapping DBI code inside subroutines, especially when I can't see those subroutines.
Note also that you should be using placeholders
dbDo( $dbh, sprintf("DELETE FROM enm_ulsa_spectrum_analyser_logs WHERE siteid = %d AND time BETWEEN '%s' AND '%s'",
$self->{'siteId'}, $self->{'r_instrDataEvent'}->{'ulsaStats'}->[0]->{'epochtime'},$self->{'r_instrDataEvent'}->{'ulsaStats'}->[$#{$self->{'r_instrDataEvent'}->{'ulsaStats'}}]->{'epochtime'}))
or die "Failed to delete from enm_ulsa_spectrum_analyser_logs" . $dbh->errstr;#'
would be much better as
my $sth = $dbh->prepare(<<END_SQL);
DELETE FROM enm_ulsa_spectrum_analyser_logs
WHERE siteid = ? AND time BETWEEN ? AND ?
END_SQL
my $ulsa_stats = $self->{r_instrDataEvent}{ulsaStats};
$sth->execute(
$self->{siteId},
$ulsa_stats->[0]{epochtime},
$ulsa_stats->[-1]{epochtime}
) or die "Failed to delete from enm_ulsa_spectrum_analyser_logs: " . $dbh->errstr;