[Dshield] Patch for the IPfilter log parser

Olivier Nicole on at cs.ait.ac.th
Thu Nov 10 05:56:39 GMT 2005


Hi,

This is a patch for the parser that is part of FreeBSD ports for
ipfilter log parser.

What the script can do:

 - reads both plain log file and compressed (gzip) log files (use
   IO::Zlib and File::Type)

 - decodes some ICMP and produce the right type and code

 - fills-in with missing ports with ??? for protocols like OSPF, PIM,
   etc.

 - checks that the log reccord blongs to IPF/ipmon (mixed syslog
   reccords)

- checks that the log reccord is really a block (ipf can generate log
   for accept/permit rules)

 - can exclude by interface where the rule applies

 - some changes in the logic of the script

Enjoy,

olivier

--- dshield-freebsd.pl	Thu Nov 10 11:14:49 2005
+++ dshield-ipf.pl	Thu Nov 10 11:27:20 2005
@@ -51,11 +51,21 @@
 #
 #       There is also an cvsup port version of p5-Net-Netmask or you can use
 #       the perl cpan method to add the module.
+#
+# Changes: Olivier Nicole, November 2005
+# - reads both plain file and compressed files (use IO::Zlib and File::Type)
+# - decodes some ICMP (see the list that I know at the end)
+# - fills-in with missing ports for protocols like OSPF, PIM, etc.
+# - checks that the log reccord blongs to IPF/ipmon (mixed syslog reccords)
+# - checks that the log reccord is really a block (ipf can log the pass rules)
+# - can exclude by interface where the rule applies
+# - some changes in the logic of the script
 
 use Net::Netmask;
 use POSIX (strftime);
 use Getopt::Std;
 use IO::File;
+use File::Type;
 
 getopts("h:f:t:c:u:l:m:e:v:s:");
 $verbose=$opt_v;
@@ -131,6 +141,7 @@
 #   TI (Target IP)
 #   SP (Source Port)
 #   TP (Target port)
+#   IF (Interface)
 # port=single value between 1-65535
 # and IP:
 #   216.240.32.0/24                     The preferred form.
@@ -182,6 +193,7 @@
 
 ($timezone = strftime("%z", localtime)) =~ s/(\d\d)(\d\d)/$1:$2/;
 
+&initicmp();
 
 debug("Checking for flag override arguments\n");
 # Command line flags to over ride script default arguments
@@ -238,13 +250,28 @@
 close(IN);
 
 
+debug("Check the file type\n");
+$m=File::Type->new();
+$logfiletype=$m->checktype_filename($logfile);
+
 debug("Opening log file\n");
-$fd = new IO::File;
-$fd->open($logfile,"r") || die "Cant open log file";
+if ($logfiletype=~/application\/x-gzip/) {
+    use IO::Zlib;
+    $fd = new IO::Zlib;
+    $mode="rb";
+    debug("This is a compressed file\n");
+}
+else {
+    $fd = new IO::File;
+    $mode="r";
+}
 
+$fd->open($logfile, $mode) || die "Cant open log file";
+
+if ($logfiletype!~/application\/x-gzip/) {
 # position offset
-if ($offset_line != 0) # not empty or absent offset file
-{
+    if ($offset_line != 0) # not empty or absent offset file
+    {
   debug("Positioning offset ($offset_line)\n");
   $fd->seek($offset_line,0); # set to old pointer value
   $line=<$fd>;
@@ -262,16 +289,16 @@
   {
     debug("Line positioning matches\n");
   }
-}
+    }
 
-if ($fd->eof()) 
-{ 
+    if ($fd->eof()) 
+    { 
   $fd->close();
    close(OUT);
    print "This log file has been previously processed.\n";
    exit;
+    } 
 } 
-
 # Prepare sendmail
 debug("Piping to sendmail\n");
 if ($verbose==1)
@@ -280,11 +307,16 @@
 { open(OUT,"| $sendmail -t -oi") || die "Cant open sendmail pipe"; }
 print(OUT "From: $email_from\n");
 print(OUT "To: $email_to\n");
-print(OUT "Cc: $email_cc\n");
+print(OUT "Cc: $email_cc\n") if defined $email_cc;
 print(OUT "Subject: FORMAT DSHIELD USERID $userid TZ $timezone ($version)\n\n");
 
 
-$curpos=$fd->tell();
+if ($logfiletype=~/application\/x-gzip/) {
+    $curpos=0;
+}
+else {
+    $curpos=$fd->tell();
+}
 debug("File pointer located at $curpos, reading logs\n");
 
 # init some control fields
@@ -302,14 +334,27 @@
   # Log line format is:
   # Aug 14 22:49:36 hostname ipmon[xx]: 22:49:35.901496 tun0 @0:16 b 200.60.255.45,4123 -> 80.13.151.54,135 PR tcp len 20 48 -S IN 
 
+    if ($logfiletype!~/application\/x-gzip/) {
   $line_curpos=$fd->tell();
-  $line=<$fd>;
+    }
+    undef $line;
+    $line=$fd->getline();
+#    $line=<$fd>;
   chop($line);
   debug("log record = $line\n");
 
   # parse line to extract relevant fields
   @f=split(/\s+/,$line);
 
+
+# refuse: Nov  9 14:25:43 firewall ipmon[64]: 14:25:42.371145 fxp0 @100:152 b 
+#         203.178.142.146 -> 192.41.170.16 PR tcp len 20 (68) frag 48 at 1432 IN 
+# for it has no port identified!
+
+# refuse log lines if the are not from ipmon and are not a block (b)
+
+    if (($f[4]=~/^ipmon\[\d+\]\:$/) && ($f[8]=~/b/i) && ($f[17] ne "frag")) {
+
   # re-init work fields to null  
   $log_line_date="";
   $log_line_time="";
@@ -323,6 +368,7 @@
   $log_line_dupnum="";
   $log_line_dupsufx="";
 
+	$log_line_interface=$f[6];
 
   $month=convmonth($f[0]);
   $log_line_date=sprintf("%04d-%02d-%02d",$year,$month,$f[1]);
@@ -359,6 +405,9 @@
          $log_line_flags=$_;
          if ($log_line_flags =~ /frag/i) { $log_line_flags=""; }
        }
+	    if ($log_line_proto =~ /ICMP/i) {
+		$log_line_icmptype=$f[19];
+	    }
      }
      else
      {
@@ -379,8 +428,54 @@
         $log_line_flags=$_;
         if ($log_line_flags =~ /frag/i) { $log_line_flags=""; }
       }
+	    if ($log_line_proto =~ /ICMP/i) {
+		$log_line_icmptype=$f[18];
+	    }
+	}
+	
+	# decode ICMP type and code
+	# 3 types of lines:
+	#   icmptype(31)/204
+	#   echo/0
+	#   unreach/port
+	if ($log_line_proto =~ /ICMP/i) {
+	    $log_line_icmptype=~/^(\w+)(\((\d+)\))?\/(\w+)/;
+	    if (defined $3) {
+		# icmptype(31)/204
+		$log_line_port_src=$3;
+		$log_line_port_trg=$4;
+	    }
+	    else {
+		# echo/0 or unreach/host
+		$log_line_port_src=$icmptype{$1};
+		my $code=$4;
+		my $type=$1;
+		if ($code=~/^\d+/) {
+		    # echo/0 (code is numeric)
+		    $log_line_port_trg=$code;
+		}
+		else {
+		    # unreach/port (code is word)
+		    $log_line_port_trg=$icmpcode{"$type/$code"};
+		}
+	    }
      }
 
+	# logic to drop excluded log records based on exclusion file 
+	if (($line =~ /$log_pattern/)
+	    && ($log_line_ip_src =~ /\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/)
+	    && ($log_line_ip_trg =~ /\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/)
+	    && (NotExcluded("IF",$log_line_interface))
+	    && (NotExcluded("SI",$log_line_ip_src))
+	    && (NotExcluded("SP",$log_line_port_src))
+	    && (NotExcluded("TI",$log_line_ip_trg))
+	    && (NotExcluded("TP",$log_line_port_trg)))
+	{
+
+	    # case of protocols not TCP and UDP
+	    $log_line_port_src="???" unless defined $log_line_port_src;
+	    $log_line_port_trg="???" unless defined $log_line_port_trg;
+	
   if ($first_time eq $yes)
   {
       debug("first time logic $first_time\n");
@@ -397,16 +492,6 @@
       debug("first time logic $first_time\n");
   }
 
-
-  # logic to drop excluded log records based on exclusion file 
-  if (($line =~ /$log_pattern/)
-   && ($log_line_ip_src =~ /[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/)
-   && ($log_line_ip_trg =~ /[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/)
-   && (NotExcluded("SI",$log_line_ip_src))
-   && (NotExcluded("SP",$log_line_port_src))
-   && (NotExcluded("TI",$log_line_ip_trg))
-   && (NotExcluded("TP",$log_line_port_trg)))
-  {
     # logic to summarize dup log records in same seconds
     if (($prev_log_line_date eq $log_line_date)
      && ($prev_log_line_time eq $log_line_time)
@@ -440,18 +525,19 @@
 
       $prev_log_line_count=1;
  
-     if ($log_line_dupsufx eq "x") {$prev_log_line_count = $log_line_dupnum};
+		if ($log_line_dupsufx eq "x") {
+		    $prev_log_line_count = $log_line_dupnum;
     }
   }
-  
-  if ($fd->eof()) 
-  {
+	}
+    }
+    if (! defined $line) {
       debug("EOF load email body line with log line contents\n");
-      print(OUT "$log_line_date $log_line_time $timezone\t");
+	print(OUT "$prev_log_line_date $prev_log_line_time $timezone\t");
       print(OUT "$log_line_author\t$prev_log_line_count\t");
-      print(OUT "$log_line_ip_src\t$log_line_port_src\t");
-      print(OUT "$log_line_ip_trg\t$log_line_port_trg\t");
-      print(OUT "$log_line_proto\t$log_line_flags\n");
+	print(OUT "$prev_log_line_ip_src\t$prev_log_line_port_src\t");
+	print(OUT "$prev_log_line_ip_trg\t$prev_log_line_port_trg\t");
+	print(OUT "$prev_log_line_proto\t$prev_log_line_flags\n");
       last;
   }
 }
@@ -491,14 +577,20 @@
     $type=$f[0];
     $value=$f[1];
 
+    next unless $type eq $excl_type;
+
     if (($excl_type =~ /^SI$/i) || ($excl_type =~ /^TI$/))
     {
       $t=new Net::Netmask($value);
-      if (($t->match($tocheck)) && ($excl_type eq $type)) { return(0); }
+      if ($t->match($tocheck)) { return(0); }
     }
     elsif (($excl_type =~ /^SP$/i) || ($excl_type =~ /^TP$/))
     {
-      if (($value==$tocheck) && ($excl_type eq $type)) { return(0); }
+      if ($value==$tocheck) { return(0); }
+    }
+    elsif (($excl_type =~ /^IF$/i))
+    {
+      if (($value eq $tocheck)) { return(0); }
     }
   }
   return(1);
@@ -510,4 +602,212 @@
   { print(STDERR @_); }
 }
 
+# ICMP name and type
+# 
+# Type	Name					Reference
+# ----	-------------------------		---------
+#   0	Echo Reply				 [RFC792]
+#   1	Unassigned				    [JBP]
+#   2	Unassigned				    [JBP]
+#   3	Destination Unreachable			 [RFC792]
+#   4	Source Quench			 	 [RFC792]
+#   5	Redirect				 [RFC792]
+#   6	Alternate Host Address			    [JBP]
+#   7	Unassigned				    [JBP]
+#   8	Echo					 [RFC792]
+#   9	Router Advertisement			[RFC1256]
+#  10	Router Selection			[RFC1256]
+#  11	Time Exceeded				 [RFC792]
+#  12	Parameter Problem			 [RFC792]
+#  13	Timestamp				 [RFC792]
+#  14	Timestamp Reply				 [RFC792]
+#  15	Information Request			 [RFC792]
+#  16	Information Reply			 [RFC792]
+#  17	Address Mask Request                     [RFC950]
+#  18	Address Mask Reply			 [RFC950]
+#  19	Reserved (for Security)			   [Solo]
+#  20-29	Reserved (for Robustness Experiment)	    [ZSu]
+#  30	Traceroute				[RFC1393]
+#  31	Datagram Conversion Error		[RFC1475]
+#  32     Mobile Host Redirect              [David Johnson]
+#  33     IPv6 Where-Are-You                 [Bill Simpson]
+#  34     IPv6 I-Am-Here                     [Bill Simpson]
+#  35     Mobile Registration Request        [Bill Simpson]
+#  36     Mobile Registration Reply          [Bill Simpson]
+#  37     Domain Name Request                     [Simpson]
+#  38     Domain Name Reply                       [Simpson]
+#  39     SKIP                                    [Markson]
+#  40     Photuris                                [Simpson]
+#  41-255 Reserved				    [JBP]
+# Many of these ICMP types have a "code" field.  Here we list the types
+# again with their assigned code fields.
+# 
+# Type    Name                                    Reference
+# ----    -------------------------               ---------
+#   0     Echo Reply                               [RFC792]
+# 
+#         Codes
+#             0  No Code
+# 
+#   1     Unassigned                                  [JBP]
+# 
+#   2     Unassigned                                  [JBP]
+# 
+#   3     Destination Unreachable                  [RFC792]
+# 
+# 	Codes
+# 	    0  Net Unreachable
+# 	    1  Host Unreachable
+#             2  Protocol Unreachable
+#             3  Port Unreachable
+#             4  Fragmentation Needed and Don't Fragment was Set
+#             5  Source Route Failed
+#             6  Destination Network Unknown
+#             7  Destination Host Unknown
+#             8  Source Host Isolated
+#             9  Communication with Destination Network is
+#                Administratively Prohibited
+#            10  Communication with Destination Host is
+#                Administratively Prohibited
+#            11  Destination Network Unreachable for Type of Service
+#            12  Destination Host Unreachable for Type of Service
+#            13  Communication Administratively Prohibited      [RFC1812]
+#            14  Host Precedence Violation                      [RFC1812]
+#            15  Precedence cutoff in effect                    [RFC1812]
+# 
+# 
+#   4     Source Quench                            [RFC792]
+#         Codes
+#             0  No Code
+# 
+#   5     Redirect                                 [RFC792]
+# 
+#         Codes
+#             0  Redirect Datagram for the Network (or subnet)
+#             1  Redirect Datagram for the Host
+#             2  Redirect Datagram for the Type of Service and Network
+#             3  Redirect Datagram for the Type of Service and Host
+# 
+#   6     Alternate Host Address                      [JBP]
+# 
+#         Codes
+#             0  Alternate Address for Host
+# 
+#   7     Unassigned                                  [JBP]
+# 
+#   8     Echo                                     [RFC792]
+# 
+#         Codes
+#             0  No Code
+# 
+#   9     Router Advertisement                    [RFC1256]
+# 
+#         Codes
+#             0  No Code
+# 
+#  10     Router Selection                        [RFC1256]
+# 
+#         Codes
+#             0  No Code
+# 
+#  11     Time Exceeded                            [RFC792]
+# 
+#         Codes
+#             0  Time to Live exceeded in Transit
+#             1  Fragment Reassembly Time Exceeded
+# 
+#  12     Parameter Problem                        [RFC792]
+# 
+#         Codes
+#             0  Pointer indicates the error
+#             1  Missing a Required Option        [RFC1108]
+#             2  Bad Length
+# 
+# 
+#  13     Timestamp                                [RFC792]
+# 
+#         Codes
+#             0  No Code
+# 
+#  14     Timestamp Reply                          [RFC792]
+# 
+#         Codes
+#             0  No Code
+# 
+#  15     Information Request                      [RFC792]
+# 
+#         Codes
+#             0  No Code
+# 
+#  16     Information Reply                        [RFC792]
+# 
+#         Codes
+#             0  No Code
+# 
+#  17     Address Mask Request                     [RFC950]
+# 
+#         Codes
+#             0  No Code
+# 
+#  18     Address Mask Reply                       [RFC950]
+# 
+#         Codes
+#             0  No Code
+# 
+#  19     Reserved (for Security)                    [Solo]
+# 
+#  20-29  Reserved (for Robustness Experiment)        [ZSu]
+# 
+#  30     Traceroute                              [RFC1393]
+# 
+#  31     Datagram Conversion Error               [RFC1475]
+# 
+#  32     Mobile Host Redirect              [David Johnson]
+# 
+#  33     IPv6 Where-Are-You                 [Bill Simpson]
+# 
+#  34     IPv6 I-Am-Here                     [Bill Simpson]
+# 
+#  35     Mobile Registration Request        [Bill Simpson]
+# 
+#  36     Mobile Registration Reply          [Bill Simpson]
+# 
+#  39     SKIP                                    [Markson]
+# 
+#  40     Photuris                                [Simpson]
+# 
+#         Code
+# 
+#         0	Reserved
+#         1	unknown security parameters index
+#         2	valid security parameters, but authentication failed
+#         3 	valid security parameters, but decryption failed
+# 
 
+sub initicmp {
+    %icmptype=(
+	"echo" => 8,
+	"echoreply" => 0,
+	"inforeply" => 16,
+	"inforeq" => 15,
+	"maskreply" => 18,
+	"maskreq" => 17,
+	"paramprob" => 12,
+	"redirect" => 5,
+	"routeradvert" => 9,
+	"routersolicit" => 10,
+	"timestamp" => 13,
+	"timestampreply" => 14,
+	"timxceed" => 11,
+	"unreach" => 3
+	);
+    
+    %icmpcode=(
+	"redirect/host" => 1,
+	"redirect/net" => 0,
+	"timxceed/transit" => 0,
+	"unreach/admin_prohibit" => 9,
+	"unreach/host" => 1,
+	"unreach/port" => 3
+	);
+}


More information about the list mailing list