In a similar manner in which we created an ARC file, we can create a DAT file. We simply need to modify our script, create_arc_file, in two places:

#!/usr/bin/perl
use strict;

my $i = 0;
my $drive;
my $host;
my $index;
my $filename;
my $offset;

`rm shark_index shark_html`;
foreach $host(glob "/net/ia001*"){ #iterate through each host

print "searching host $host\n";
for($drive = 0; $drive < 4; $drive ++){ # iterate through each drive

print " searching drive $drive\n";
`/local/home/brad/bin_search -all "shark" $host/$drive/crawl.cdx.gz | cut -f12,14 -d" " >> ~/shark_index`;

foreach $index(`/local/home/brad/bin_search -all "" shark_index`){

($offset, $filename) = split(" ", $index);
print "offset = $offset\n";
print "filename = $filename\n";
# #add .arc.gz to each filename
$filename = $filename.".dat.gz";
print "filename = $filename\n";
`/local/home/brad/av_getpage $host/$drive/$filename $offset >> ~/shark.dat;

}

}

}