Observed System State

Raw signals, no interpretation

Events Observed
1,284
Last 24 hours
Active Sources
17
Hosts emitting signals
Background Noise
93%
Unclassified traffic

Log volume (last 24h)

Error constellations (last 24h)
Observer Script

Why Perl

Because it gets the job done.

This is a small site with a simple need. Perl is very good at reading logs, parsing text, and writing to databases, and I have been using it for over twenty years. There was no reason to add complexity.

What this script does

This script ingests an nginx access log into MySQL.

It runs repeatedly, reads only new log entries, and inserts each request as a row. It does not analyze, aggregate, or interpret the data. It records what nginx emitted.

How it works

  • Tracks inode and byte offset to avoid rereading logs
  • Resets safely on log rotation or truncation
  • Parses standard nginx fields per line
  • Inserts every line, even if parsing fails
  • Commits once per run

Design intent

This is an observer, not a pipeline.

No daemon. No framework. No transformation.

Just raw signals, stored as-is.

#!/usr/bin/env perl
use strict;
use warnings;
use Fcntl qw(SEEK_SET);
use DBI;

# ------------------------------------------------------------
# CONFIG
# ------------------------------------------------------------

my $logfile = shift or die "Usage: $0 /path/to/access.log
";
#my $state_file = "$logfile.state";
my $state_file = "/var/lib/nginx-log-ingest/state.json";

my $db_host = 'db_host_to_send_metrics';     # metrics host private IP
my $db_name = 'metrics';
my $db_user = 'YOURUSER';
my $db_pass = 'YOURPASSWORD';

# ------------------------------------------------------------
# DB CONNECT
# ------------------------------------------------------------

my $dsn = "DBI:mysql:database=$db_name;host=$db_host";
my $dbh = DBI->connect(
    $dsn,
    $db_user,
    $db_pass,
    {
        RaiseError => 0,
        AutoCommit => 0,
        mysql_enable_utf8mb4 => 1,
    }
) or die "DB connect failed";

my $sth = $dbh->prepare(q{
    INSERT INTO nginx_access_logs (
        ts_raw,
        client_ip,
        method,
        path,
        protocol,
        status,
        bytes,
        referer,
        user_agent,
        parse_ok
    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
});

# ------------------------------------------------------------
# FRAME-LEVEL REGEX
# ------------------------------------------------------------

my $FRAME_RE = qr{
    ^
    (S+)
    s+S+s+S+s+
    [([^]]+)]s+
    "([^"]*)"
    s+
    (d{3}|-)
    s+
    (d+|-)
    (?:s+"([^"]*)")?
    (?:s+"([^"]*)")?
}x;

# ------------------------------------------------------------
# HELPERS
# ------------------------------------------------------------

sub load_state {
    my ($file) = @_;
    return {} unless -f $file;

    open my $fh, '<', $file or die "Cannot read state file: $!";
    my %state;
    while (<$fh>) {
        chomp;
        my ($k, $v) = split /=/, $_, 2;
        $state{$k} = $v if defined $k;
    }
    close $fh;
    return %state;
}

sub save_state {
    my ($file, $state) = @_;
    open my $fh, '>', $file or die "Cannot write state file: $!";
    for my $k (sort keys %$state) {
        print $fh "$k=$state->{$k}
";
    }
    close $fh;
}

sub nullify {
    my ($v) = @_;
    return undef if !defined $v || $v eq '' || $v eq '-';
    return $v;
}

# ------------------------------------------------------------
# LOAD STATE
# ------------------------------------------------------------

my $state = load_state($state_file);
my $stored_inode  = $state->{inode};
my $stored_offset = $state->{offset} // 0;

# ------------------------------------------------------------
# STAT FILE
# ------------------------------------------------------------

open my $fh, '<', $logfile or die "Cannot open log file";
my @stat = stat($fh) or die "Cannot stat log file";
my $inode     = $stat[1];
my $file_size = $stat[7];

# ------------------------------------------------------------
# OFFSET / ROTATION DECISION
# ------------------------------------------------------------

my $start_offset = 0;

if (!defined $stored_inode) {
    $start_offset = 0;
}
elsif ($stored_inode != $inode) {
    $start_offset = 0;
}
elsif ($file_size < $stored_offset) {
    $start_offset = 0;
}
else {
    $start_offset = $stored_offset;
}

seek($fh, $start_offset, SEEK_SET);

# ------------------------------------------------------------
# MAIN LOOP
# ------------------------------------------------------------

my $current_offset = $start_offset;
my $inserted = 0;

while (my $line = <$fh>) {
    $current_offset = tell($fh);
    chomp $line;
    my (
        $ip,
        $ts_raw,
        $request,
        $status,
        $bytes,
        $referer,
        $ua
    );

    my ($method, $path, $proto) = ('', '', '');
    my $parse_ok = 0;

    if ($line =~ $FRAME_RE) {
        (
            $ip,
            $ts_raw,
            $request,
            $status,
            $bytes,
            $referer,
            $ua
        ) = ($1,$2,$3,$4,$5,$6,$7);

        if ($request =~ /^(S+)s+(S+)(?:s+(S+))?$/) {
            $method = $1;
            $path   = $2;
            $proto  = $3 // '';
            $parse_ok = 1;
        }
    }

    $sth->execute(
        nullify($ts_raw),
        nullify($ip),
        nullify($method),
        nullify($path),
        nullify($proto),
        nullify($status),
        nullify($bytes),
        nullify($referer),
        nullify($ua),
        $parse_ok
    );

    $inserted++;
}

close $fh;

# ------------------------------------------------------------
# COMMIT + SAVE STATE
# ------------------------------------------------------------

$dbh->commit;

$state->{inode}  = $inode;
$state->{offset} = $current_offset;
save_state($state_file, $state);

$sth->finish;
$dbh->disconnect;

print "Ingest complete. Rows inserted: $inserted
";