1st Implementation of MTA-STS Report processing (reverted)

migrations/Version20230809184012.php

@@ -50,4 +50,9 @@ public function down(Schema $schema): void
         $this->addSql('DROP TABLE seen');
         $this->addSql('DROP TABLE users');
     }
+
+    public function isTransactional(): bool
+    {
+        return false;
+    }
 }

migrations/Version20230914181942.php

@@ -0,0 +1,40 @@
+<?php
+
+declare(strict_types=1);
+
+namespace DoctrineMigrations;
+
+use Doctrine\DBAL\Schema\Schema;
+use Doctrine\Migrations\AbstractMigration;
+
+/**
+ * Auto-generated Migration: Please modify to your needs!
+ */
+final class Version20230914181942 extends AbstractMigration
+{
+    public function getDescription(): string
+    {
+        return '';
+    }
+
+    public function up(Schema $schema): void
+    {
+        // this up() migration is auto-generated, please modify it to your needs
+        $this->addSql('RENAME TABLE records TO dmarc_records');
+        $this->addSql('RENAME TABLE reports TO dmarc_reports');
+        $this->addSql('RENAME TABLE results TO dmarc_results');
+    }
+
+    public function down(Schema $schema): void
+    {
+        // this down() migration is auto-generated, please modify it to your needs
+        $this->addSql('RENAME TABLE dmarc_records TO records');
+        $this->addSql('RENAME TABLE dmarc_reports TO reports');
+        $this->addSql('RENAME TABLE dmarc_results TO results');
+    }
+
+    public function isTransactional(): bool
+    {
+        return false;
+    }
+}

migrations/Version20230918182847.php

@@ -0,0 +1,60 @@
+<?php
+
+declare(strict_types=1);
+
+namespace DoctrineMigrations;
+
+use Doctrine\DBAL\Schema\Schema;
+use Doctrine\Migrations\AbstractMigration;
+
+/**
+ * Auto-generated Migration: Please modify to your needs!
+ */
+final class Version20230918182847 extends AbstractMigration
+{
+    public function getDescription(): string
+    {
+        return '';
+    }
+
+    public function up(Schema $schema): void
+    {
+        // this up() migration is auto-generated, please modify it to your needs
+        $this->addSql('RENAME TABLE seen TO dmarc_seen');
+        $this->addSql('CREATE TABLE mtasts_seen (id INT AUTO_INCREMENT NOT NULL, report_id INT NOT NULL, user_id INT NOT NULL, INDEX IDX_CC664AEC4BD2A4C0 (report_id), INDEX IDX_CC664AECA76ED395 (user_id), PRIMARY KEY(id)) DEFAULT CHARACTER SET utf8mb4 COLLATE `utf8mb4_unicode_ci` ENGINE = InnoDB');
+        $this->addSql('CREATE TABLE mtasts_mxrecords (id INT AUTO_INCREMENT NOT NULL, mxrecord_id INT NOT NULL, policy_id INT NOT NULL, priority INT NOT NULL, INDEX IDX_9D877D03BEADCE1D (mxrecord_id), INDEX IDX_9D877D032D29E3C6 (policy_id), PRIMARY KEY(id)) DEFAULT CHARACTER SET utf8mb4 COLLATE `utf8mb4_unicode_ci` ENGINE = InnoDB');
+        $this->addSql('CREATE TABLE mtasts_policies (id INT AUTO_INCREMENT NOT NULL, policy_domain_id INT NOT NULL, report_id INT NOT NULL, policy_type VARCHAR(255) NOT NULL, policy_string_version VARCHAR(255) DEFAULT NULL, policy_string_mode VARCHAR(255) DEFAULT NULL, policy_string_maxage INT DEFAULT NULL, summary_successful_count INT NOT NULL, summary_failed_count INT NOT NULL, INDEX IDX_6156A9636FCFA580 (policy_domain_id), INDEX IDX_6156A9634BD2A4C0 (report_id), PRIMARY KEY(id)) DEFAULT CHARACTER SET utf8mb4 COLLATE `utf8mb4_unicode_ci` ENGINE = InnoDB');
+        $this->addSql('CREATE TABLE mtasts_reports (id INT AUTO_INCREMENT NOT NULL, begin_time DATETIME NOT NULL, end_time DATETIME NOT NULL, organisation VARCHAR(255) NOT NULL, contact_info VARCHAR(255) NOT NULL, external_id VARCHAR(255) NOT NULL, PRIMARY KEY(id)) DEFAULT CHARACTER SET utf8mb4 COLLATE `utf8mb4_unicode_ci` ENGINE = InnoDB');
+        $this->addSql('CREATE TABLE mxrecords (id INT AUTO_INCREMENT NOT NULL, domain_id INT NOT NULL, name VARCHAR(255) NOT NULL, INDEX IDX_35617F40115F0EE5 (domain_id), PRIMARY KEY(id)) DEFAULT CHARACTER SET utf8mb4 COLLATE `utf8mb4_unicode_ci` ENGINE = InnoDB');
+        $this->addSql('ALTER TABLE mtasts_seen ADD CONSTRAINT FK_CC664AEC4BD2A4C0 FOREIGN KEY (report_id) REFERENCES mtasts_reports (id)');
+        $this->addSql('ALTER TABLE mtasts_seen ADD CONSTRAINT FK_CC664AECA76ED395 FOREIGN KEY (user_id) REFERENCES users (id)');
+        $this->addSql('ALTER TABLE mtasts_mxrecords ADD CONSTRAINT FK_9D877D03BEADCE1D FOREIGN KEY (mxrecord_id) REFERENCES mxrecords (id)');
+        $this->addSql('ALTER TABLE mtasts_mxrecords ADD CONSTRAINT FK_9D877D032D29E3C6 FOREIGN KEY (policy_id) REFERENCES mtasts_policies (id)');
+        $this->addSql('ALTER TABLE mtasts_policies ADD CONSTRAINT FK_6156A9636FCFA580 FOREIGN KEY (policy_domain_id) REFERENCES domains (id)');
+        $this->addSql('ALTER TABLE mtasts_policies ADD CONSTRAINT FK_6156A9634BD2A4C0 FOREIGN KEY (report_id) REFERENCES mtasts_reports (id)');
+        $this->addSql('ALTER TABLE mxrecords ADD CONSTRAINT FK_35617F40115F0EE5 FOREIGN KEY (domain_id) REFERENCES domains (id)');
+    }
+
+    public function down(Schema $schema): void
+    {
+        // this down() migration is auto-generated, please modify it to your needs
+        $this->addSql('ALTER TABLE mtasts_mxrecords DROP FOREIGN KEY FK_9D877D03BEADCE1D');
+        $this->addSql('ALTER TABLE mtasts_mxrecords DROP FOREIGN KEY FK_9D877D032D29E3C6');
+        $this->addSql('ALTER TABLE mtasts_policies DROP FOREIGN KEY FK_6156A9636FCFA580');
+        $this->addSql('ALTER TABLE mtasts_policies DROP FOREIGN KEY FK_6156A9634BD2A4C0');
+        $this->addSql('ALTER TABLE mxrecords DROP FOREIGN KEY FK_35617F40115F0EE5');
+        $this->addSql('ALTER TABLE mtasts_seen DROP FOREIGN KEY FK_CC664AEC4BD2A4C0');
+        $this->addSql('ALTER TABLE mtasts_seen DROP FOREIGN KEY FK_CC664AECA76ED395');
+        $this->addSql('DROP TABLE mtasts_mxrecords');
+        $this->addSql('DROP TABLE mtasts_policies');
+        $this->addSql('DROP TABLE mtasts_reports');
+        $this->addSql('DROP TABLE mxrecords');
+        $this->addSql('DROP TABLE mtasts_seen');
+        $this->addSql('RENAME TABLE dmarc_seen TO seen');
+    }
+
+    public function isTransactional(): bool
+    {
+        return false;
+    }
+}

src/Command/CheckmailboxCommand.php

@@ -14,9 +14,13 @@
 use SecIT\ImapBundle\Service\Imap;
 
 use App\Entity\Domains;
-use App\Entity\Reports;
-use App\Entity\Records;
-use App\Entity\Results;
+use App\Entity\MXRecords;
+use App\Entity\DMARC_Reports;
+use App\Entity\DMARC_Records;
+use App\Entity\DMARC_Results;
+use App\Entity\MTASTS_Reports;
+use App\Entity\MTASTS_Policies;
+use App\Entity\MTASTS_MXRecords;
 use App\Entity\Logs;
 
 #[AsCommand(
@@ -50,47 +54,55 @@ protected function execute(InputInterface $input, OutputInterface $output): int
         $stats=array(
             'new_emails' => 0,
             'new_domains' => 0,
-            'new_reports' => 0,
-            'new_records' => 0,
-            'new_results' => 0,
+            'new_mxrecords' => 0,
+            'new_dmarc_reports' => 0,
+            'new_dmarc_records' => 0,
+            'new_dmarc_results' => 0,
+            'new_mtasts_reports' => 0,
+            'new_mtasts_policies' => 0,
+            'new_mtasts_mxmapping' => 0,
         );
 
         $mailresult = $this->open_mailbox($this->imap);
         $stats['new_emails'] = $mailresult['num_emails'];
+
+        // dump($mailresult['reports']['dmarc_reports']);
+        // dump($mailresult['reports']['mtasts_reports']);
+        // dd();
 
-        foreach($mailresult['reports'] as $report){
-            $stats['new_reports']++;
+        foreach($mailresult['reports']['dmarc_reports'] as $dmarcreport){
+            $stats['new_dmarc_reports']++;
 
             $domain_repository = $this->em->getRepository(Domains::class);
-            $dbdomain = $domain_repository->findOneBy(array('fqdn' => $report->policy_published->domain->__toString()));
+            $dbdomain = $domain_repository->findOneBy(array('fqdn' => $dmarcreport->policy_published->domain->__toString()));
             if(!$dbdomain){
                 $stats['new_domains']++;
 
                 $dbdomain = new Domains;
-                $dbdomain->setFqdn($report->policy_published->domain->__toString());
+                $dbdomain->setFqdn($dmarcreport->policy_published->domain->__toString());
                 $this->em->persist($dbdomain);
                 $this->em->flush();
             }
-            $dbreport = new Reports;
-            $dbreport->setBeginTime((new \DateTime)->setTimestamp($report->report_metadata->date_range->begin->__toString()));
-            $dbreport->setEndTime((new \DateTime)->setTimestamp($report->report_metadata->date_range->end->__toString()));
-            $dbreport->setOrganisation($report->report_metadata->org_name->__toString());
-            $dbreport->setEmail($report->report_metadata->email->__toString());
-            $dbreport->setContactInfo($report->report_metadata->extra_contact_info->__toString());
-            $dbreport->setExternalId($report->report_metadata->report_id->__toString());
+            $dbreport = new DMARC_Reports;
+            $dbreport->setBeginTime((new \DateTime)->setTimestamp($dmarcreport->report_metadata->date_range->begin->__toString()));
+            $dbreport->setEndTime((new \DateTime)->setTimestamp($dmarcreport->report_metadata->date_range->end->__toString()));
+            $dbreport->setOrganisation($dmarcreport->report_metadata->org_name->__toString());
+            $dbreport->setEmail($dmarcreport->report_metadata->email->__toString());
+            $dbreport->setContactInfo($dmarcreport->report_metadata->extra_contact_info->__toString());
+            $dbreport->setExternalId($dmarcreport->report_metadata->report_id->__toString());
             $dbreport->setDomain($dbdomain);
-            $dbreport->setPolicyAdkim($report->policy_published->adkim->__toString());
-            $dbreport->setPolicyAspf($report->policy_published->aspf->__toString());
-            $dbreport->setPolicyP($report->policy_published->p->__toString());
-            $dbreport->setPolicySp($report->policy_published->sp->__toString());
-            $dbreport->setPolicyPct($report->policy_published->pct->__toString());
+            $dbreport->setPolicyAdkim($dmarcreport->policy_published->adkim->__toString());
+            $dbreport->setPolicyAspf($dmarcreport->policy_published->aspf->__toString());
+            $dbreport->setPolicyP($dmarcreport->policy_published->p->__toString());
+            $dbreport->setPolicySp($dmarcreport->policy_published->sp->__toString());
+            $dbreport->setPolicyPct($dmarcreport->policy_published->pct->__toString());
             $this->em->persist($dbreport);
             $this->em->flush();
 
-            foreach($report->record as $record){
-                $stats['new_records']++;
+            foreach($dmarcreport->record as $record){
+                $stats['new_dmarc_records']++;
 
-                $dbrecord = new Records;
+                $dbrecord = new DMARC_Records;
                 $dbrecord->setReport($dbreport);
                 $dbrecord->setSourceIp($record->row->source_ip->__toString());
                 $dbrecord->setCount($record->row->count->__toString());
@@ -104,9 +116,9 @@ protected function execute(InputInterface $input, OutputInterface $output): int
                 $this->em->flush();
 
                 foreach($record->auth_results->dkim as $dkim_result){
-                    $stats['new_results']++;
+                    $stats['new_dmarc_results']++;
 
-                    $dbresult = new Results;
+                    $dbresult = new DMARC_Results;
                     $dbresult->setRecord($dbrecord);
                     $dbresult->setDomain($dkim_result->domain->__toString());
                     $dbresult->setType('dkim');
@@ -116,9 +128,9 @@ protected function execute(InputInterface $input, OutputInterface $output): int
                 }
 
                 foreach($record->auth_results->spf as $spf_result){
-                    $stats['new_results']++;
+                    $stats['new_dmarc_results']++;
 
-                    $dbresult = new Results;
+                    $dbresult = new DMARC_Results;
                     $dbresult->setRecord($dbrecord);
                     $dbresult->setDomain($spf_result->domain->__toString());
                     $dbresult->setType('spf');
@@ -129,13 +141,86 @@ protected function execute(InputInterface $input, OutputInterface $output): int
             }
         }
 
+        foreach($mailresult['reports']['mtasts_reports'] as $mtastsreport){
+            $stats['new_mtasts_reports']++;
+
+            $dbreport = new MTASTS_Reports;
+            $dbreport->setOrganisation($mtastsreport->{'organization-name'});
+            $dbreport->setContactInfo($mtastsreport->{'contact-info'});
+            $dbreport->setExternalId($mtastsreport->{'report-id'});
+            $dbreport->setBeginTime(new \DateTime($mtastsreport->{'date-range'}->{'start-datetime'}));
+            $dbreport->setEndTime(new \DateTime($mtastsreport->{'date-range'}->{'end-datetime'}));
+            $this->em->persist($dbreport);
+            $this->em->flush();
+
+            foreach($mtastsreport->policies as $policy){
+                $stats['new_mtasts_policies']++;
+
+                $domain_repository = $this->em->getRepository(Domains::class);
+                $dbdomain = $domain_repository->findOneBy(array('fqdn' => $policy->policy->{'policy-domain'}));
+                if(!$dbdomain){
+                    $stats['new_domains']++;
+
+                    $dbdomain = new Domains;
+                    $dbdomain->setFqdn($policy->policy->{'policy-domain'});
+                    $this->em->persist($dbdomain);
+                    $this->em->flush();
+                }
+
+                $dbpolicy = new MTASTS_Policies;
+                $dbpolicy->setReport($dbreport);
+                $dbpolicy->setPolicyType($policy->policy->{'policy-type'});
+                $dbpolicy->setPolicyDomain($dbdomain);
+                $dbpolicy->setSummarySuccessfulCount($policy->summary->{'total-successful-session-count'});
+                $dbpolicy->setSummaryFailedCount($policy->summary->{'total-failure-session-count'});
+                $this->em->persist($dbpolicy);
+                $this->em->flush();
+
+                if(property_exists($policy->policy, 'policy-string')){
+                    $dbpolicy->setPolicyStringVersion(str_replace("version: ","",array_slice(preg_grep('/^version:.*/', $policy->policy->{'policy-string'}), 0, 1)[0]));
+                    $dbpolicy->setPolicyStringMode(str_replace("mode: ","",array_slice(preg_grep('/^mode:.*/', $policy->policy->{'policy-string'}), 0, 1)[0]));
+                    $dbpolicy->setPolicyStringMaxage(str_replace("max_age: ","",array_slice(preg_grep('/^max_age:.*/', $policy->policy->{'policy-string'}), 0, 1)[0]));
+                    $mxrecords=str_replace("mx: ","",array_values(preg_grep('/^mx:.*/', $policy->policy->{'policy-string'})));
+                    $this->em->persist($dbpolicy);
+                    $this->em->flush();
+
+                    $i=0;
+                    foreach($mxrecords as $mxrecord){
+                        $stats['new_mtasts_mxmapping']++;
+                        $i++;
+
+                        $mx_repository = $this->em->getRepository(MXRecords::class);
+                        $dbmxrecord = $mx_repository->findOneBy(array('domain' => $dbdomain, 'name' => $mxrecord));
+                        if(!$dbmxrecord){
+                            $stats['new_mxrecords']++;
+
+                            $dbmxrecord = new MXRecords;
+                            $dbmxrecord->setDomain($dbdomain);
+                            $dbmxrecord->setName($mxrecord);
+                            $this->em->persist($dbmxrecord);
+                            $this->em->flush();
+                        }
+
+                        $dbmx = new MTASTS_MXRecords;
+                        $dbmx->setMXRecord($dbmxrecord);
+                        $dbmx->setPolicy($dbpolicy);
+                        $dbmx->setPriority($i);
+                        $this->em->persist($dbmx);
+                        $this->em->flush();
+                    }
+                }
+            }
+        }
+
+        $message = 'Mailbox checked: '.$stats['new_emails'].' new emails ('.$stats['new_domains'].' domains, '.$stats['new_mxrecords'].' mx), '.$stats['new_dmarc_reports'].' new dmarc reports ('.$stats['new_dmarc_records'].' records, '.$stats['new_dmarc_results'].' results), '.$stats['new_mtasts_reports'].' new mtasts reports ('.$stats['new_mtasts_policies'].' policies, '.$stats['new_mtasts_mxmapping'].' mxmapping)';
+
         $log = new Logs;
         $log->setTime(new \DateTime);
-        $log->setMessage('Mailbox checked: '.$stats['new_emails'].' new emails, '.$stats['new_domains'].' new domains, '.$stats['new_reports'].' new reports, '.$stats['new_records'].' new records, '.$stats['new_results'].' new results.');
+        $log->setMessage($message);
         $this->em->persist($log);
         $this->em->flush();
 
-        $io->success('Mailbox checked: '.$stats['new_emails'].' new emails, '.$stats['new_domains'].' new domains, '.$stats['new_reports'].' new reports, '.$stats['new_records'].' new records, '.$stats['new_results'].' new results.');
+        $io->success($message);
 
         return Command::SUCCESS;
     }
@@ -145,36 +230,56 @@ private function open_mailbox(Imap $imap):array
         $num_emails=0;
         $mailbox = $imap->get('default');
         $mailsIds = $mailbox->searchMailbox('UNSEEN');
-        $reports=array();
+        $dmarc_reports = array();
+        $mtasts_reports = array();
         foreach($mailsIds as $mailId) {
             $num_emails++;
             $mail = $mailbox->getMail($mailId);
             $attachments = $mail->getAttachments();
             foreach ($attachments as $attachment) {
-                $reports = array_merge($reports, $this->open_archive($attachment->filePath));
+                $new_reports = $this->open_archive($attachment->filePath);
+                $dmarc_reports = array_merge($dmarc_reports,$new_reports['dmarc_reports']);
+                $mtasts_reports = array_merge($mtasts_reports,$new_reports['mtasts_reports']);
                 unlink($attachment->filePath);
             }
         }
-        return array('num_emails' => $num_emails, 'reports' => $reports);
+        return array('num_emails' => $num_emails, 'reports' => array('dmarc_reports' => $dmarc_reports, 'mtasts_reports' => $mtasts_reports));
     }
 
     private function open_archive($file): array
     {
-        $reports = array();
+        $dmarc_reports = array();
+        $mtasts_reports = array();
         $ziparchive = new \ZipArchive;
+        $filecontents = null;
 
         if ($ziparchive->open($file) === TRUE) {
             for($i=0; $i<$ziparchive->numFiles; $i++){
                 $stat = $ziparchive->statIndex($i);
-                $reports[] = new \SimpleXMLElement(file_get_contents("zip://$file#".$stat["name"]));
+                $filecontents = file_get_contents("zip://$file#".$stat["name"]);
             }
         } elseif($gzarchive = gzopen($file, 'r')) {
             $gzcontents=null;
             while (!feof($gzarchive)) {
                 $gzcontents .= gzread($gzarchive, filesize($file));
             }
-            $reports[] = new \SimpleXMLElement($gzcontents);
+            $filecontents = $gzcontents;
+        }
+
+        if(substr($filecontents, 0, 5) == "<?xml") {
+            //Expecting an DMARC XML Report
+            $dmarc_reports[] = new \SimpleXMLElement($filecontents);
         }
-        return $reports;
+        elseif($this->isJson($filecontents)) {
+            //Expecting an MTA-STS JSON Report
+            $mtasts_reports[] = json_decode($filecontents);
+        }
+
+        return array('dmarc_reports' => $dmarc_reports, 'mtasts_reports' => $mtasts_reports);
     }
+
+    private function isJson($string) {
+        json_decode($string);
+        return json_last_error() === JSON_ERROR_NONE;
+     }
 }