simpod/kafka

PHP Kafka boilerplate wrapper around RdKafka

0.2.3 2023-12-01 11:57 UTC

Requires

• php: ^8.2
• ext-pcntl: *
• ext-rdkafka: ^6
• psr/log: ^3
• thecodingmachine/safe: ^2.2

Requires (Dev)

• doctrine/coding-standard: ^12.0
• infection/infection: ^0.27.0
• kwn/php-rdkafka-stubs: ^2.0
• phpstan/extension-installer: ^1.1
• phpstan/phpstan: 1.10.46
• phpstan/phpstan-phpunit: 1.3.15
• phpstan/phpstan-strict-rules: 1.5.2
• phpunit/phpunit: ^10.3
• psalm/plugin-phpunit: ^0.18.0
• roave/infection-static-analysis-plugin: ^1.7
• thecodingmachine/phpstan-safe-rule: ^1.0
• vimeo/psalm: ^5.0

Suggests

• kwn/php-rdkafka-stubs: Support and autocompletion for RDKafka in IDE | require as dev dependency

MIT

• Simon Podlipsky

README

Installation

Add as Composer dependency:

composer require simpod/kafka

Config Constants

Some config constants are provided like ConsumerConfig, ProducerConfig or CommonClientConfigs.

However, they are copied from Java API and not all are applicable to librdkafka. Consult with librdkafka documentation before use.

Clients

Consumer

KafkaConsumer boilerplate is available with startBatch() method (to suplement this example in librdkafka) and with start(). They also handle termination signals for you.

Classic Consumer

<?php

declare(strict_types=1);

namespace Your\AppNamespace;

use RdKafka\Message;
use SimPod\Kafka\Clients\Consumer\ConsumerConfig;
use SimPod\Kafka\Clients\Consumer\KafkaConsumer;

final class ExampleConsumer
{
    public function run(): void
    {
        $kafkaConsumer = new KafkaConsumer($this->getConfig(), Logger::get());

        $kafkaConsumer->subscribe(['topic1']);

        $kafkaConsumer->start(
            120 * 1000,
            static function (Message $message) use ($kafkaConsumer) : void {
                // Process message here

                $kafkaConsumer->commit($message); // Autocommit is disabled
            }
        );
    }

    private function getConfig(): ConsumerConfig
    {
        $config = new ConsumerConfig();

        $config->set(ConsumerConfig::BOOTSTRAP_SERVERS_CONFIG, '127.0.0.1:9092');
        $config->set(ConsumerConfig::ENABLE_AUTO_COMMIT_CONFIG, false);
        $config->set(ConsumerConfig::CLIENT_ID_CONFIG, gethostname());
        $config->set(ConsumerConfig::AUTO_OFFSET_RESET_CONFIG, 'earliest');
        $config->set(ConsumerConfig::GROUP_ID_CONFIG, 'consumer_group_name');

        return $config;
    }
}

Batching Consumer

<?php

declare(strict_types=1);

namespace Your\AppNamespace;

use RdKafka\Message;
use SimPod\Kafka\Clients\Consumer\ConsumerConfig;
use SimPod\Kafka\Clients\Consumer\ConsumerRecords;
use SimPod\Kafka\Clients\Consumer\KafkaConsumer;

final class ExampleBatchConsumer
{
    public function run(): void
    {
        $kafkaConsumer = new KafkaConsumer($this->getConfig());

        $kafkaConsumer->subscribe(['topic1']);

        $kafkaConsumer->startBatch(
            200000, 
            120 * 1000,
            static function (Message $message): void {
                // Process record
            },
            static function (ConsumerRecords $consumerRecords) use ($kafkaConsumer) : void {
                // Process records batch
    
                $kafkaConsumer->commit($consumerRecords->getLast());
            }
        );
    }

    private function getConfig(): ConsumerConfig
    {
        $config = new ConsumerConfig();

        $config->set(ConsumerConfig::BOOTSTRAP_SERVERS_CONFIG, '127.0.0.1:9092');
        $config->set(ConsumerConfig::ENABLE_AUTO_COMMIT_CONFIG, false);
        $config->set(ConsumerConfig::CLIENT_ID_CONFIG, gethostname());
        $config->set(ConsumerConfig::AUTO_OFFSET_RESET_CONFIG, 'earliest');
        $config->set(ConsumerConfig::GROUP_ID_CONFIG, 'consumer_group_name');

        return $config;
    }
}