coderovich/clickhouse

ClickHouse for Yii2

Maintainers

Details

github.com/coderovich/clickhouse

Source

Issues

Installs: 7

Dependents: 0

Suggesters: 0

Security: 0

Stars: 0

Watchers: 0

Forks: 43

Type:yii2-extension

dev-master 2024-03-07 15:55 UTC

Requires

Requires (Dev)

MIT 4827e750d9cd2b8888efcef56bb4771f72b891af

  • Tutik Alexsandr <sanchezzzhak.woop@ya.ru>

yii2clickhouseClickHouse ActiveRecord Yii2

This package is auto-updated.

Last update: 2024-12-07 17:37:28 UTC

README

Installation

Composer

The preferred way to install this extension is through Composer.

Either run

  • stable php composer.phar require kak/clickhouse ~1.1
  • dev php composer.phar require kak/clickhouse @dev

or add to composer.json manual

  • stable "kak/clickhouse": "~1.1
  • dev "kak/clickhouse": "@dev"

to the require section of your composer.json

Configuration example

   'components' => [
        'clickhouse' => [
            'class' => 'coderovich\clickhouse\Connection',
            'dsn' => '127.0.0.1',
            'port' => '8123',
           // 'database' => 'default',  // use other database name
            'username' => 'web',
            'password' => '123',
            'enableSchemaCache' => true,
            'schemaCache' => 'cache',
            'schemaCacheDuration' => 86400
        ],
   // ...

Notes

  • If clickhouse server responds with no response == 200, then you will get the exception

Usage

   /** @var \coderovich\clickhouse\Connection $client */
    $client = \Yii::$app->clickhouse;
    $sql = 'select * from stat where counter_id=:counter_id';
    $client->createCommand($sql, [
        ':counter_id' => 122
    ])->queryAll();
    
    // ====== insert data ORM ======
    
    $client->createCommand(null)
    ->insert('stat', [
        'event_data' => date('Y-m-d'),
        'counter_id' => 122
    ])
    ->execute();

batch insert files

    /** @var \coderovich\clickhouse\Connection $clickhouse */
    $clickhouse = \Yii::$app->clickhouse;

    $files = [
        'dump_20170502' => Yii::getAlias('@app/dump_20170502.csv'),
        'dump_20170503' => Yii::getAlias('@app/dump_20170503.csv'),
        'dump_20170504' => Yii::getAlias('@app/dump_20170504.csv'),
    ];	
    		
    $responses = $clickhouse->createCommand(null)
    ->batchInsertFiles('stat', null, [
        $files
    ], 'CSV');

    foreach ($responses as $keyId => $response) {
        var_dump($keyId . ' ' . $response->isOk);
    }

batch insert files, batch size = 100 lines

    /** @var \coderovich\clickhouse\Connection $clickhouse */
    $clickhouse = \Yii::$app->clickhouse;

    $responses = $clickhouse->createCommand(null)
    ->batchInsertFilesDataSize('stat', null, [
        $files
    ], 'CSV', 100);	
     foreach ($responses as $keyId => $parts) {
        foreach ($parts as $partId => $response) {
            var_dump($keyId . '_' . $partId. ' ' . $response->isOk);
        }
     }

old methods: meta, rows, countAll, statistics

   	
    $sql = 'SELECT 
        user_id, sum(income) AS sum_income
        FROM stat
        GROUP BY event_date
        WITH TOTALS
        LIMIT 10
    '; 	

    /** @var \coderovich\clickhouse\Connection $clickhouse */
    $clickhouse = \Yii::$app->clickhouse;
    
    $command = $clickhouse->createCommand($sql);  	
    $result = $command->queryAll();
    
    var_dump($command->getMeta());  	      // columns meta info (columnName, dataType)
    var_dump($command->getTotals());          // get totals rows to read
    var_dump($command->getData());  	      // get rows data
    var_dump($command->getRows());  	      // rows count current result
    var_dump($command->getCountAll());        // rows count before limit at least	
    var_dump($command->getExtremes());  	
    var_dump($command->getStatistics());      // stat query 
    
 //or
     
    $command = $clickhouse->createCommand($sql);  
    $result = $command->queryAll($command::FETCH_MODE_ALL);
    var_dump($result);

old examples ORM

use coderovich\clickhouse\Query;

$q = (new Query())
    ->from('stat')
    ->withTotals()
    ->where(['event_date' => '2017-05-01' , 'user_id' => 5])
    ->offset(2)
    ->limit(1);

$command = $q->createCommand();
$result  = $command->queryAll();
$total   = $command->getTotals();

var_dump($result);
var_dump($total); 

// -----
$command = (new Query())
    ->select(['event_stat', 'count()'])
    ->from('test_stat')
    ->groupBy('event_date')
    ->limit(1)
    ->withTotals();
    
$result =  $command->all();
var_dump($command->getTotals());

Group With Modifiers

use coderovich\clickhouse\Query;

$command = (new Query());
// ...
$command->withTotals();
// or
$command->withCube();
// or
$command->withRollup();

Set specific options

  /** @var \coderovich\clickhouse\Connection $client */
    $client = \Yii::$app->clickhouse;
    $sql = 'select * from stat where counter_id=:counter_id';
    $client->createCommand($sql, [
        ':counter_id' => 122
    ])->setOptions([
        'max_threads' => 2
    ])->queryAll();

// add options use method
// ->addOptions([])

Select with

    use coderovich\clickhouse\Query;
    // ...

    $db = \Yii::$app->clickhouse;
    $query = new Query();
    // first argument scalar var or Query object
    $query->withQuery($db->quoteValue('2021-10-05'), 'date1');
    $query->select('*');
    $query->from('stat');
    $query->where('event_stat < date1');
    $query->all();
/*
    WITH '2020-07-26' AS date1 SELECT * FROM stat WHERE event_stat < date1
*/

Save custom model

use yii\base\Model;

class Stat extends Model
{
    public $event_date; // Date;
    public $counter_id  = 0; // Int32,

    public function save($validate = true)
    {
        /** @var \coderovich\clickhouse\Connection $client */
        $client = \Yii::$app->clickhouse;
        $this->event_date = date('Y-m-d');

        if ($validate && !$this->validate()) {
            return false;
        }

        $attributes = $this->getAttributes();
        $client->createCommand(null)
            ->insert('stat', $attributes)
            ->execute();

        return true;	
    }
}

ActiveRecord model

use coderovich\clickhouse\ActiveRecord;
use app\models\User;

class Stat extends ActiveRecord
{
    // pls overwrite method is config section !=clickhouse
    // default clickhouse
	public static function getDb()
	{
	    return \Yii::$app->clickhouse;
	}


    public static function tableName()
    {
        return 'stat';
    }
    
    // use relation in mysql (Only with, do not use joinWith)
    public function getUser()
    {
    	return $this->hasOne(User::class, ['id' => 'user_id']);
    }
}

Using Gii generator

<?php
return [
    //....
    'modules' => [
        // ...
        'gii' => [
            'class' => 'yii\gii\Module',
            'allowedIPs' => [
                '127.0.0.1',
                '::1',
                '192.168.*',
                '10.*',
            ],
            'generators' => [
                'clickhouseDbModel' => [
                    'class' => 'coderovich\clickhouse\gii\model\Generator'
                ]
            ],
        ],
    ]
];

Using Debug panel

$config['bootstrap'][] = 'debug';
    $config['modules']['debug'] = [
        'class' => 'yii\debug\Module',
        'allowedIPs' => [
            '127.0.0.1',
            '::1',
            '192.168.*',
            '10.*',
        ],
        'panels' => [
            'clickhouse' => [
                'class' => 'coderovich\clickhouse\debug\Panel',
                'db' => 'clickhouse'
            ],
        ]

    ];

Using SqlDataProvider

$sql = 'select * from stat where counter_id=:counter_id and event_date=:date';
$provider = new \coderovich\clickhouse\data\SqlDataProvider([
    'db' => 'clickhouse',
    'sql' => $sql,
    'params' => [
        ':counter_id' => 1,
        ':date' => date('Y-m-d')
    ]
]);

Using Migration Data

convert schema mysql >>> clickhouse
create custom console controller

    // ...
    public function actionIndex()
    {
        $exportSchemaCommand = new \coderovich\clickhouse\console\MigrationSchemaCommand([
            'sourceTable' => 'stat',
            'sourceDb' => \Yii::$app->db,
            'excludeSourceColumns' => [
                'id',
            ]
            'columns' => [
                '`event_date` Date' 
            ]
        ]);
        // result string SQL schema  
        $sql = $exportSchemaCommand->run();
        echo $sql;
    }

migration mysql,mssql data >>> clickhouse
create custom console controller

  // ...
    public function actionIndex()
    {
        $exportDataCommand = new \coderovich\clickhouse\console\MigrationDataCommand([
            'sourceQuery' => (new Query())->select('*')->from('stat'),
            'sourceDb' => \Yii::$app->db,
            'storeTable' => 'test_stat',
            'storeDb' => \Yii::$app->clickhouse,
            'batchSize' => 10000,
            'filterSourceRow' => function($data){
                // if result false then skip save row
                $time = strtotime($data['hour_at']);
                return $time > 0;
            },
            'mapData' => [
                // key storeTable column => sourceTable column|call function 
                'event_date' => function($data){
                    return date('Y-m-d',strtotime($data['hour_at']));
                },
                'time' => function($data){
                    return strtotime($data['hour_at']);
                },
                'user_id' => 'partner_id'
            ]    
        ]);
        $exportDataCommand->run();  
     
    }

Result

php yii export-test/index

total count rows source table 38585
part data files count 4
save files dir: /home/user/test-project/www/runtime/clickhouse/stat
parts:
 >>> part0.data time 4.749
 >>> part1.data time 4.734
 >>> part2.data time 4.771
 >>> part3.data time 4.089
insert files
 <<< part0.data  time 3.289
 <<< part1.data  time 2.024
 <<< part2.data  time 1.938
 <<< part3.data  time 3.359
done

ClickHouse Reference Manual

https://clickhouse.yandex/reference_en.html

Summary of recommendations insert data

  • 1 Accumulated data and insert at one time, it will reduce the operations io disk
  • 2 @todo how that will add...

Run tests

  • 1 git clone repository https://github.com/sanchezzzhak/kak-clickhouse.git
  • 2 composer install --ignore-platform-reqs
  • 3 create the config clickhouse touch tests/_config/clickhouse.php if you non-standard access to the server connection
<?php

return [
    'class' => 'coderovich\clickhouse\Connection',
    'dsn' => '127.0.0.1',
    'port' => '8123',
    'username' => 'web',
    'password' => '123',
    'enableSchemaCache' => true,
    'schemaCache' => 'cache',
    'schemaCacheDuration' => 86400
];
  • 4 run tests php vendor/bin/codecept run