peterkahl/curl-master

v7.4.3 2020-10-02 06:16 UTC

README

Downloads License If this project has business value for you then don't hesitate to support me with a small donation.

Wrapper for the cURL extension to handle GET, POST, HEAD requests. Features include (optionally compressed) response caching, caching override, ETAG support, cookie storage, SSL certificate verification.

Ordinary request GET:

$curlm=new curlmaster;

$curlm->CacheDir='/srv/cache';
$curlm->ca_file='/srv/cert-ca/cacert.pem';
// Method GET is default.
$response = $curlm->Request('https://www.google.com/');

var_export($response);

...produces this response, which is not being cached.

/*
array (
  'library' => 'peterkahl\\curlmaster\\curlmaster',
  'library-version' => '7.1.2.2',
  'origin' => 'new',
  'timestamp' => 1601043487,
  'exectime' => '0.092391',
  'status' => '200',
  'forced' => false,
  'cachingtime' => 0,
  'cachecompress' => false,
  'filename' => '/srv/cache/curlmaster_response_f648fed2369cff9219f1c65e54701c.json',
  'cookiefile' => '/srv/cache/curlmaster_cookie_www.google.com.cookie',
  'request' =>
  array (
    'method' => 'GET',
    'url' => 'https://www.google.com/',
    'user-agent' => 'Mozilla/5.0 (curlmaster/7.1.2.2; +https://github.com/peterkahl/curlmaster)',
    'headers' =>
    array (
    ),
    'etag-enable' => false,
    'ca-file' => '/srv/cert-ca/cacert.pem',
    'cipher' => '',
    'post-data' => '',
  ),
  'verbose' =>
  array (
    0 => '*   Trying 2a00:1450:4001:81f::2004:443...',
    1 => '* TCP_NODELAY set',
    2 => '* Connected to www.google.com (2a00:1450:4001:81f::2004) port 443 (#0)',
    3 => '* ALPN, offering h2',
    4 => '* ALPN, offering http/1.1',
    5 => '* successfully set certificate verify locations:',
    6 => '*   CAfile: /srv/cert-ca/cacert.pem',
    7 => '  CApath: /etc/ssl/certs',
    8 => '* SSL connection using TLSv1.3 / TLS_AES_256_GCM_SHA384',
    9 => '* ALPN, server accepted to use h2',
    10 => '* Server certificate:',
    11 => '*  subject: C=US; ST=California; L=Mountain View; O=Google LLC; CN=www.google.com',
    12 => '*  start date: Sep  3 06:42:46 2020 GMT',
    13 => '*  expire date: Nov 26 06:42:46 2020 GMT',
    14 => '*  subjectAltName: host "www.google.com" matched cert\'s "www.google.com"',
    15 => '*  issuer: C=US; O=Google Trust Services; CN=GTS CA 1O1',
    16 => '*  SSL certificate verify ok.',
    17 => '* Using HTTP2, server supports multi-use',
    18 => '* Connection state changed (HTTP/2 confirmed)',
    19 => '* Copying HTTP/2 data in stream buffer to connection buffer after upgrade: len=0',
    20 => '* Using Stream ID: 1 (easy handle 0x55c5b37ebf80)',
    21 => '> GET / HTTP/2',
    22 => 'Host: www.google.com',
    23 => 'user-agent: Mozilla/5.0 (curlmaster/7.1.2.2; +https://github.com/peterkahl/curlmaster)',
    24 => 'accept: */*',
    25 => 'accept-encoding: deflate, gzip, br',
    26 => 'cookie: NID=204=mKHrdeEZ7MsIM_tAU0UbcXlfdzx6lsXvLES8lH54G27cTF9skpRfAOEWk9XqM6Ks-MQKZY-9aBQRMTCDSToCkP3K578C7cbdxIiTbq9ZRcqDp5nP5Uke2sZ5d8Lo9W_aw-soUOXQKM8qnG1F2by3MhACB29kOPGsDVmjJAt4oRw',
    27 => '',
    28 => '* old SSL session ID is stale, removing',
    29 => '* Connection state changed (MAX_CONCURRENT_STREAMS == 100)!',
    30 => '< HTTP/2 200 ',
    31 => '< date: Fri, 25 Sep 2020 14:18:07 GMT',
    32 => '< expires: -1',
    33 => '< cache-control: private, max-age=0',
    34 => '< content-type: text/html; charset=UTF-8',
    35 => '< content-encoding: gzip',
    36 => '< server: gws',
    37 => '< content-length: 15675',
    38 => '< x-xss-protection: 0',
    39 => '< x-frame-options: SAMEORIGIN',
    40 => '< alt-svc: h3-Q050=":443"; ma=2592000,h3-29=":443"; ma=2592000,h3-27=":443"; ma=2592000,h3-T051=":443"; ma=2592000,h3-T050=":443"; ma=2592000,h3-Q046=":443"; ma=2592000,h3-Q043=":443"; ma=2592000,quic=":443"; ma=2592000; v="46,43"',
    41 => '< ',
    42 => '* Closing connection 0',
    43 => '',
  ),
  'response' =>
  array (
    'timestamp' => 1601043487,
    'exectime' => '0.092378',
    'status' => '200',
    'cachingtime' => 0,
    'headers' =>
    array (
      'status' => 'HTTP/2 200',
      'date' => 'Fri, 25 Sep 2020 14:18:07 GMT',
      'expires' => '-1',
      'cache-control' => 'private, max-age=0',
      'content-type' => 'text/html; charset=UTF-8',
      'content-encoding' => 'gzip',
      'server' => 'gws',
      'content-length' => '15675',
      'x-xss-protection' => '0',
      'x-frame-options' => 'SAMEORIGIN',
      'alt-svc' => 'h3-Q050=":443"; ma=2592000,h3-29=":443"; ma=2592000,h3-27=":443"; ma=2592000,h3-T051=":443"; ma=2592000,h3-T050=":443"; ma=2592000,h3-Q046=":443"; ma=2592000,h3-Q043=":443"; ma=2592000,quic=":443"; ma=2592000; v="46,43"',
    ),
    'error-num' => 0,
    'error-verb' => 'CURLE_OK',
    'etag' => '',
    'last-modified' => 0,
    'body' => '<!doctype html>...[truncated]...</body></html>',
  ),
)
*/

Force caching of response:

$curlm=new curlmaster;

$curlm->CacheDir='/srv/cache';
$curlm->ca_file='/srv/cert-ca/cacert.pem';
$curlm->ForcedCacheMaxAge=3600;
$response = $curlm->Request('https://www.google.com/');

...produces this response, which is being cached as JSON.

{
    "library": "peterkahl\\curlmaster\\curlmaster",
    "library-version": "7.1.2.2",
    "origin": "new",
    "timestamp": 1601044020,
    "exectime": "0.120096",
    "status": "200",
    "forced": false,
    "cachingtime": 3600,
    "cachecompress": false,
    "filename": "\/srv\/cache\/curlmaster_response_f648fed2369cff9219f1c65e54701c.json",
    "cookiefile": "\/srv\/cache\/curlmaster_cookie_www.google.com.cookie",
    "request": {
        "method": "GET",
        "url": "https:\/\/www.google.com\/",
        "user-agent": "Mozilla\/5.0 (curlmaster\/7.1.2.2; +https:\/\/github.com\/peterkahl\/curlmaster)",
        "headers": [],
        "etag-enable": false,
        "ca-file": "\/srv\/cert-ca\/cacert.pem",
        "cipher": "",
        "post-data": ""
    },
    "verbose": [
        "*   Trying 2a00:1450:4001:81f::2004:443...",
        "* TCP_NODELAY set",
        "* Connected to www.google.com (2a00:1450:4001:81f::2004) port 443 (#0)",
        "* ALPN, offering h2",
        "* ALPN, offering http\/1.1",
        "* successfully set certificate verify locations:",
        "*   CAfile: \/srv\/cert-ca\/cacert.pem",
        "  CApath: \/etc\/ssl\/certs",
        "* SSL connection using TLSv1.3 \/ TLS_AES_256_GCM_SHA384",
        "* ALPN, server accepted to use h2",
        "* Server certificate:",
        "*  subject: C=US; ST=California; L=Mountain View; O=Google LLC; CN=www.google.com",
        "*  start date: Sep  3 06:42:46 2020 GMT",
        "*  expire date: Nov 26 06:42:46 2020 GMT",
        "*  subjectAltName: host \"www.google.com\" matched cert's \"www.google.com\"",
        "*  issuer: C=US; O=Google Trust Services; CN=GTS CA 1O1",
        "*  SSL certificate verify ok.",
        "* Using HTTP2, server supports multi-use",
        "* Connection state changed (HTTP\/2 confirmed)",
        "* Copying HTTP\/2 data in stream buffer to connection buffer after upgrade: len=0",
        "* Using Stream ID: 1 (easy handle 0x5635d76eaf80)",
        "> GET \/ HTTP\/2",
        "Host: www.google.com",
        "user-agent: Mozilla\/5.0 (curlmaster\/7.1.2.2; +https:\/\/github.com\/peterkahl\/curlmaster)",
        "accept: *\/*",
        "accept-encoding: deflate, gzip, br",
        "cookie: NID=204=mKHrdeEZ7MsIM_tAU0UbcXlfdzx6lsXvLES8lH54G27cTF9skpRfAOEWk9XqM6Ks-MQKZY-9aBQRMTCDSToCkP3K578C7cbdxIiTbq9ZRcqDp5nP5Uke2sZ5d8Lo9W_aw-soUOXQKM8qnG1F2by3MhACB29kOPGsDVmjJAt4oRw",
        "",
        "* old SSL session ID is stale, removing",
        "* Connection state changed (MAX_CONCURRENT_STREAMS == 100)!",
        "< HTTP\/2 200 ",
        "< date: Fri, 25 Sep 2020 14:27:00 GMT",
        "< expires: -1",
        "< cache-control: private, max-age=0",
        "< content-type: text\/html; charset=UTF-8",
        "< content-encoding: gzip",
        "< server: gws",
        "< content-length: 15675",
        "< x-xss-protection: 0",
        "< x-frame-options: SAMEORIGIN",
        "< alt-svc: h3-Q050=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000,h3-27=\":443\"; ma=2592000,h3-T051=\":443\"; ma=2592000,h3-T050=\":443\"; ma=2592000,h3-Q046=\":443\"; ma=2592000,h3-Q043=\":443\"; ma=2592000,quic=\":443\"; ma=2592000; v=\"46,43\"",
        "< ",
        "* Closing connection 0",
        ""
    ],
    "response": {
        "timestamp": 1601044020,
        "exectime": "0.120084",
        "status": "200",
        "cachingtime": 0,
        "headers": {
            "status": "HTTP\/2 200",
            "date": "Fri, 25 Sep 2020 14:27:00 GMT",
            "expires": "-1",
            "cache-control": "private, max-age=0",
            "content-type": "text\/html; charset=UTF-8",
            "content-encoding": "gzip",
            "server": "gws",
            "content-length": "15675",
            "x-xss-protection": "0",
            "x-frame-options": "SAMEORIGIN",
            "alt-svc": "h3-Q050=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000,h3-27=\":443\"; ma=2592000,h3-T051=\":443\"; ma=2592000,h3-T050=\":443\"; ma=2592000,h3-Q046=\":443\"; ma=2592000,h3-Q043=\":443\"; ma=2592000,quic=\":443\"; ma=2592000; v=\"46,43\""
        },
        "error-num": 0,
        "error-verb": "CURLE_OK",
        "etag": "",
        "last-modified": 0,
        "body": "<!doctype html>...[truncated]...<\/body><\/html>"
    }
}

Usage example, method POST:

use peterkahl\curlmaster\curlmaster;

$curlm = new curlmaster;
$curlm->CacheDir = '/srv/cache';
$curlm->ca_file = '/srv/cert-ca/cacert.pem';
$post_data = array(
  'user' => 'admin',
  'pwd'  => 'oracle',
);

$response = $curlm->Request('https://whatever.anything/login', 'POST', $post_data);

Usage example, method HEAD:

use peterkahl\curlmaster\curlmaster;

$curlm = new curlmaster;
$curlm->CacheDir = '/srv/cache';
$curlm->ca_file = '/srv/cert-ca/cacert.pem';
$curlm->headers = array('accept: text/html');
$response = $curlm->Request('https://github.com/', 'HEAD');

/*
The resulting response is an array. HEAD responses are not cached.

array (
  'library' => 'peterkahl\\curlmaster\\curlmaster',
  'library-version' => '7.1.1',
  'origin' => 'new',
  'timestamp' => 1600930605,
  'exectime' => '0.330269',
  'status' => '200',
  'forced' => false,
  'cachingtime' => -1,
  'filename' => '/srv/cache/curlmaster_response_ba5453820c9f8f87fec8b54e7540bf.json',
  'cookiefile' => '/srv/cache/curlmaster_cookie_github.com.cookie',
  'request' =>
  array (
    'method' => 'HEAD',
    'url' => 'https://github.com/',
    'user-agent' => 'Mozilla/5.0 (curlmaster/7.1.1; +https://github.com/peterkahl/curlmaster)',
    'headers' =>
    array (
      0 => 'accept: text/html',
    ),
    'etag-enable' => false,
    'ca-file' => '/srv/cert-ca/cacert.pem',
    'cipher' => '',
    'post-data' => '',
  ),
  'verbose' =>
  array (
    0 => '*   Trying 140.82.121.4:443...',
    1 => '* TCP_NODELAY set',
    2 => '* Connected to github.com (140.82.121.4) port 443 (#0)',
    3 => '* ALPN, offering h2',
    4 => '* ALPN, offering http/1.1',
    5 => '* successfully set certificate verify locations:',
    6 => '*   CAfile: /srv/cert-ca/cacert.pem',
    7 => '  CApath: /etc/ssl/certs',
    8 => '* SSL connection using TLSv1.3 / TLS_AES_128_GCM_SHA256',
    9 => '* ALPN, server accepted to use http/1.1',
    10 => '* Server certificate:',
    11 => '*  subject: C=US; ST=California; L=San Francisco; O=GitHub, Inc.; CN=github.com',
    12 => '*  start date: May  5 00:00:00 2020 GMT',
    13 => '*  expire date: May 10 12:00:00 2022 GMT',
    14 => '*  subjectAltName: host "github.com" matched cert\'s "github.com"',
    15 => '*  issuer: C=US; O=DigiCert Inc; OU=www.digicert.com; CN=DigiCert SHA2 High Assurance Server CA',
    16 => '*  SSL certificate verify ok.',
    17 => '> HEAD / HTTP/1.1',
    18 => 'Host: github.com',
    19 => 'User-Agent: Mozilla/5.0 (curlmaster/7.1.1; +https://github.com/peterkahl/curlmaster)',
    20 => 'Cookie: logged_in=no; _octo=GH1.1.586556027.1600930239',
    21 => 'accept: text/html',
    22 => '',
    23 => '* old SSL session ID is stale, removing',
    24 => '* Mark bundle as not supporting multiuse',
    25 => '< HTTP/1.1 200 OK',
    26 => '< date: Thu, 24 Sep 2020 06:56:45 GMT',
    27 => '< content-type: text/html; charset=utf-8',
    28 => '< server: GitHub.com',
    29 => '< status: 200 OK',
    30 => '< vary: X-PJAX, Accept-Encoding, Accept, X-Requested-With, Accept-Encoding',
    31 => '< etag: W/"4da62c2bc712cef0599ed6a7f550f4d2"',
    32 => '< cache-control: max-age=0, private, must-revalidate',
    33 => '< strict-transport-security: max-age=31536000; includeSubdomains; preload',
    34 => '< x-frame-options: deny',
    35 => '< x-content-type-options: nosniff',
    36 => '< x-xss-protection: 1; mode=block',
    37 => '< referrer-policy: origin-when-cross-origin, strict-origin-when-cross-origin',
    38 => '< expect-ct: max-age=2592000, report-uri="https://api.github.com/_private/browser/errors"',
    39 => '< content-security-policy: default-src \'none\'; base-uri \'self\'; block-all-mixed-content; connect-src \'self\' uploads.github.com www.githubstatus.com collector.githubapp.com api.github.com www.google-analytics.com github-cloud.s3.amazonaws.com github-production-repository-file-5c1aeb.s3.amazonaws.com github-production-upload-manifest-file-7fdce7.s3.amazonaws.com github-production-user-asset-6210df.s3.amazonaws.com cdn.optimizely.com logx.optimizely.com/v1/events wss://alive.github.com; font-src github.githubassets.com; form-action \'self\' github.com gist.github.com; frame-ancestors \'none\'; frame-src render.githubusercontent.com; img-src \'self\' data: github.githubassets.com identicons.github.com collector.githubapp.com github-cloud.s3.amazonaws.com *.githubusercontent.com customer-stories-feed.github.com spotlights-feed.github.com; manifest-src \'self\'; media-src github.githubassets.com; script-src github.githubassets.com; style-src \'unsafe-inline\' github.githubassets.com; worker-src github.com/socket-worker.js gist.github.com/socket-worker.js',
    40 => '* Added cookie _gh_sess="YT1whftzqTDiFdxWK6B8KsA6%2FF%2Fo4DhCEPXJ8%2FA%2FB97tPOmGTQ9aOEwwrlntWc8THlcYj91gzSznAgR%2BGMHR%2BIRfOW7ErgXFbJZpNrzwn5KjLpdPQHoz3bkLvZiKQ%2B7xiXBt4IcsO7wdJN4zrHcehaZgTTELIEVrIkLOoV3ZktjVMBhJf3SkBF3of%2B3zPbM2UjAfEDrVK%2FaKM3YXgwDbsKoAdWsrqgOO4Gv%2FprqpT2VqSgPSwCNnF3kzz4ivaTf%2B8IHIS7lAHlr5N82STNzLUg%3D%3D--lyzCNBt0J30oi58f--CFgqDV2Di71wa7gw%2BGmk3A%3D%3D" for domain github.com, path /, expire 0',
    41 => '< Set-Cookie: _gh_sess=YT1whftzqTDiFdxWK6B8KsA6%2FF%2Fo4DhCEPXJ8%2FA%2FB97tPOmGTQ9aOEwwrlntWc8THlcYj91gzSznAgR%2BGMHR%2BIRfOW7ErgXFbJZpNrzwn5KjLpdPQHoz3bkLvZiKQ%2B7xiXBt4IcsO7wdJN4zrHcehaZgTTELIEVrIkLOoV3ZktjVMBhJf3SkBF3of%2B3zPbM2UjAfEDrVK%2FaKM3YXgwDbsKoAdWsrqgOO4Gv%2FprqpT2VqSgPSwCNnF3kzz4ivaTf%2B8IHIS7lAHlr5N82STNzLUg%3D%3D--lyzCNBt0J30oi58f--CFgqDV2Di71wa7gw%2BGmk3A%3D%3D; Path=/; HttpOnly; Secure; SameSite=Lax',
    42 => '< Accept-Ranges: bytes',
    43 => '< X-GitHub-Request-Id: AF42:E4AE:14B52F0:1CD1320:5F6C432C',
    44 => '< ',
    45 => '* Closing connection 0',
    46 => '',
  ),
  'response' =>
  array (
    'timestamp' => 1600930605,
    'exectime' => '0.330255',
    'status' => '200',
    'cachingtime' => 0,
    'headers' =>
    array (
      'status' => 'HTTP/1.1 200 OK',
      'date' => 'Thu, 24 Sep 2020 06:56:45 GMT',
      'content-type' => 'text/html; charset=utf-8',
      'server' => 'GitHub.com',
      'status-1' => '200 OK',
      'vary' => 'X-PJAX, Accept-Encoding, Accept, X-Requested-With, Accept-Encoding',
      'etag' => 'W/"4da62c2bc712cef0599ed6a7f550f4d2"',
      'cache-control' => 'max-age=0, private, must-revalidate',
      'strict-transport-security' => 'max-age=31536000; includeSubdomains; preload',
      'x-frame-options' => 'deny',
      'x-content-type-options' => 'nosniff',
      'x-xss-protection' => '1; mode=block',
      'referrer-policy' => 'origin-when-cross-origin, strict-origin-when-cross-origin',
      'expect-ct' => 'max-age=2592000, report-uri="https://api.github.com/_private/browser/errors"',
      'content-security-policy' => 'default-src \'none\'; base-uri \'self\'; block-all-mixed-content; connect-src \'self\' uploads.github.com www.githubstatus.com collector.githubapp.com api.github.com www.google-analytics.com github-cloud.s3.amazonaws.com github-production-repository-file-5c1aeb.s3.amazonaws.com github-production-upload-manifest-file-7fdce7.s3.amazonaws.com github-production-user-asset-6210df.s3.amazonaws.com cdn.optimizely.com logx.optimizely.com/v1/events wss://alive.github.com; font-src github.githubassets.com; form-action \'self\' github.com gist.github.com; frame-ancestors \'none\'; frame-src render.githubusercontent.com; img-src \'self\' data: github.githubassets.com identicons.github.com collector.githubapp.com github-cloud.s3.amazonaws.com *.githubusercontent.com customer-stories-feed.github.com spotlights-feed.github.com; manifest-src \'self\'; media-src github.githubassets.com; script-src github.githubassets.com; style-src \'unsafe-inline\' github.githubassets.com; worker-src github.com/socket-worker.js gist.github.com/socket-worker.js',
      'set-cookie' => '_gh_sess=YT1whftzqTDiFdxWK6B8KsA6%2FF%2Fo4DhCEPXJ8%2FA%2FB97tPOmGTQ9aOEwwrlntWc8THlcYj91gzSznAgR%2BGMHR%2BIRfOW7ErgXFbJZpNrzwn5KjLpdPQHoz3bkLvZiKQ%2B7xiXBt4IcsO7wdJN4zrHcehaZgTTELIEVrIkLOoV3ZktjVMBhJf3SkBF3of%2B3zPbM2UjAfEDrVK%2FaKM3YXgwDbsKoAdWsrqgOO4Gv%2FprqpT2VqSgPSwCNnF3kzz4ivaTf%2B8IHIS7lAHlr5N82STNzLUg%3D%3D--lyzCNBt0J30oi58f--CFgqDV2Di71wa7gw%2BGmk3A%3D%3D; Path=/; HttpOnly; Secure; SameSite=Lax',
      'accept-ranges' => 'bytes',
      'x-github-request-id' => 'AF42:E4AE:14B52F0:1CD1320:5F6C432C',
    ),
    'error-num' => 0,
    'error-verb' => 'CURLE_OK',
    'etag' => 'W/"4da62c2bc712cef0599ed6a7f550f4d2"',
    'last-modified' => 0,
    'body' => '

',
  ),
)
*/

Cache Purging:

Best to set this up on a crontab job.

use peterkahl\curlmaster\curlmaster;

$curlm = new curlmaster;
$curlm->CacheDir = '/srv/cache';
$curlm->PurgeCache();