$json = preg_replace('/[[:cntrl:]]/', '', $json);
# 把控制字符干掉就好了
# 实践证明,会把换行符也干掉。。。
分类目录归档:PHP
php上传目录到阿里云oss,python上传目录到阿里云oss
首先是python版
# -*- coding: utf-8 -*-
import os
import oss2
accessKeyId = "xxx"
accessKeySecret = "xxx"
endpoint = "oss-cn-shanghai-internal.aliyuncs.com"
# endpoint = "oss-cn-shanghai.aliyuncs.com"
bucket = "xxx"
localDirectory = "/data/mysql"
prefix = "db_bak/mysql"
auth = oss2.Auth(accessKeyId, accessKeySecret)
bucket = oss2.Bucket(auth, endpoint, bucket)
# 当文件长度大于或等于可选参数multipart_threshold(默认值为10MB)时,会使用分片上传。如未使用参数store指定目录,则会在HOME目录下建立.py-oss-upload目录来保存断点信息。
# oss2.resumable_upload(bucket, '<yourObjectName>', '<yourLocalFile>')
# 遍历目录进行上传
for fpathe, dirs, fs in os.walk(localDirectory):
for f in fs:
localFileName = os.path.join(fpathe, f)
newObjectName = localFileName[len(localDirectory) + 1:]
newObjectName = prefix + '/' + newObjectName
# 跳过已存在的文件
if bucket.object_exists(newObjectName):
print('pass ' + newObjectName)
continue
# SDK自己分片大文件似乎有bug,用php版吧
oss2.resumable_upload(bucket, newObjectName, localFileName)
print(newObjectName)
print(localFileName)
然后是php版(上传300G的大文件没问题)
<?php
use OSS\Core\OssException;
use OSS\Core\OssUtil;
use OSS\OssClient;
require 'vendor/autoload.php';
$accessKeyId = "xxx";
$accessKeySecret = "xxx";
$endpoint = "oss-cn-shanghai-internal.aliyuncs.com";
//$endpoint = "oss-cn-shanghai.aliyuncs.com";
$bucket = "xxx";
$localDirectory = "/data/mysql";
$prefix = "db/data";
function iScanDir($path = '.')
{
$dirs = scandir($path);
foreach ($dirs as $filename) {
if ($filename == '.' || $filename == '..') {
continue;
}
$fullFileName = "{$path}/{$filename}";
if (is_dir($fullFileName)) {
foreach (iScanDir($fullFileName) as $subFileName) {
yield $subFileName;
}
} else {
yield "{$fullFileName}";
}
}
}
try {
$ossClient = new OssClient($accessKeyId, $accessKeySecret, $endpoint);
} catch (OssException $e) {
exit($e->getMessage() . "\n");
}
$ossClient->setMaxTries(16);
foreach (iScanDir($localDirectory) as $fileName) {
$object = substr($fileName, strlen($localDirectory));
$object = trim($object, '.');
$object = trim($object, '/');
$object = "{$prefix}/{$object}";
$uploadFile = $fileName;
echo "{$fileName} ";
$exist = $ossClient->doesObjectExist($bucket, $object);
if ($exist) {
print(" : exist\n");
continue;
}
try {
$uploadId = $ossClient->initiateMultipartUpload($bucket, $object);
} catch (OssException $e) {
echo "initiateMultipartUpload error: " . $e->getMessage() . "\n";
}
$partSize = 10 * 1024 * 1024;
$uploadFileSize = filesize($uploadFile);
while (true) {
if ($partSize * 9999 < $uploadFileSize) {
$partSize = $partSize + 10 * 1024 * 1024;
} else {
break;
}
}
$pieces = $ossClient->generateMultiuploadParts($uploadFileSize, $partSize);
$responseUploadPart = [];
$uploadPosition = 0;
$isCheckMd5 = true;
foreach ($pieces as $i => $piece) {
$fromPos = $uploadPosition + (integer)$piece[$ossClient::OSS_SEEK_TO];
$toPos = (integer)$piece[$ossClient::OSS_LENGTH] + $fromPos - 1;
$upOptions = [
$ossClient::OSS_FILE_UPLOAD => $uploadFile,
$ossClient::OSS_PART_NUM => ($i + 1),
$ossClient::OSS_SEEK_TO => $fromPos,
$ossClient::OSS_LENGTH => $toPos - $fromPos + 1,
$ossClient::OSS_CHECK_MD5 => $isCheckMd5,
];
// MD5校验。
if ($isCheckMd5) {
$contentMd5 = OssUtil::getMd5SumForFile($uploadFile, $fromPos, $toPos);
$upOptions[$ossClient::OSS_CONTENT_MD5] = $contentMd5;
}
try {
// 上传分片。
$responseUploadPart[] = $ossClient->uploadPart($bucket, $object, $uploadId, $upOptions);
} catch (OssException $e) {
printf(__FUNCTION__ . ": initiateMultipartUpload, uploadPart - part#{$i} FAILED\n");
printf($e->getMessage() . "\n");
return;
}
printf(__FUNCTION__ . ": initiateMultipartUpload, uploadPart - part#{$i} OK\n");
}
// $uploadParts是由每个分片的ETag和分片号(PartNumber)组成的数组。
$uploadParts = [];
foreach ($responseUploadPart as $i => $eTag) {
$uploadParts[] = [
'PartNumber' => ($i + 1),
'ETag' => $eTag,
];
}
/**
* 步骤3:完成上传。
*/
try {
// 在执行该操作时,需要提供所有有效的$uploadParts。OSS收到提交的$uploadParts后,会逐一验证每个分片的有效性。当所有的数据分片验证通过后,OSS将把这些分片组合成一个完整的文件。
$ossClient->completeMultipartUpload($bucket, $object, $uploadId, $uploadParts);
} catch (OssException $e) {
printf(__FUNCTION__ . ": completeMultipartUpload FAILED\n");
printf($e->getMessage() . "\n");
return;
}
printf(__FUNCTION__ . ": completeMultipartUpload OK\n");
}
简单实现MySQL负载低时才继续运行任务,负载高的时候睡眠
public static function runSleep($sec = 10, $echo = null) {
while (true) {
$result = NoThingModel::getInstance()->db->select("show status like 'Threads_running'", [], false);
$result = reset($result);
if ($result) {
if ($result->Variable_name == 'Threads_running') {
if ($result->Value > 10) {
if ($echo) {
echo "sleep 10\n";
}
sleep($sec);
} else {
break;
}
} else {
break;
}
} else {
break;
}
}
}
GuzzleHttp 400 500 时获取内容的方法
http_errors
- Summary
- Set to
falseto disable throwing exceptions on an HTTP protocol errors (i.e., 4xx and 5xx responses). Exceptions are thrown by default when HTTP protocol errors are encountered. - Types
- bool
- Default
true- Constant
GuzzleHttp\RequestOptions::HTTP_ERRORS
$client->request('GET', '/status/500');
// Throws a GuzzleHttp\Exception\ServerException
$res = $client->request('GET', '/status/500', ['http_errors' => false]);
echo $res->getStatusCode();
// 500
phpstorm 代码提示不区分大小写,自动导入命名空间
默认效果:

zend studio 效果
手动效果:
phpstorm配置自动导入方法:
phpstorm代码提示不区分大小写配置



beanstalkd 安装和配置
安装
安装以centos为例
yum install beanstalkd
配置
使用centos yum安装,通过查看服务脚本发现有这个配置文件
cat /etc/sysconfig/beanstalkd
主要修改几个地方
ADDR=-l 10.1.3.16 # 修改为内网或本地地址,否则会开放外网非常危险
PORT=-p 11111 # 修改下端口
BINLOG_DIR=-b /var/lib/beanstalkd/binlog # 启用二进制日志防止意外断电或重启导致数据丢失
客户端连接问题:
Connection error 113: No route to host
解决办法:防火墙开放对应端口
性能:
$start = microtime(true);
for ($i = 0; $i < 100000; $i ++) {
$beansClient->useTube('myAwesomeTube')->put("哈哈哈job's payload呵呵呵呵");
}
echo microtime(true) - $start, "\n";
插入一万条数据一秒(本地连接)
插入十万条数据十秒(本地连接)
插入一万条数据14秒(内网连接)
本地取出1w条数据要1.5秒
内网取出1w条数据要33秒
本地取出 1173359条数据 要 200秒
结论:吃内存,快还是慢你觉得呢?
php-fpm优化参数合集
RabbitMQ 简单队列
我想要的是定时任务延时队列,似乎不满足需求,现在用redis做的队列又可以满足大部分需求,先放着吧
# 推送到队列
public function aa032() {
$connection = new AMQPStreamConnection('localhost', 5672, 'guest', 'guest');
$channel = $connection->channel();
$channel->queue_declare('hello', false, false, false, false);
$msg = new AMQPMessage('Hello World 233!');
$channel->basic_publish($msg, '', 'hello');
echo " [x] Sent 'Hello World 233!'\n";
$channel->close();
$connection->close();
}
# 守护处理队列
public function aa033() {
$connection = new AMQPStreamConnection('localhost', 5672, 'guest', 'guest');
$channel = $connection->channel();
$channel->queue_declare('hello', false, false, false, false);
echo " [*] Waiting for messages. To exit press CTRL+C\n";
$callback = function ($msg) {
echo ' [x] Received ', $msg->body, "\n";
};
$channel->basic_consume('hello', '', false, true, false, false, $callback);
while (count($channel->callbacks)) {
$channel->wait();
}
}
cURL error 60: SSL certificate problem: unable to get local issuer certificate (see http://curl.haxx.se/libcurl/c/libcurl-errors.html)
CURL证书错误问题正确解决方法
方法1:自己自己下载一个根证书(不推荐)
下载地址:https://github.com/bagder/ca-bundle
// curl_setopt ($ch, CURLOPT_SSL_VERIFYHOST, 0);
// curl_setopt ($ch, CURLOPT_SSL_VERIFYPEER, 0);
# curl_setopt($ch, CURLOPT_CAPATH, __DIR__);
curl_setopt($ch, CURLOPT_CAINFO, __DIR__ . '/ca-bundle.crt');
方法2:自动找到系统里的证书路径(推荐)
# 安装
composer require composer/ca-bundle
# 通过下面方法即可返回证书路径
\Composer\CaBundle\CaBundle::getSystemCaRootBundlePath()
// curl_setopt ($ch, CURLOPT_SSL_VERIFYHOST, 0);
// curl_setopt ($ch, CURLOPT_SSL_VERIFYPEER, 0);
# curl_setopt($ch, CURLOPT_CAPATH, __DIR__);
curl_setopt($ch, CURLOPT_CAINFO, \Composer\CaBundle\CaBundle::getSystemCaRootBundlePath());
GuzzleHttp https证书问题解决
$client = new \GuzzleHttp\Client([
\GuzzleHttp\RequestOptions::VERIFY => \Composer\CaBundle\CaBundle::getSystemCaRootBundlePath()
]);