作者归档:杨龙

php上传目录到阿里云oss,python上传目录到阿里云oss

首先是python版

# -*- coding: utf-8 -*-
import os

import oss2

accessKeyId = "xxx"
accessKeySecret = "xxx"

endpoint = "oss-cn-shanghai-internal.aliyuncs.com"
# endpoint = "oss-cn-shanghai.aliyuncs.com"

bucket = "xxx"
localDirectory = "/data/mysql"
prefix = "db_bak/mysql"

auth = oss2.Auth(accessKeyId, accessKeySecret)
bucket = oss2.Bucket(auth, endpoint, bucket)

# 当文件长度大于或等于可选参数multipart_threshold(默认值为10MB)时,会使用分片上传。如未使用参数store指定目录,则会在HOME目录下建立.py-oss-upload目录来保存断点信息。
# oss2.resumable_upload(bucket, '<yourObjectName>', '<yourLocalFile>')

# 遍历目录进行上传
for fpathe, dirs, fs in os.walk(localDirectory):
for f in fs:
localFileName = os.path.join(fpathe, f)
newObjectName = localFileName[len(localDirectory) + 1:]
newObjectName = prefix + '/' + newObjectName
# 跳过已存在的文件
if bucket.object_exists(newObjectName):
print('pass ' + newObjectName)
continue
# SDK自己分片大文件似乎有bug,用php版吧
oss2.resumable_upload(bucket, newObjectName, localFileName)
print(newObjectName)
print(localFileName)

然后是php版(上传300G的大文件没问题)

<?php
use OSS\Core\OssException;
use OSS\Core\OssUtil;
use OSS\OssClient;

require 'vendor/autoload.php';

$accessKeyId = "xxx";
$accessKeySecret = "xxx";

$endpoint = "oss-cn-shanghai-internal.aliyuncs.com";
//$endpoint = "oss-cn-shanghai.aliyuncs.com";
$bucket = "xxx";
$localDirectory = "/data/mysql";
$prefix = "db/data";

function iScanDir($path = '.')
{
$dirs = scandir($path);
foreach ($dirs as $filename) {
if ($filename == '.' || $filename == '..') {
continue;
}
$fullFileName = "{$path}/{$filename}";
if (is_dir($fullFileName)) {
foreach (iScanDir($fullFileName) as $subFileName) {
yield $subFileName;
}
} else {
yield "{$fullFileName}";
}
}
}

try {
$ossClient = new OssClient($accessKeyId, $accessKeySecret, $endpoint);
} catch (OssException $e) {
exit($e->getMessage() . "\n");
}
$ossClient->setMaxTries(16);

foreach (iScanDir($localDirectory) as $fileName) {
$object = substr($fileName, strlen($localDirectory));
$object = trim($object, '.');
$object = trim($object, '/');
$object = "{$prefix}/{$object}";

$uploadFile = $fileName;
echo "{$fileName} ";

$exist = $ossClient->doesObjectExist($bucket, $object);
if ($exist) {
print(" : exist\n");
continue;
}

try {
$uploadId = $ossClient->initiateMultipartUpload($bucket, $object);
} catch (OssException $e) {
echo "initiateMultipartUpload error: " . $e->getMessage() . "\n";
}

$partSize = 10 * 1024 * 1024;
$uploadFileSize = filesize($uploadFile);
while (true) {
if ($partSize * 9999 < $uploadFileSize) {
$partSize = $partSize + 10 * 1024 * 1024;
} else {
break;
}
}
$pieces = $ossClient->generateMultiuploadParts($uploadFileSize, $partSize);
$responseUploadPart = [];
$uploadPosition = 0;
$isCheckMd5 = true;
foreach ($pieces as $i => $piece) {
$fromPos = $uploadPosition + (integer)$piece[$ossClient::OSS_SEEK_TO];
$toPos = (integer)$piece[$ossClient::OSS_LENGTH] + $fromPos - 1;
$upOptions = [
$ossClient::OSS_FILE_UPLOAD => $uploadFile,
$ossClient::OSS_PART_NUM => ($i + 1),
$ossClient::OSS_SEEK_TO => $fromPos,
$ossClient::OSS_LENGTH => $toPos - $fromPos + 1,
$ossClient::OSS_CHECK_MD5 => $isCheckMd5,
];
// MD5校验。
if ($isCheckMd5) {
$contentMd5 = OssUtil::getMd5SumForFile($uploadFile, $fromPos, $toPos);
$upOptions[$ossClient::OSS_CONTENT_MD5] = $contentMd5;
}
try {
// 上传分片。
$responseUploadPart[] = $ossClient->uploadPart($bucket, $object, $uploadId, $upOptions);
} catch (OssException $e) {
printf(__FUNCTION__ . ": initiateMultipartUpload, uploadPart - part#{$i} FAILED\n");
printf($e->getMessage() . "\n");
return;
}
printf(__FUNCTION__ . ": initiateMultipartUpload, uploadPart - part#{$i} OK\n");
}
// $uploadParts是由每个分片的ETag和分片号(PartNumber)组成的数组。
$uploadParts = [];
foreach ($responseUploadPart as $i => $eTag) {
$uploadParts[] = [
'PartNumber' => ($i + 1),
'ETag' => $eTag,
];
}
/**
* 步骤3:完成上传。
*/
try {
// 在执行该操作时,需要提供所有有效的$uploadPartsOSS收到提交的$uploadParts后,会逐一验证每个分片的有效性。当所有的数据分片验证通过后,OSS将把这些分片组合成一个完整的文件。
$ossClient->completeMultipartUpload($bucket, $object, $uploadId, $uploadParts);
} catch (OssException $e) {
printf(__FUNCTION__ . ": completeMultipartUpload FAILED\n");
printf($e->getMessage() . "\n");
return;
}
printf(__FUNCTION__ . ": completeMultipartUpload OK\n");
}

pt-online-schema-change 最佳参数

pt-online-schema-change \
--user username \
--password 123456 \
--host x.x.x.x D=db,t=table \
--execute --alter "ADD COLUMN vip_version TINYINT NOT NULL DEFAULT 0" \
--charset utf8mb4 \
--no-version-check \
--nocheck-replication-filters \
--max-load Threads_running=10 \
--critical-load Threads_running=200 \
--chunk-time=0.5

–charset utf8mb4 设置下编码

–max-load Threads_running=10 低负载才运行,设置尽量小点

–critical-load Threads_running=200 终止条件 设置大点防止容易失败

–no-version-check 阿里云rds可能需要此参数

–chunk-time=0.5 这个优于设置chunk-size

–nocheck-replication-filters 有主从配置的时候需要 阿里云rds可能不需要这个参数

简单实现MySQL负载低时才继续运行任务,负载高的时候睡眠

public static function runSleep($sec = 10, $echo = null) {
while (true) {
$result = NoThingModel::getInstance()->db->select("show status like 'Threads_running'", [], false);
$result = reset($result);
if ($result) {
if ($result->Variable_name == 'Threads_running') {
if ($result->Value > 10) {
if ($echo) {
echo "sleep 10\n";
}
sleep($sec);
} else {
break;
}
} else {
break;
}
} else {
break;
}
}
}

GuzzleHttp 400 500 时获取内容的方法

http_errors

Summary
Set to false to disable throwing exceptions on an HTTP protocol errors (i.e., 4xx and 5xx responses). Exceptions are thrown by default when HTTP protocol errors are encountered.
Types
bool
Default
true
Constant
GuzzleHttp\RequestOptions::HTTP_ERRORS
$client->request('GET', '/status/500');
// Throws a GuzzleHttp\Exception\ServerException

$res = $client->request('GET', '/status/500', ['http_errors' => false]);
echo $res->getStatusCode();
// 500