月度归档:2018年09月

php上传目录到阿里云oss,python上传目录到阿里云oss

首先是python版

# -*- coding: utf-8 -*-
import os

import oss2

accessKeyId = "xxx"
accessKeySecret = "xxx"

endpoint = "oss-cn-shanghai-internal.aliyuncs.com"
# endpoint = "oss-cn-shanghai.aliyuncs.com"

bucket = "xxx"
localDirectory = "/data/mysql"
prefix = "db_bak/mysql"

auth = oss2.Auth(accessKeyId, accessKeySecret)
bucket = oss2.Bucket(auth, endpoint, bucket)

# 当文件长度大于或等于可选参数multipart_threshold(默认值为10MB)时,会使用分片上传。如未使用参数store指定目录,则会在HOME目录下建立.py-oss-upload目录来保存断点信息。
# oss2.resumable_upload(bucket, '<yourObjectName>', '<yourLocalFile>')

# 遍历目录进行上传
for fpathe, dirs, fs in os.walk(localDirectory):
for f in fs:
localFileName = os.path.join(fpathe, f)
newObjectName = localFileName[len(localDirectory) + 1:]
newObjectName = prefix + '/' + newObjectName
# 跳过已存在的文件
if bucket.object_exists(newObjectName):
print('pass ' + newObjectName)
continue
# SDK自己分片大文件似乎有bug,用php版吧
oss2.resumable_upload(bucket, newObjectName, localFileName)
print(newObjectName)
print(localFileName)

然后是php版(上传300G的大文件没问题)

<?php
use OSS\Core\OssException;
use OSS\Core\OssUtil;
use OSS\OssClient;

require 'vendor/autoload.php';

$accessKeyId = "xxx";
$accessKeySecret = "xxx";

$endpoint = "oss-cn-shanghai-internal.aliyuncs.com";
//$endpoint = "oss-cn-shanghai.aliyuncs.com";
$bucket = "xxx";
$localDirectory = "/data/mysql";
$prefix = "db/data";

function iScanDir($path = '.')
{
$dirs = scandir($path);
foreach ($dirs as $filename) {
if ($filename == '.' || $filename == '..') {
continue;
}
$fullFileName = "{$path}/{$filename}";
if (is_dir($fullFileName)) {
foreach (iScanDir($fullFileName) as $subFileName) {
yield $subFileName;
}
} else {
yield "{$fullFileName}";
}
}
}

try {
$ossClient = new OssClient($accessKeyId, $accessKeySecret, $endpoint);
} catch (OssException $e) {
exit($e->getMessage() . "\n");
}
$ossClient->setMaxTries(16);

foreach (iScanDir($localDirectory) as $fileName) {
$object = substr($fileName, strlen($localDirectory));
$object = trim($object, '.');
$object = trim($object, '/');
$object = "{$prefix}/{$object}";

$uploadFile = $fileName;
echo "{$fileName} ";

$exist = $ossClient->doesObjectExist($bucket, $object);
if ($exist) {
print(" : exist\n");
continue;
}

try {
$uploadId = $ossClient->initiateMultipartUpload($bucket, $object);
} catch (OssException $e) {
echo "initiateMultipartUpload error: " . $e->getMessage() . "\n";
}

$partSize = 10 * 1024 * 1024;
$uploadFileSize = filesize($uploadFile);
while (true) {
if ($partSize * 9999 < $uploadFileSize) {
$partSize = $partSize + 10 * 1024 * 1024;
} else {
break;
}
}
$pieces = $ossClient->generateMultiuploadParts($uploadFileSize, $partSize);
$responseUploadPart = [];
$uploadPosition = 0;
$isCheckMd5 = true;
foreach ($pieces as $i => $piece) {
$fromPos = $uploadPosition + (integer)$piece[$ossClient::OSS_SEEK_TO];
$toPos = (integer)$piece[$ossClient::OSS_LENGTH] + $fromPos - 1;
$upOptions = [
$ossClient::OSS_FILE_UPLOAD => $uploadFile,
$ossClient::OSS_PART_NUM => ($i + 1),
$ossClient::OSS_SEEK_TO => $fromPos,
$ossClient::OSS_LENGTH => $toPos - $fromPos + 1,
$ossClient::OSS_CHECK_MD5 => $isCheckMd5,
];
// MD5校验。
if ($isCheckMd5) {
$contentMd5 = OssUtil::getMd5SumForFile($uploadFile, $fromPos, $toPos);
$upOptions[$ossClient::OSS_CONTENT_MD5] = $contentMd5;
}
try {
// 上传分片。
$responseUploadPart[] = $ossClient->uploadPart($bucket, $object, $uploadId, $upOptions);
} catch (OssException $e) {
printf(__FUNCTION__ . ": initiateMultipartUpload, uploadPart - part#{$i} FAILED\n");
printf($e->getMessage() . "\n");
return;
}
printf(__FUNCTION__ . ": initiateMultipartUpload, uploadPart - part#{$i} OK\n");
}
// $uploadParts是由每个分片的ETag和分片号(PartNumber)组成的数组。
$uploadParts = [];
foreach ($responseUploadPart as $i => $eTag) {
$uploadParts[] = [
'PartNumber' => ($i + 1),
'ETag' => $eTag,
];
}
/**
* 步骤3:完成上传。
*/
try {
// 在执行该操作时,需要提供所有有效的$uploadPartsOSS收到提交的$uploadParts后,会逐一验证每个分片的有效性。当所有的数据分片验证通过后,OSS将把这些分片组合成一个完整的文件。
$ossClient->completeMultipartUpload($bucket, $object, $uploadId, $uploadParts);
} catch (OssException $e) {
printf(__FUNCTION__ . ": completeMultipartUpload FAILED\n");
printf($e->getMessage() . "\n");
return;
}
printf(__FUNCTION__ . ": completeMultipartUpload OK\n");
}