Commit f64ed480 authored by Thorsten Buss's avatar Thorsten Buss

* deprecate AWS S3 SDKver1 Adapter

* add UrlAware Interface for Adapter
* add ctime(), atime(), contentType(), append(), downloadFile()
* change download/upload to pull/push
* optimize keys() ($prefix, $withDirectoryFlag)
* add uri-handling (bnrepo://REPONAME/KEY) for Linker and generation in repo
* add tests for new functions
parent 7ea4f1fc
......@@ -12,7 +12,7 @@ bnrepo-test-s3-DISABLED:
create: true
use_old_version: false
options:
default_acl: public-read #DEFAULT is only BucketOwner can read, so everyone with the link can read
default_acl: public-read #bucket-owner-full-control[DEFAULT]|private|public-read|public-read-write|authenticated-read|bucket-owner-read
bnrepo-test-ftp-DISABLED:
type: ftp
......
<?php
/**
* User: thorsten
* Date: 18.04.13
* Time: 14:03
*/
namespace BNRepo\Repository\Adapter;
use Gaufrette\Exception\FileNotFound;
use Gaufrette\Exception\UnexpectedFile;
interface Adapter extends \Gaufrette\Adapter {
/**
* Returns the created time
*
* @param string $key
*
* @return integer|boolean An UNIX like timestamp or false
*/
public function ctime($key);
/**
* Returns the last accessed time
*
* @param string $key
*
* @return integer|boolean An UNIX like timestamp or false
*/
public function atime($key);
/**
* Returns an iterator over all keys (files and directories)
*
* @param null $prefix
* @return mixed
*/
public function getKeyIterator($prefix = null);
/**
* Returns an array of all keys (files and directories)
*
* @param null $prefix
* @return array
*/
public function keys($prefix=null, $withDirectories=false);
/**
* Returns the MimeType of the given Key
* @param $key
* @return mixed
*/
public function getContentType($key);
/**
* Downloads a file to Local
*
* @param string $sourceKey
* @param string $localTargetFile
*
* @return boolean TRUE if the download was successful
* @throws FileNotFound when sourceKey does not exist
* @throws UnexpectedFile when targetKey exists
* @throws \RuntimeException when cannot download
*/
public function pull($sourceKey, $localTargetFile);
/**
* Uploads a Local file
*
* @param string $localFile
* @param string $targetKey
*
* @return boolean TRUE if the upload was successful
* @throws FileNotFound when sourceKey does not exist
* @throws UnexpectedFile when targetKey exists
* @throws \RuntimeException when cannot upload
*/
public function push($localFile, $targetKey);
/**
* Appends the given content on the file
*
* @param string $key
* @param string $content
*
* @return integer|boolean The number of bytes that were written into the file
*/
public function append($key, $content);
}
\ No newline at end of file
......@@ -7,8 +7,20 @@ use Gaufrette\Adapter\AmazonS3;
use Gaufrette\Exception\FileNotFound;
use Gaufrette\Exception\UnexpectedFile;
class AdapterAmazonS3 extends AmazonS3 implements AdapterUploadable, AdapterDownloadable, AdapterLinkable {
public function __construct(\AmazonS3 $service, $bucket, $options = array()) {
/**
* Class AdapterAmazonS3
* @package BNRepo\Repository\Adapter
* @deprecated
*/
class AdapterAmazonS3 extends AmazonS3 implements Adapter {
/**
* @var \AmazonS3
*/
protected $service;
public function __construct(\AmazonS3 $service, $bucket, $options = array()) {
throw new \RuntimeException('Adapter AWS-S3 SDKver1 not longer supported');
if (isset($options['directory'])) { // strip double slashes and secure that the dir not start and ends with an slash
$options['directory'] = substr(preg_replace('~/+~', '/', '/'. $options['directory'].'/'), 1, -1);
}
......@@ -17,8 +29,29 @@ class AdapterAmazonS3 extends AmazonS3 implements AdapterUploadable, AdapterDown
parent::__construct($service, $bucket, $options);
}
/**
* Returns the created time
*
* @param string $key
*
* @return integer|boolean An UNIX like timestamp or false
*/
public function ctime($key) {
throw new \RuntimeException('Adapter does not support ctime function.');
}
/**
* Returns the last accessed time
*
* @param string $key
*
* @return integer|boolean An UNIX like timestamp or false
*/
public function atime($key) {
throw new \RuntimeException('Adapter does not support atime function.');
}
/**
* Uploads a Local file
*
* @param string $localFile
......@@ -29,7 +62,7 @@ class AdapterAmazonS3 extends AmazonS3 implements AdapterUploadable, AdapterDown
* @throws UnexpectedFile when targetKey exists
* @throws \RuntimeException when cannot rename
*/
public function upload($localFile, $targetKey) {
public function push($localFile, $targetKey) {
return $this->write($targetKey, file_get_contents($localFile));
}
......@@ -45,7 +78,7 @@ class AdapterAmazonS3 extends AmazonS3 implements AdapterUploadable, AdapterDown
* @throws UnexpectedFile when targetKey exists
* @throws \RuntimeException when cannot rename
*/
public function download($sourceKey, $localTargetFile) {
public function pull($sourceKey, $localTargetFile) {
return file_put_contents($localTargetFile, $this->read($sourceKey));
}
......@@ -139,8 +172,10 @@ class AdapterAmazonS3 extends AmazonS3 implements AdapterUploadable, AdapterDown
/**
* Add Directory-Prefix and remove fullPath and emptyDirLine from output for similar response to local/ftp etc
* {@inheritDoc}
* @deprecated
*/
public function keys() {
public function keys($prefix=null, $withDirectories=false) {
//@todo: implement $prefix and $withDirectories
$this->ensureBucketExists();
$list = $this->service->get_object_list($this->bucket, array(
......@@ -200,4 +235,36 @@ class AdapterAmazonS3 extends AmazonS3 implements AdapterUploadable, AdapterDown
);
}
/**
* {@inheritDoc}
*/
public function getKeyIterator($prefix = null) {
throw new \RuntimeException('function getKeyIterator not support by this adapter');
}
/**
* Returns the MimeType of the given Key
* @param $key
* @return mixed
*/
public function getContentType($key) {
// @todo implement
throw new \RuntimeException('function getContentType not support by this adapter');
}
/**
* Appends the given content on the file
*
* @param string $key
* @param string $content
*
* @return integer|boolean The number of bytes that were written into the file
*/
public function append($key, $content) {
// TODO: Implement append() method.
throw new \RuntimeException('function getContentType not support by this adapter');
}
}
......@@ -13,7 +13,7 @@ use Gaufrette\Exception\UnexpectedFile;
use Gaufrette\Adapter;
use Guzzle\Service\Resource\Model;
class AdapterAmazonS3Ver2 extends AmazonS3 implements AdapterUploadable, AdapterDownloadable, AdapterLinkable, Adapter {
class AdapterAmazonS3Ver2 extends AmazonS3 implements Adapter, UrlAware {
protected $service;
protected $bucket;
......@@ -32,8 +32,30 @@ class AdapterAmazonS3Ver2 extends AmazonS3 implements AdapterUploadable, Adapter
$this->setDirectory($options['directory']);
}
/**
* Returns the created time
*
* @param string $key
*
* @return integer|boolean An UNIX like timestamp or false
*/
public function ctime($key) {
throw new \RuntimeException('Adapter does not support ctime function.');
}
/**
/**
* Returns the last accessed time
*
* @param string $key
*
* @return integer|boolean An UNIX like timestamp or false
*/
public function atime($key) {
throw new \RuntimeException('Adapter does not support atime function.');
}
/**
* Uploads a Local file
*
* @param string $localFile
......@@ -44,7 +66,7 @@ class AdapterAmazonS3Ver2 extends AmazonS3 implements AdapterUploadable, Adapter
* @throws UnexpectedFile when targetKey exists
* @throws \RuntimeException when cannot rename
*/
public function upload($localFile, $targetKey) {
public function push($localFile, $targetKey) {
return $this->write($targetKey, fopen($localFile, 'r'));
}
......@@ -60,7 +82,7 @@ class AdapterAmazonS3Ver2 extends AmazonS3 implements AdapterUploadable, Adapter
* @throws UnexpectedFile when targetKey exists
* @throws \RuntimeException when cannot rename
*/
public function download($sourceKey, $localTargetFile) {
public function pull($sourceKey, $localTargetFile) {
try {
$this->getObject($sourceKey, array(
'SaveAs' => $localTargetFile
......@@ -76,7 +98,7 @@ class AdapterAmazonS3Ver2 extends AmazonS3 implements AdapterUploadable, Adapter
* Retrieve the S3 object URL for the given key.
*
* @param string $key
* @param integer|string $preauth Look at \AmazonS3::get_object_url() docs
* @param integer|string $validTime seconds or strtotime() string | 0=> endless
* @param array $opt Look at \AmazonS3::get_object_url() docs
*
* @see \AmazonS3::get_object_url()
......@@ -84,15 +106,34 @@ class AdapterAmazonS3Ver2 extends AmazonS3 implements AdapterUploadable, Adapter
* @return string The S3 object URL
*/
public function getUrl($key, $validTime = 0, $options = array()) {
$url = "{$this->bucket}/{$this->computePath($key)}";
if (array_key_exists('filename', $options))
$url .= '?response-content-disposition='.urlencode("attachment; filename=\"{$options['filename']}\"");
$request = $this->service->get($url);
if (is_numeric($validTime))
$validTime = '+'.$validTime.' seconds';
return $this->service->createPresignedUrl($request, $validTime);
// for CloudFront or static Domain, set download_url in Repositories.yml
if (array_key_exists('download_url', $options)) {
return $options['download_url'];
}
$url = "https://{$this->bucket}.s3.amazonaws.com/{$this->computePath($key)}";
// Public Access, or Signed URL
if (in_array($this->options['default_acl'], array(CannedAcl::PUBLIC_READ, CannedAcl::PUBLIC_READ_WRITE))) {
return $url;
} else {
// Avialable request_options: response-content-type, response-content-language, response-expires, response-cache-control, response-content-disposition, response-content-encoding
$request_options = array();
if (array_key_exists('filename', $options))
$request_options['response-content-disposition'] = "attachment; filename=\"{$options['filename']}\"";
$request_options['response-content-type'] = @$options['content_type'] ?: $this->getContentType($key);
$url .= "?". http_build_query($request_options);
$request = $this->service->get($url);
if (empty($validTime))
$validTime = 2147483647; // End of UnixTime
elseif (is_numeric($validTime))
$validTime = '+'.$validTime.' seconds';
return $this->service->createPresignedUrl($request, $validTime);
}
}
/**
......@@ -159,28 +200,50 @@ class AdapterAmazonS3Ver2 extends AmazonS3 implements AdapterUploadable, Adapter
}
/**
* Add Directory-Prefix and remove fullPath and emptyDirLine from output for similar response to local/ftp etc
* {@inheritDoc}
* @return Model
*/
public function keys() {
$this->ensureBucketExists();
public function getKeyIterator($prefix = null) {
$this->ensureBucketExists();
// add slash to beginning and remove from end
$prefix = rtrim(preg_replace('/^[\/]*([^\/].*)[\/]?$/', '/$1', $prefix), '/');
/** @var $list Model */
$list = $this->service->getIterator('ListObjects', array(
return $this->service->getIterator('ListObjects', array(
'Bucket' => $this->bucket,
'Prefix' => $this->getDirectory()
'Prefix' => $this->getDirectory().$prefix
));
}
/**
* Add Directory-Prefix and remove fullPath and emptyDirLine from output for similar response to local/ftp etc
* {@inheritDoc}
*/
public function keys($prefix = null, $withDirectories = false) {
$this->ensureBucketExists();
// add slash to beginning and remove from end
$prefix = preg_replace('/^[\/]*([^\/].*)[\/]?$/', '/$1', $prefix);
$iterator = $this->getKeyIterator($prefix);
$keys = array();
$dirLength = strlen($this->getDirectory())+1; //+1 to remove the starting slash
foreach ($list as $file) {
$key = $file['Key'];
if (strlen(dirname($key)) > $dirLength)
$keys[] = substr(dirname($key), $dirLength);
elseif (strlen($key) > $dirLength)
$keys[] = substr($key, $dirLength);
}
sort($keys);
$paths = array();
$prefix_dir = rtrim(substr($prefix, -1) != '/'?dirname($prefix):$prefix, '/');
$dirLength = strlen($this->getDirectory() . $prefix_dir) + 1; //+1 to remove the starting slash
foreach ($iterator as $item) {
$file = substr($item['Key'], $dirLength);
if (!$file) continue;
$dir = dirname($file);
// Directory
if (strlen($dir) > 0 && $dir != '.')
$paths[$dir] = true;
// File
$keys[] = $file;
}
if ($withDirectories)
$keys = array_merge(array_keys($paths), $keys);
sort($keys);
return $keys;
}
......@@ -194,7 +257,7 @@ class AdapterAmazonS3Ver2 extends AmazonS3 implements AdapterUploadable, Adapter
$this->ensureBucketExists();
$options = array_replace_recursive(
array('acl' => $this->options['default_acl']),
array('ACL' => $this->options['default_acl']),
$this->getMetadata($key),
array(
'Body' => $content,
......@@ -291,7 +354,8 @@ class AdapterAmazonS3Ver2 extends AmazonS3 implements AdapterUploadable, Adapter
*/
public function mtime($key) {
$this->ensureBucketExists();
return strtotime($this->getObject($key)['LastModified']);
$obj = $this->getObject($key);
return strtotime($obj['LastModified']);
}
......@@ -339,4 +403,21 @@ class AdapterAmazonS3Ver2 extends AmazonS3 implements AdapterUploadable, Adapter
return $this->service->getObject($options);
}
/**
* Returns the MimeType of the given Key
* @param $key
* @return mixed
*/
public function getContentType($key) {
$obj = $this->getObject($key);
return $obj['ContentType'];
}
/**
* {@inheritDoc}
*/
public function append($key, $content) {
return $this->write($key, $this->read($key).$content);
}
}
<?php
namespace BNRepo\Repository\Adapter;
use Gaufrette\Exception\FileNotFound;
use Gaufrette\Exception\UnexpectedFile;
interface AdapterDownloadable {
/**
* Downloads a file to Local
*
* @param string $sourceKey
* @param string $localTargetFile
*
* @return boolean TRUE if the download was successful
* @throws FileNotFound when sourceKey does not exist
* @throws UnexpectedFile when targetKey exists
* @throws \RuntimeException when cannot download
*/
public function download($sourceKey, $localTargetFile);
}
......@@ -7,15 +7,49 @@ use Gaufrette\Adapter\Ftp;
use Gaufrette\Exception\FileNotFound;
use Gaufrette\Exception\UnexpectedFile;
class AdapterFtp extends Ftp implements AdapterUploadable, AdapterDownloadable {
public function __construct($directory, $host, $options = array()) {
class AdapterFtp extends Ftp implements Adapter {
public function __construct($directory, $host, $options = array()) {
// strip double slashes and secure that the dir start with an slash and not end with one
$directory = substr(preg_replace('~/+~', '/', '/' . $directory . '/'), 0, -1);
parent::__construct($directory, $host, $options);
}
/**
* Returns the created time
*
* @param string $key
*
* @return integer|boolean An UNIX like timestamp or false
*/
public function ctime($key) {
throw new \RuntimeException('Adapter does not support ctime function.');
// $curl = curl_init();
// curl_setopt($curl, CURLOPT_URL, "ftp://server/file");
//
// curl_setopt($curl, CURLOPT_USERPWD, "user:pass");
// curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
// curl_setopt($curl, CURLOPT_NOBODY, 1);
//
// curl_setopt($curl, CURLOPT_FILETIME, TRUE);
//
// $result = curl_exec($curl);
// $time = curl_getinfo($curl, CURLINFO_FILETIME);
// print date('d/m/y H:i:s', $time);
}
/**
* Returns the last accessed time
*
* @param string $key
*
* @return integer|boolean An UNIX like timestamp or false
*/
public function atime($key) {
throw new \RuntimeException('Adapter does not support atime function.');
}
/**
/**
* Uploads a Local file
*
* @param string $localFile
......@@ -26,7 +60,7 @@ class AdapterFtp extends Ftp implements AdapterUploadable, AdapterDownloadable {
* @throws UnexpectedFile when targetKey exists
* @throws \RuntimeException when cannot rename
*/
public function upload($localFile, $targetKey) {
public function push($localFile, $targetKey) {
return $this->write($targetKey, file_get_contents($localFile));
}
......@@ -42,8 +76,122 @@ class AdapterFtp extends Ftp implements AdapterUploadable, AdapterDownloadable {
* @throws UnexpectedFile when targetKey exists
* @throws \RuntimeException when cannot rename
*/
public function download($sourceKey, $localTargetFile) {
public function pull($sourceKey, $localTargetFile) {
return file_put_contents($localTargetFile, $this->read($sourceKey));
}
/**
* Returns an iterator over all keys (files and directories)
*
* @param null $prefix
* @return mixed
*/
/**
* {@inheritDoc}
*/
public function getKeyIterator($prefix = null) {
throw new \RuntimeException('Adapter does not support getKeyIterator function.');
}
/**
* Add Directory-Prefix and remove fullPath and emptyDirLine from output for similar response to local/ftp etc
* {@inheritDoc}
*/
public function keys($prefix = null, $withDirectories = false) {
return $this->fetchKeys($prefix, $withDirectories);
}
public function fetchKeys($prefix = null, $withDirectories = false, $active_dir='') {
$path = $prefix;
if (substr($path, -1) != '/') {
$path = ltrim(dirname($prefix), '.');
$fileSearch = basename($prefix);
}
// add slash to beginning and remove from end
$path = rtrim(preg_replace('/^[\/]*([^\/].*)[\/]?$/', '/$1', $path), '/');
$items = $this->listDirectory($path.$active_dir);
$keys = $withDirectories?$items['dirs']:array();
foreach ($items['dirs'] as $dir) {
// if subfolder dont start with fileSearch -> ignore
if (!empty($fileSearch) && strpos($dir, $fileSearch) === false) continue;
$keys = array_merge($keys, $this->fetchKeys($path, $withDirectories, $dir));
}
if ($withDirectories)
$keys = array_merge($items['dirs'], $keys);
$keys = array_merge($items['keys'], array_unique($keys));
// remove path
if (!empty($prefix)) {
$pathLen = mb_strlen($path);
array_walk($keys, function(&$file) use($pathLen) {
$file = substr($file, $pathLen);
});
if (!empty($fileSearch))
$keys = array_filter($keys, function($file) use($fileSearch) {
return strpos($file, $fileSearch) !== false;
});
}
return $keys;
}
/**
* Computes the path for the given key
*
* @param string $key
*/
protected function computePath($key) {
return rtrim($this->directory, '/') . '/' . $key;
}
/**
* Returns the MimeType of the given Key
* @param $key
* @return mixed
*/
public function getContentType($key) {
/* List of Options
* FILEINFO_NONE => 'PHP script, ASCII text'
* FILEINFO_MIME_TYPE => 'text/x-php'
* FILEINFO_MIME => 'text/x-php; charset=us-ascii'
*/
$finfo = new \finfo(); // return mime type ala mimetype extension
return $finfo->file($this->getFtpUrl($key), FILEINFO_MIME_TYPE);
}
/**
* Build the FTP Url with this Data
* @param $key
* @return string
*/
protected function getFtpUrl($key) {
return 'ftp://' . $this->username . ':' . $this->password . '@' . $this->host . $this->computePath($key);
}
/**
* {@inheritDoc}
*/
public function append($key, $content) {
// @todo make better if private method getConnection() in GaufretteAdapter is changed
return $this->write($key, $this->read($key) . $content);
// $path = $this->computePath($key);
// $directory = dirname($path);
//
// $this->ensureDirectoryExists($directory, true);
//
// $temp = fopen('php://temp', 'a');
// $size = fwrite($temp, $content);
//
// if (!ftp_fput($this->getConnection(), $path, $temp, $this->mode)) {
// fclose($temp);
// return false;
// }
//
// fclose($temp);
//
// return $size;
}
}
......@@ -7,7 +7,29 @@ use Gaufrette\Adapter\Local;
use Gaufrette\Exception\FileNotFound;
use Gaufrette\Exception\UnexpectedFile;
class AdapterLocal extends Local implements AdapterUploadable, AdapterDownloadable {
class AdapterLocal extends Local implements Adapter {
/**
* Returns the created time
*
* @param string $key
*
* @return integer|boolean An UNIX like timestamp or false
*/
public function ctime($key) {
return filectime($this->computePath($key));
}
/**
* Returns the last accessed time
*
* @param string $key
*
* @return integer|boolean An UNIX like timestamp or false
*/
public function atime($key) {
return fileatime($this->computePath($key));
}
/**
* Uploads a Local file
......@@ -20,8 +42,10 @@ class AdapterLocal extends Local implements AdapterUploadable, AdapterDownloadab
* @throws UnexpectedFile when targetKey exists
* @throws \RuntimeException when cannot rename
*/
public function upload($localFile, $targetKey) {
return copy($localFile, $this->computePath($targetKey));
public function push($localFile, $targetKey) {
$path = $this->computePath($targetKey);
$this->ensureDirectoryExists(dirname($path), true);
return copy($localFile, $path);
}
......@@ -36,8 +60,94 @@ class AdapterLocal extends Local implements AdapterUploadable, AdapterDownloadab
* @throws UnexpectedFile when targetKey exists
* @throws \RuntimeException when cannot rename
*/
public function download($sourceKey, $localTargetFile) {
public function pull($sourceKey, $localTargetFile) {
return copy($this->computePath($sourceKey), $localTargetFile);
}
/**
* {@inheritDoc}
*/
public function getKeyIterator($prefix=null) {
$this->ensureDirectoryExists($this->directory, false);
// add slash to beginning and remove from end
$prefix = rtrim(preg_replace('/^[\/]*([^\/].*)[\/]?$/', '/$1', $prefix), '/');
$path = $this->directory .$prefix;
try {
$iterator = new \RecursiveIteratorIterator(
new \RecursiveDirectoryIterator(
$path,
\FilesystemIterator::SKIP_DOTS | \FilesystemIterator::UNIX_PATHS
)
);
} catch (\Exception $e) {
$iterator = new \EmptyIterator;