Actualización

This commit is contained in:
Xes
2025-04-10 12:24:57 +02:00
parent 8969cc929d
commit 45420b6f0d
39760 changed files with 4303286 additions and 0 deletions
+13
View File
@@ -0,0 +1,13 @@
<?php
namespace Ddeboer\DataImport;
/**
* Base exception
*
* @author David de Boer <david@ddeboer.nl>
*/
interface Exception
{
}
@@ -0,0 +1,17 @@
<?php
namespace Ddeboer\DataImport\Exception;
/**
* @author David de Boer <david@ddeboer.nl>
*/
class DuplicateHeadersException extends ReaderException
{
/**
* @param array $duplicates
*/
public function __construct(array $duplicates)
{
parent::__construct(sprintf('File contains duplicate headers: %s', implode($duplicates, ', ')));
}
}
@@ -0,0 +1,15 @@
<?php
namespace Ddeboer\DataImport\Exception;
use Ddeboer\DataImport\Exception;
/**
* Description of MappingException
*
* @author gnat
*/
class MappingException extends \Exception implements Exception
{
}
@@ -0,0 +1,11 @@
<?php
namespace Ddeboer\DataImport\Exception;
/**
* @author David de Boer <david@ddeboer.nl>
*/
class ReaderException extends UnexpectedValueException
{
}
@@ -0,0 +1,13 @@
<?php
namespace Ddeboer\DataImport\Exception;
use Ddeboer\DataImport\Exception;
/**
* @author Markus Bachmann <markus.bachmann@bachi.biz>
*/
class SourceNotFoundException extends \Exception implements Exception
{
}
@@ -0,0 +1,22 @@
<?php
namespace Ddeboer\DataImport\Exception;
/**
* @author David de Boer <david@ddeboer.nl>
*/
class UnexpectedTypeException extends UnexpectedValueException
{
/**
* @param mixed $value
* @param string $expectedType
*/
public function __construct($value, $expectedType)
{
parent::__construct(sprintf(
'Expected argument of type "%s", "%s" given',
$expectedType,
is_object($value) ? get_class($value) : gettype($value)
));
}
}
@@ -0,0 +1,14 @@
<?php
namespace Ddeboer\DataImport\Exception;
use Ddeboer\DataImport\Exception;
/**
* @author David de Boer <david@ddeboer.nl>
*/
class UnexpectedValueException extends \UnexpectedValueException implements Exception
{
}
@@ -0,0 +1,48 @@
<?php
namespace Ddeboer\DataImport\Exception;
use Ddeboer\DataImport\Exception;
use Symfony\Component\Validator\ConstraintViolationListInterface;
/**
* @author Markus Bachmann <markus.bachmann@bachi.biz>
*/
class ValidationException extends \Exception implements Exception
{
/**
* @var ConstraintViolationListInterface
*/
private $violations;
/**
* @var integer
*/
private $lineNumber;
/**
* @param ConstraintViolationListInterface $list
* @param integer $line
*/
public function __construct(ConstraintViolationListInterface $list, $line)
{
$this->violations = $list;
$this->lineNumber = $line;
}
/**
* @return ConstraintViolationListInterface
*/
public function getViolations()
{
return $this->violations;
}
/**
* @return integer
*/
public function getLineNumber()
{
return $this->lineNumber;
}
}
@@ -0,0 +1,13 @@
<?php
namespace Ddeboer\DataImport\Exception;
use Ddeboer\DataImport\Exception;
/**
* @author David de Boer <david@ddeboer.nl>
*/
class WriterException extends \Exception implements Exception
{
}
@@ -0,0 +1,84 @@
<?php
namespace Ddeboer\DataImport\Filter;
use Ddeboer\DataImport\ValueConverter\DateTimeValueConverter;
/**
* This filter can be used to filter out some items from a specific date.
*
* Useful to do incremental imports
*
* @author Grégoire Paris
*/
class DateTimeThresholdFilter
{
/**
* Threshold dates strictly before this date will be filtered out
*
* @var \DateTime|null
*/
protected $threshold;
/**
* Used to convert the values in the time column
*
* @var DateTimeValueConverter
*/
protected $valueConverter;
/**
* The name of the column that should contain the value the filter will compare the threshold with
*
* @var string
*/
protected $timeColumnName = 'updated_at';
/**
* @var integer
*/
protected $priority = 512;
/**
* @param DateTimeValueConverter $valueConverter
* @param \DateTime|null $threshold
* @param string $timeColumnName
* @param integer $priority
*/
public function __construct(
DateTimeValueConverter $valueConverter,
\DateTime $threshold = null,
$timeColumnName = 'updated_at',
$priority = 512
) {
$this->valueConverter = $valueConverter;
$this->threshold = $threshold;
$this->timeColumnName = $timeColumnName;
$this->priority = $priority;
}
/**
* {@inheritdoc}
*/
public function __invoke(array $item)
{
if ($this->threshold == null) {
throw new \LogicException('Make sure you set a threshold');
}
$threshold = call_user_func($this->valueConverter, $item[$this->timeColumnName]);
return $threshold >= $this->threshold;
}
/**
* Useful if you build a filter service, and want to set the threshold
* dynamically afterwards.
*
* @param \DateTime $value
*/
public function setThreshold(\DateTime $value)
{
$this->threshold = $value;
}
}
+82
View File
@@ -0,0 +1,82 @@
<?php
namespace Ddeboer\DataImport\Filter;
/**
* This filter can be used to filter out some items from the beginning and/or
* end of the items.
*
* @author Ville Mattila <ville@eventio.fi>
*/
class OffsetFilter
{
/**
* @var integer
*/
protected $offset = 0;
/**
* @var integer|null
*/
protected $limit = null;
/**
* @var integer
*/
protected $offsetCount = 0;
/**
* @var integer
*/
protected $sliceCount = 0;
/**
* @var boolean
*/
protected $maxLimitHit = false;
/**
* @param integer $offset 0-based index of the item to start read from
* @param integer|null $limit Maximum count of items to read. null = no limit
*/
public function __construct($offset = 0, $limit = null)
{
$this->offset = $offset;
$this->limit = $limit;
}
/**
* {@inheritdoc}
*/
public function __invoke(array $item)
{
// In case we've already filtered up to limited
if ($this->maxLimitHit) {
return false;
}
$this->offsetCount++;
// We have not reached the start offset
if ($this->offsetCount < $this->offset + 1) {
return false;
}
// There is no maximum limit, so we'll return always true
if (null === $this->limit) {
return true;
}
$this->sliceCount++;
if ($this->sliceCount < $this->limit) {
return true;
} elseif ($this->sliceCount == $this->limit) {
$this->maxLimitHit = true;
return true;
}
return false;
}
}
@@ -0,0 +1,117 @@
<?php
namespace Ddeboer\DataImport\Filter;
use Symfony\Component\Validator\Validator\ValidatorInterface;
use Symfony\Component\Validator\Constraint;
use Symfony\Component\Validator\Constraints;
use Ddeboer\DataImport\Exception\ValidationException;
/**
* @author Markus Bachmann <markus.bachmann@bachi.biz>
*/
class ValidatorFilter
{
/**
* @var ValidatorInterface
*/
private $validator;
/**
* @var boolean
*/
private $throwExceptions = false;
/**
* @var integer
*/
private $line = 1;
/**
* @var boolean
*/
private $strict = true;
/**
* @var array
*/
private $constraints = [];
/**
* @var array
*/
private $violations = [];
/**
* @param ValidatorInterface $validator
*/
public function __construct(ValidatorInterface $validator)
{
$this->validator = $validator;
}
/**
* @param string $field
* @param Constraint $constraint
*/
public function add($field, Constraint $constraint)
{
if (!isset($this->constraints[$field])) {
$this->constraints[$field] = [];
}
$this->constraints[$field][] = $constraint;
}
/**
* @param boolean $flag
*/
public function throwExceptions($flag = true)
{
$this->throwExceptions = $flag;
}
/**
* @param boolean $strict
*/
public function setStrict($strict)
{
$this->strict = $strict;
}
/**
* @return array
*/
public function getViolations()
{
return $this->violations;
}
/**
* @param array $item
*
* @return boolean
*/
public function __invoke(array $item)
{
if (!$this->strict) {
// Only validate properties which have an constaint.
$temp = array_intersect(array_keys($item), array_keys($this->constraints));
$item = array_intersect_key($item, array_flip($temp));
}
$constraints = new Constraints\Collection($this->constraints);
$list = $this->validator->validate($item, $constraints);
$currentLine = $this->line++;
if (count($list) > 0) {
$this->violations[$currentLine] = $list;
if ($this->throwExceptions) {
throw new ValidationException($list, $currentLine);
}
}
return 0 === count($list);
}
}
+18
View File
@@ -0,0 +1,18 @@
<?php
namespace Ddeboer\DataImport;
/**
* Iterator that reads data to be imported
*
* @author David de Boer <david@ddeboer.nl>
*/
interface Reader extends \Iterator
{
/**
* Get the field (column, property) names
*
* @return array
*/
public function getFields();
}
+24
View File
@@ -0,0 +1,24 @@
<?php
namespace Ddeboer\DataImport\Reader;
/**
* Reads an array
*
* @author David de Boer <david@ddeboer.nl>
*/
class ArrayReader extends \ArrayIterator implements CountableReader
{
/**
* {@inheritdoc}
*/
public function getFields()
{
// Examine first row
if ($this->count() > 0) {
return array_keys($this[0]);
}
return [];
}
}
@@ -0,0 +1,31 @@
<?php
namespace Ddeboer\DataImport\Reader;
/**
* Use a class implementing both \Iterator and \Countable as a reader
*
* This class uses count() on iterators implementing \Countable interface
* and iterator_count in any further cases
*
* Be careful! iterator_count iterates through the whole iterator loading every data into the memory (for example from streams)
* It is not recommended for very big datasets.
*
* @author Márk Sági-Kazár <mark.sagikazar@gmail.com>
*/
class CountableIteratorReader extends IteratorReader implements CountableReader
{
/**
* {@inheritdoc}
*/
public function count()
{
$iterator = $this->getInnerIterator();
if ($iterator instanceof \Countable) {
return count($iterator);
}
return iterator_count($iterator);
}
}
@@ -0,0 +1,15 @@
<?php
namespace Ddeboer\DataImport\Reader;
use Ddeboer\DataImport\Reader;
/**
* Reader that provides the count of total items
*
* @author David de Boer <david@ddeboer.nl>
*/
interface CountableReader extends Reader, \Countable
{
// Don't add count() to interface: see https://github.com/ddeboer/data-import/pull/5
}
+414
View File
@@ -0,0 +1,414 @@
<?php
namespace Ddeboer\DataImport\Reader;
use Ddeboer\DataImport\Exception\DuplicateHeadersException;
/**
* Reads a CSV file, using as little memory as possible
*
* @author David de Boer <david@ddeboer.nl>
*/
class CsvReader implements CountableReader, \SeekableIterator
{
const DUPLICATE_HEADERS_INCREMENT = 1;
const DUPLICATE_HEADERS_MERGE = 2;
/**
* Number of the row that contains the column names
*
* @var integer
*/
protected $headerRowNumber;
/**
* CSV file
*
* @var \SplFileObject
*/
protected $file;
/**
* Column headers as read from the CSV file
*
* @var array
*/
protected $columnHeaders = [];
/**
* Number of column headers, stored and re-used for performance
*
* In case of duplicate headers, this is always the number of unmerged headers.
*
* @var integer
*/
protected $headersCount;
/**
* Total number of rows in the CSV file
*
* @var integer
*/
protected $count;
/**
* Faulty CSV rows
*
* @var array
*/
protected $errors = [];
/**
* Strict parsing - skip any lines mismatching header length
*
* @var boolean
*/
protected $strict = true;
/**
* How to handle duplicate headers
*
* @var integer
*/
protected $duplicateHeadersFlag;
/**
* @param \SplFileObject $file
* @param string $delimiter
* @param string $enclosure
* @param string $escape
*/
public function __construct(\SplFileObject $file, $delimiter = ',', $enclosure = '"', $escape = '\\')
{
ini_set('auto_detect_line_endings', true);
$this->file = $file;
$this->file->setFlags(
\SplFileObject::READ_CSV |
\SplFileObject::SKIP_EMPTY |
\SplFileObject::READ_AHEAD |
\SplFileObject::DROP_NEW_LINE
);
$this->file->setCsvControl(
$delimiter,
$enclosure,
$escape
);
}
/**
* Return the current row as an array
*
* If a header row has been set, an associative array will be returned
*
* @return array
*/
public function current()
{
// If the CSV has no column headers just return the line
if (empty($this->columnHeaders)) {
return $this->file->current();
}
// Since the CSV has column headers use them to construct an associative array for the columns in this line
do {
$line = $this->file->current();
// In non-strict mode pad/slice the line to match the column headers
if (!$this->isStrict()) {
if ($this->headersCount > count($line)) {
$line = array_pad($line, $this->headersCount, null); // Line too short
} else {
$line = array_slice($line, 0, $this->headersCount); // Line too long
}
}
// See if values for duplicate headers should be merged
if (self::DUPLICATE_HEADERS_MERGE === $this->duplicateHeadersFlag) {
$line = $this->mergeDuplicates($line);
}
// Count the number of elements in both: they must be equal.
if (count($this->columnHeaders) === count($line)) {
return array_combine(array_keys($this->columnHeaders), $line);
}
// They are not equal, so log the row as error and skip it.
if ($this->valid()) {
$this->errors[$this->key()] = $line;
$this->next();
}
} while($this->valid());
return null;
}
/**
* Get column headers
*
* @return array
*/
public function getColumnHeaders()
{
return array_keys($this->columnHeaders);
}
/**
* Set column headers
*
* @param array $columnHeaders
*/
public function setColumnHeaders(array $columnHeaders)
{
$this->columnHeaders = array_count_values($columnHeaders);
$this->headersCount = count($columnHeaders);
}
/**
* Set header row number
*
* @param integer $rowNumber Number of the row that contains column header names
* @param integer $duplicates How to handle duplicates (optional). One of:
* - CsvReader::DUPLICATE_HEADERS_INCREMENT;
* increments duplicates (dup, dup1, dup2 etc.)
* - CsvReader::DUPLICATE_HEADERS_MERGE; merges
* values for duplicate headers into an array
* (dup => [value1, value2, value3])
*
* @throws DuplicateHeadersException If duplicate headers are encountered
* and no duplicate handling has been
* specified
*/
public function setHeaderRowNumber($rowNumber, $duplicates = null)
{
$this->duplicateHeadersFlag = $duplicates;
$this->headerRowNumber = $rowNumber;
$headers = $this->readHeaderRow($rowNumber);
$this->setColumnHeaders($headers);
}
/**
* Rewind the file pointer
*
* If a header row has been set, the pointer is set just below the header
* row. That way, when you iterate over the rows, that header row is
* skipped.
*/
public function rewind()
{
$this->file->rewind();
if (null !== $this->headerRowNumber) {
$this->file->seek($this->headerRowNumber + 1);
}
}
/**
* {@inheritdoc}
*/
public function count()
{
if (null === $this->count) {
$position = $this->key();
$this->count = iterator_count($this);
$this->seek($position);
}
return $this->count;
}
/**
* {@inheritdoc}
*/
public function next()
{
$this->file->next();
}
/**
* {@inheritdoc}
*/
public function valid()
{
return $this->file->valid();
}
/**
* {@inheritdoc}
*/
public function key()
{
return $this->file->key();
}
/**
* {@inheritdoc}
*/
public function seek($pointer)
{
$this->file->seek($pointer);
}
/**
* {@inheritdoc}
*/
public function getFields()
{
return $this->getColumnHeaders();
}
/**
* Get a row
*
* @param integer $number Row number
*
* @return array
*/
public function getRow($number)
{
$this->seek($number);
return $this->current();
}
/**
* Get rows that have an invalid number of columns
*
* @return array
*/
public function getErrors()
{
if (0 === $this->key()) {
// Iterator has not yet been processed, so do that now
foreach ($this as $row) { /* noop */ }
}
return $this->errors;
}
/**
* Does the reader contain any invalid rows?
*
* @return boolean
*/
public function hasErrors()
{
return count($this->getErrors()) > 0;
}
/**
* Should the reader use strict parsing?
*
* @return boolean
*/
public function isStrict()
{
return $this->strict;
}
/**
* Set strict parsing
*
* @param boolean $strict
*/
public function setStrict($strict)
{
$this->strict = $strict;
}
/**
* Read header row from CSV file
*
* @param integer $rowNumber Row number
*
* @return array
*
* @throws DuplicateHeadersException
*/
protected function readHeaderRow($rowNumber)
{
$this->file->seek($rowNumber);
$headers = $this->file->current();
// Test for duplicate column headers
$diff = array_diff_assoc($headers, array_unique($headers));
if (count($diff) > 0) {
switch ($this->duplicateHeadersFlag) {
case self::DUPLICATE_HEADERS_INCREMENT:
$headers = $this->incrementHeaders($headers);
// Fall through
case self::DUPLICATE_HEADERS_MERGE:
break;
default:
throw new DuplicateHeadersException($diff);
}
}
return $headers;
}
/**
* Add an increment to duplicate headers
*
* So the following line:
* |duplicate|duplicate|duplicate|
* |first |second |third |
*
* Yields value:
* $duplicate => 'first', $duplicate1 => 'second', $duplicate2 => 'third'
*
* @param array $headers
*
* @return array
*/
protected function incrementHeaders(array $headers)
{
$incrementedHeaders = [];
foreach (array_count_values($headers) as $header => $count) {
if ($count > 1) {
$incrementedHeaders[] = $header;
for ($i = 1; $i < $count; $i++) {
$incrementedHeaders[] = $header . $i;
}
} else {
$incrementedHeaders[] = $header;
}
}
return $incrementedHeaders;
}
/**
* Merges values for duplicate headers into an array
*
* So the following line:
* |duplicate|duplicate|duplicate|
* |first |second |third |
*
* Yields value:
* $duplicate => ['first', 'second', 'third']
*
* @param array $line
*
* @return array
*/
protected function mergeDuplicates(array $line)
{
$values = [];
$i = 0;
foreach ($this->columnHeaders as $count) {
if (1 === $count) {
$values[] = $line[$i];
} else {
$values[] = array_slice($line, $i, $count);
}
$i += $count;
}
return $values;
}
}
+225
View File
@@ -0,0 +1,225 @@
<?php
namespace Ddeboer\DataImport\Reader;
use Doctrine\DBAL\Connection;
use Doctrine\DBAL\Statement;
/**
* Reads data through the Doctrine DBAL
*/
class DbalReader implements CountableReader
{
/**
* @var Connection
*/
private $connection;
/**
* @var array
*/
private $data;
/**
* @var Statement
*/
private $stmt;
/**
* @var string
*/
private $sql;
/**
* @var array
*/
private $params;
/**
* @var integer
*/
private $rowCount;
/**
* @var boolean
*/
private $rowCountCalculated = true;
/**
* @var string
*/
private $key;
/**
* @param Connection $connection
* @param string $sql
* @param array $params
*/
public function __construct(Connection $connection, $sql, array $params = [])
{
$this->connection = $connection;
$this->setSql($sql, $params);
}
/**
* Do calculate row count?
*
* @param boolean $calculate
*/
public function setRowCountCalculated($calculate = true)
{
$this->rowCountCalculated = (bool) $calculate;
}
/**
* Is row count calculated?
*
* @return boolean
*/
public function isRowCountCalculated()
{
return $this->rowCountCalculated;
}
/**
* {@inheritdoc}
*/
public function getFields()
{
if (null === $this->data) {
$this->rewind();
}
if (false === $this->data) {
return [];
}
return array_keys((array) $this->data);
}
/**
* Set Query string with Parameters
*
* @param string $sql
* @param array $params
*/
public function setSql($sql, array $params = [])
{
$this->sql = (string) $sql;
$this->setSqlParameters($params);
}
/**
* Set SQL parameters
*
* @param array $params
*/
public function setSqlParameters(array $params)
{
$this->params = $params;
$this->stmt = null;
$this->rowCount = null;
}
/**
* {@inheritdoc}
*/
public function current()
{
if (null === $this->data) {
$this->rewind();
}
return $this->data;
}
/**
* {@inheritdoc}
*/
public function next()
{
$this->key++;
$this->data = $this->stmt->fetch(\PDO::FETCH_ASSOC);
}
/**
* {@inheritdoc}
*/
public function key()
{
return $this->key;
}
/**
* {@inheritdoc}
*/
public function valid()
{
if (null === $this->data) {
$this->rewind();
}
return (false !== $this->data);
}
/**
* {@inheritdoc}
*/
public function rewind()
{
if (null === $this->stmt) {
$this->stmt = $this->prepare($this->sql, $this->params);
}
if (0 !== $this->key) {
$this->stmt->execute();
$this->data = $this->stmt->fetch(\PDO::FETCH_ASSOC);
$this->key = 0;
}
}
/**
* {@inheritdoc}
*/
public function count()
{
if (null === $this->rowCount) {
if ($this->rowCountCalculated) {
$this->doCalcRowCount();
} else {
if (null === $this->stmt) {
$this->rewind();
}
$this->rowCount = $this->stmt->rowCount();
}
}
return $this->rowCount;
}
private function doCalcRowCount()
{
$statement = $this->prepare(sprintf('SELECT COUNT(*) FROM (%s) AS count', $this->sql), $this->params);
$statement->execute();
$this->rowCount = (int) $statement->fetchColumn(0);
}
/**
* Prepare given statement
*
* @param string $sql
* @param array $params
*
* @return Statement
*/
private function prepare($sql, array $params)
{
$statement = $this->connection->prepare($sql);
foreach ($params as $key => $value) {
$statement->bindValue($key, $value);
}
return $statement;
}
}
+108
View File
@@ -0,0 +1,108 @@
<?php
namespace Ddeboer\DataImport\Reader;
use Doctrine\Common\Persistence\ObjectManager;
use Doctrine\ORM\Internal\Hydration\IterableResult;
use Doctrine\ORM\Query;
/**
* Reads entities through the Doctrine ORM
*
* @author David de Boer <david@ddeboer.nl>
*/
class DoctrineReader implements CountableReader
{
/**
* @var ObjectManager
*/
protected $objectManager;
/**
* @var string
*/
protected $objectName;
/**
* @var IterableResult
*/
protected $iterableResult;
/**
* @param ObjectManager $objectManager
* @param string $objectName e.g. YourBundle:YourEntity
*/
public function __construct(ObjectManager $objectManager, $objectName)
{
$this->objectManager = $objectManager;
$this->objectName = $objectName;
}
/**
* {@inheritdoc}
*/
public function getFields()
{
return $this->objectManager->getClassMetadata($this->objectName)
->getFieldNames();
}
/**
* {@inheritdoc}
*/
public function current()
{
return current($this->iterableResult->current());
}
/**
* {@inheritdoc}
*/
public function next()
{
$this->iterableResult->next();
}
/**
* {@inheritdoc}
*/
public function key()
{
return $this->iterableResult->key();
}
/**
* {@inheritdoc}
*/
public function valid()
{
return $this->iterableResult->valid();
}
/**
* {@inheritdoc}
*/
public function rewind()
{
if (!$this->iterableResult) {
$query = $this->objectManager->createQuery(
sprintf('SELECT o FROM %s o', $this->objectName)
);
$this->iterableResult = $query->iterate([], Query::HYDRATE_ARRAY);
}
$this->iterableResult->rewind();
}
/**
* {@inheritdoc}
*/
public function count()
{
$query = $this->objectManager->createQuery(
sprintf('SELECT COUNT(o) FROM %s o', $this->objectName)
);
return $query->getSingleScalarResult();
}
}
+206
View File
@@ -0,0 +1,206 @@
<?php
namespace Ddeboer\DataImport\Reader;
/**
* Reads Excel files with the help of PHPExcel
*
* PHPExcel must be installed.
*
* @author David de Boer <david@ddeboer.nl>
*
* @link http://phpexcel.codeplex.com/
* @link https://github.com/logiQ/PHPExcel
*/
class ExcelReader implements CountableReader, \SeekableIterator
{
/**
* @var array
*/
protected $worksheet;
/**
* @var integer
*/
protected $headerRowNumber;
/**
* @var integer
*/
protected $pointer = 0;
/**
* @var array
*/
protected $columnHeaders;
/**
* Total number of rows
*
* @var integer
*/
protected $count;
/**
* @param \SplFileObject $file Excel file
* @param integer $headerRowNumber Optional number of header row
* @param integer $activeSheet Index of active sheet to read from
* @param boolean $readOnly If set to false, the reader take care of the excel formatting (slow)
*/
public function __construct(\SplFileObject $file, $headerRowNumber = null, $activeSheet = null, $readOnly = true)
{
$reader = \PHPExcel_IOFactory::createReaderForFile($file->getPathName());
$reader->setReadDataOnly($readOnly);
/** @var \PHPExcel $excel */
$excel = $reader->load($file->getPathname());
if (null !== $activeSheet) {
$excel->setActiveSheetIndex($activeSheet);
}
$this->worksheet = $excel->getActiveSheet()->toArray();
if (null !== $headerRowNumber) {
$this->setHeaderRowNumber($headerRowNumber);
}
}
/**
* Return the current row as an array
*
* If a header row has been set, an associative array will be returned
*
* @return array
*/
public function current()
{
$row = $this->worksheet[$this->pointer];
// If the CSV has column headers, use them to construct an associative
// array for the columns in this line
if (!empty($this->columnHeaders)) {
// Count the number of elements in both: they must be equal.
// If not, ignore the row
if (count($this->columnHeaders) == count($row)) {
return array_combine(array_values($this->columnHeaders), $row);
}
} else {
// Else just return the column values
return $row;
}
}
/**
* Get column headers
*
* @return array
*/
public function getColumnHeaders()
{
return $this->columnHeaders;
}
/**
* Set column headers
*
* @param array $columnHeaders
*/
public function setColumnHeaders(array $columnHeaders)
{
$this->columnHeaders = $columnHeaders;
}
/**
* Rewind the file pointer
*
* If a header row has been set, the pointer is set just below the header
* row. That way, when you iterate over the rows, that header row is
* skipped.
*/
public function rewind()
{
if (null === $this->headerRowNumber) {
$this->pointer = 0;
} else {
$this->pointer = $this->headerRowNumber + 1;
}
}
/**
* Set header row number
*
* @param integer $rowNumber Number of the row that contains column header names
*/
public function setHeaderRowNumber($rowNumber)
{
$this->headerRowNumber = $rowNumber;
$this->columnHeaders = $this->worksheet[$rowNumber];
}
/**
* {@inheritdoc}
*/
public function next()
{
$this->pointer++;
}
/**
* {@inheritdoc}
*/
public function valid()
{
return isset($this->worksheet[$this->pointer]);
}
/**
* {@inheritdoc}
*/
public function key()
{
return $this->pointer;
}
/**
* {@inheritdoc}
*/
public function seek($pointer)
{
$this->pointer = $pointer;
}
/**
* {@inheritdoc}
*/
public function count()
{
$count = count($this->worksheet);
if (null !== $this->headerRowNumber) {
$count--;
}
return $count;
}
/**
* {@inheritdoc}
*/
public function getFields()
{
return $this->columnHeaders;
}
/**
* Get a row
*
* @param integer $number
*
* @return array
*/
public function getRow($number)
{
$this->seek($number);
return $this->current();
}
}
@@ -0,0 +1,76 @@
<?php
namespace Ddeboer\DataImport\Reader\Factory;
use Ddeboer\DataImport\Reader\CsvReader;
/**
* Factory that creates CsvReaders
*
* @author David de Boer <david@ddeboer.nl>
*/
class CsvReaderFactory
{
/**
* @var integer
*/
protected $headerRowNumber;
/**
* @var boolean
*/
protected $strict;
/**
* @var string
*/
protected $delimiter;
/**
* @var string
*/
protected $enclosure;
/**
* @var string
*/
protected $escape;
/**
* @param integer $headerRowNumber
* @param boolean $strict
* @param string $delimiter
* @param string $enclosure
* @param string $escape
*/
public function __construct(
$headerRowNumber = null,
$strict = true,
$delimiter = ',',
$enclosure = '"',
$escape = '\\'
) {
$this->headerRowNumber = $headerRowNumber;
$this->strict = $strict;
$this->delimiter = $delimiter;
$this->enclosure = $enclosure;
$this->escape = $escape;
}
/**
* @param \SplFileObject $file
*
* @return CsvReader
*/
public function getReader(\SplFileObject $file)
{
$reader = new CsvReader($file, $this->delimiter, $this->enclosure, $this->escape);
if (null !== $this->headerRowNumber) {
$reader->setHeaderRowNumber($this->headerRowNumber);
}
$reader->setStrict($this->strict);
return $reader;
}
}
@@ -0,0 +1,38 @@
<?php
namespace Ddeboer\DataImport\Reader\Factory;
use Ddeboer\DataImport\Reader\DbalReader;
use Doctrine\DBAL\Connection;
/**
* Factory that creates DbalReaders
*
* @author David de Boer <david@ddeboer.nl>
*/
class DbalReaderFactory
{
/**
* @var Connection
*/
protected $connection;
/**
* @param Connection $connection
*/
public function __construct(Connection $connection)
{
$this->connection = $connection;
}
/**
* @param string $sql
* @param array $params
*
* @return DbalReader
*/
public function getReader($sql, array $params = [])
{
return new DbalReader($this->connection, $sql, $params);
}
}
@@ -0,0 +1,37 @@
<?php
namespace Ddeboer\DataImport\Reader\Factory;
use Ddeboer\DataImport\Reader\DoctrineReader;
use Doctrine\Common\Persistence\ObjectManager;
/**
* Factory that creates DoctrineReaders
*
* @author David de Boer <david@ddeboer.nl>
*/
class DoctrineReaderFactory
{
/**
* @var ObjectManager
*/
protected $objectManager;
/**
* @param ObjectManager $objectManager
*/
public function __construct(ObjectManager $objectManager)
{
$this->objectManager = $objectManager;
}
/**
* @param string $object
*
* @return DoctrineReader
*/
public function getReader($object)
{
return new DoctrineReader($this->objectManager, $object);
}
}
@@ -0,0 +1,43 @@
<?php
namespace Ddeboer\DataImport\Reader\Factory;
use Ddeboer\DataImport\Reader\ExcelReader;
/**
* Factory that creates ExcelReaders
*
* @author David de Boer <david@ddeboer.nl>
*/
class ExcelReaderFactory
{
/**
* @var integer
*/
protected $headerRowNumber;
/**
* @var integer
*/
protected $activeSheet;
/**
* @param integer $headerRowNumber
* @param integer $activeSheet
*/
public function __construct($headerRowNumber = null, $activeSheet = null)
{
$this->headerRowNumber = $headerRowNumber;
$this->activeSheet = $activeSheet;
}
/**
* @param \SplFileObject $file
*
* @return ExcelReader
*/
public function getReader(\SplFileObject $file)
{
return new ExcelReader($file, $this->headerRowNumber, $this->activeSheet);
}
}
@@ -0,0 +1,21 @@
<?php
namespace Ddeboer\DataImport\Reader;
use Ddeboer\DataImport\Reader;
/**
* Use an iterator as a reader
*
* @author Márk Sági-Kazár <mark.sagikazar@gmail.com>
*/
class IteratorReader extends \IteratorIterator implements Reader
{
/**
* {@inheritdoc}
*/
public function getFields()
{
return array_keys($this->current());
}
}
@@ -0,0 +1,183 @@
<?php
namespace Ddeboer\DataImport\Reader;
use Ddeboer\DataImport\Reader;
use Ddeboer\DataImport\Exception\ReaderException;
/**
* Takes multiple readers for processing in the same workflow
*
* @author Adam Paterson <hello@adampaterson.co.uk>
* @author Aydin Hassan <aydin@hotmail.co.uk>
*/
class OneToManyReader implements CountableReader
{
/**
* @var Reader
*/
protected $leftReader;
/**
* @var Reader
*/
protected $rightReader;
/**
* @var string
*/
protected $leftJoinField;
/**
* @var string
*/
protected $rightJoinField;
/**
* Key to nest the rightRows under
*
* @var string
*/
protected $nestKey;
/**
* @param Reader $leftReader
* @param Reader $rightReader
* @param string $nestKey
* @param string $leftJoinField
* @param string $rightJoinField
*/
public function __construct(
Reader $leftReader,
Reader $rightReader,
$nestKey,
$leftJoinField,
$rightJoinField = null
) {
if (is_null($rightJoinField)) {
$rightJoinField = $leftJoinField;
}
$this->leftJoinField = $leftJoinField;
$this->rightJoinField = $rightJoinField;
$this->leftReader = $leftReader;
$this->rightReader = $rightReader;
$this->nestKey = $nestKey;
}
/**
* Create an array of children in the leftRow,
* with the data returned from the right reader
* Where the ID fields Match
*
* @return array
*
* @throws ReaderException
*/
public function current()
{
$leftRow = $this->leftReader->current();
if (array_key_exists($this->nestKey, $leftRow)) {
throw new ReaderException(
sprintf(
'Left Row: "%s" Reader already contains a field named "%s". Please choose a different nest key field',
$this->key(),
$this->nestKey
)
);
}
$leftRow[$this->nestKey] = [];
$leftId = $this->getRowId($leftRow, $this->leftJoinField);
$rightRow = $this->rightReader->current();
$rightId = $this->getRowId($rightRow, $this->rightJoinField);
while ($leftId == $rightId && $this->rightReader->valid()) {
$leftRow[$this->nestKey][] = $rightRow;
$this->rightReader->next();
$rightRow = $this->rightReader->current();
if($this->rightReader->valid()) {
$rightId = $this->getRowId($rightRow, $this->rightJoinField);
}
}
return $leftRow;
}
/**
* @param array $row
* @param string $idField
*
* @return mixed
*
* @throws ReaderException
*/
protected function getRowId(array $row, $idField)
{
if (!array_key_exists($idField, $row)) {
throw new ReaderException(
sprintf(
'Row: "%s" has no field named "%s"',
$this->key(),
$idField
)
);
}
return $row[$idField];
}
/**
* {@inheritdoc}
*/
public function next()
{
$this->leftReader->next();
//right reader is iterated in current() method.
}
/**
* {@inheritdoc}
*/
public function key()
{
return $this->leftReader->key();
}
/**
* {@inheritdoc}
*/
public function valid()
{
return $this->leftReader->valid() && $this->rightReader->valid();
}
/**
* {@inheritdoc}
*/
public function rewind()
{
$this->leftReader->rewind();
$this->rightReader->rewind();
}
/**
* {@inheritdoc}
*/
public function getFields()
{
return array_merge($this->leftReader->getFields(), [$this->nestKey]);
}
/**
* {@inheritdoc}
*/
public function count()
{
return $this->leftReader->count();
}
}
+128
View File
@@ -0,0 +1,128 @@
<?php
namespace Ddeboer\DataImport\Reader;
/**
* Reads data through PDO
*
* @author Robbie Mackay
*/
class PdoReader implements CountableReader
{
/**
* @var \PDO
*/
protected $pdo;
/**
* @var string
*/
protected $tableName;
/**
* @var \PDOStatement
*/
protected $statement;
/**
* @var array
*/
private $data;
/**
* @param \PDO $pdo
* @param string $sql
* @param array $params
*/
public function __construct(\PDO $pdo, $sql, array $params = [])
{
$this->pdo = $pdo;
$this->statement = $this->pdo->prepare($sql);
foreach ($params as $key => $value) {
$this->statement->bindValue($key, $value);
}
}
/**
* {@inheritdoc}
*/
public function getFields()
{
if ($this->statement->execute()) {
// Statement executed successfully
// Grab the first row to find keys
$row = $this->statement->fetch(\PDO::FETCH_ASSOC);
// Return field keys, or empty array no rows remain
return array_keys($row ? $row : []);
} else {
// If the statement errors return empty
return [];
}
}
/**
* {@inheritdoc}
*/
public function current()
{
return current($this->data);
}
/**
* {@inheritdoc}
*/
public function next()
{
next($this->data);
}
/**
* {@inheritdoc}
*/
public function key()
{
return key($this->data);
}
/**
* {@inheritdoc}
*/
public function valid()
{
$key = key($this->data);
return ($key !== null && $key !== false);
}
/**
* {@inheritdoc}
*/
public function rewind()
{
$this->loadData();
reset($this->data);
}
/**
* {@inheritdoc}
*/
public function count()
{
$this->loadData();
return count($this->data);
}
/**
* Load data if it hasn't been loaded yet
*/
protected function loadData()
{
if (null === $this->data) {
$this->statement->execute();
$this->data = $this->statement->fetchAll(\PDO::FETCH_ASSOC);
}
}
}
+146
View File
@@ -0,0 +1,146 @@
<?php
namespace Ddeboer\DataImport;
use Ddeboer\DataImport\Exception\ExceptionInterface;
/**
* Simple Container for Workflow Results
*
* @author Aydin Hassan <aydin@hotmail.co.uk>
*/
class Result
{
/**
* Identifier given to the import/export
*
* @var string
*/
protected $name;
/**
* @var \DateTime
*/
protected $startTime;
/**
* @var \DateTime
*/
protected $endTime;
/**
* @var \DateInterval
*/
protected $elapsed;
/**
* @var integer
*/
protected $errorCount = 0;
/**
* @var integer
*/
protected $successCount = 0;
/**
* @var integer
*/
protected $totalProcessedCount = 0;
/**
* @var \SplObjectStorage
*/
protected $exceptions;
/**
* @param string $name
* @param \DateTime $startTime
* @param \DateTime $endTime
* @param integer $totalCount
* @param \SplObjectStorage $exceptions
*/
public function __construct($name, \DateTime $startTime, \DateTime $endTime, $totalCount, \SplObjectStorage $exceptions)
{
$this->name = $name;
$this->startTime = $startTime;
$this->endTime = $endTime;
$this->elapsed = $startTime->diff($endTime);
$this->totalProcessedCount = $totalCount;
$this->errorCount = count($exceptions);
$this->successCount = $totalCount - $this->errorCount;
$this->exceptions = $exceptions;
}
/**
* @return string
*/
public function getName()
{
return $this->name;
}
/**
* @return \DateTime
*/
public function getStartTime()
{
return $this->startTime;
}
/**
* @return \DateTime
*/
public function getEndTime()
{
return $this->endTime;
}
/**
* @return \DateInterval
*/
public function getElapsed()
{
return $this->elapsed;
}
/**
* @return integer
*/
public function getErrorCount()
{
return $this->errorCount;
}
/**
* @return integer
*/
public function getSuccessCount()
{
return $this->successCount;
}
/**
* @return integer
*/
public function getTotalProcessedCount()
{
return $this->totalProcessedCount;
}
/**
* @return boolean
*/
public function hasErrors()
{
return $this->errorCount > 0;
}
/**
* @return \SplObjectStorage
*/
public function getExceptions()
{
return $this->exceptions;
}
}
+18
View File
@@ -0,0 +1,18 @@
<?php
namespace Ddeboer\DataImport;
/**
* @author Markus Bachmann <markus.bachmann@bachi.biz>
*/
interface Step
{
/**
* Any processing done on each item in the data stack
*
* @param mixed &$item
*
* @return boolean False return value means the item should be skipped
*/
public function process(&$item);
}
+51
View File
@@ -0,0 +1,51 @@
<?php
namespace Ddeboer\DataImport\Step;
use Ddeboer\DataImport\Step;
use Ddeboer\DataImport\Exception\UnexpectedTypeException;
/**
* @author Markus Bachmann <markus.bachmann@bachi.biz>
*/
class ConverterStep implements Step
{
/**
* @var callable[]
*/
private $converters;
/**
* @param array $converters
*/
public function __construct(array $converters = [])
{
foreach ($converters as $converter) {
$this->add($converter);
}
}
/**
* @param callable $converter
*
* @return $this
*/
public function add(callable $converter)
{
$this->converters[] = $converter;
return $this;
}
/**
* {@inheritdoc}
*/
public function process(&$item)
{
foreach ($this->converters as $converter) {
$item = call_user_func($converter, $item);
}
return true;
}
}
+48
View File
@@ -0,0 +1,48 @@
<?php
namespace Ddeboer\DataImport\Step;
use Ddeboer\DataImport\Step;
/**
* @author Markus Bachmann <markus.bachmann@bachi.biz>
*/
class FilterStep implements Step
{
/**
* @var \SplPriorityQueue
*/
private $filters;
public function __construct()
{
$this->filters = new \SplPriorityQueue();
}
/**
* @param callable $filter
* @param integer $priority
*
* @return $this
*/
public function add(callable $filter, $priority = null)
{
$this->filters->insert($filter, $priority);
return $this;
}
/**
* {@inheritdoc}
*/
public function process(&$item)
{
foreach (clone $this->filters as $filter) {
if (false === call_user_func($filter, $item)) {
return false;
}
}
return true;
}
}
+75
View File
@@ -0,0 +1,75 @@
<?php
namespace Ddeboer\DataImport\Step;
use Ddeboer\DataImport\Exception\MappingException;
use Ddeboer\DataImport\Step;
use Symfony\Component\PropertyAccess\Exception\NoSuchPropertyException;
use Symfony\Component\PropertyAccess\Exception\UnexpectedTypeException;
use Symfony\Component\PropertyAccess\PropertyAccessor;
/**
* @author Markus Bachmann <markus.bachmann@bachi.biz>
*/
class MappingStep implements Step
{
/**
* @var array
*/
private $mappings = [];
/**
* @var PropertyAccessor
*/
private $accessor;
/**
* @param array $mappings
* @param PropertyAccessor $accessor
*/
public function __construct(array $mappings = [], PropertyAccessor $accessor = null)
{
$this->mappings = $mappings;
$this->accessor = $accessor ?: new PropertyAccessor();
}
/**
* @param string $from
* @param string $to
*
* @return $this
*/
public function map($from, $to)
{
$this->mappings[$from] = $to;
return $this;
}
/**
* {@inheritdoc}
*
* @throws MappingException
*/
public function process(&$item)
{
try {
foreach ($this->mappings as $from => $to) {
$value = $this->accessor->getValue($item, $from);
$this->accessor->setValue($item, $to, $value);
$from = str_replace(['[',']'], '', $from);
// Check if $item is an array, because properties can't be unset.
// So we don't call unset for objects to prevent side affects.
if (is_array($item) && isset($item[$from])) {
unset($item[$from]);
}
}
} catch (NoSuchPropertyException $exception) {
throw new MappingException('Unable to map item', null, $exception);
} catch (UnexpectedTypeException $exception) {
throw new MappingException('Unable to map item', null, $exception);
}
}
}
+16
View File
@@ -0,0 +1,16 @@
<?php
namespace Ddeboer\DataImport\Step;
use Ddeboer\DataImport\Step;
/**
* @author Markus Bachmann <markus.bachmann@bachi.biz>
*/
interface PriorityStep extends Step
{
/**
* @return integer
*/
public function getPriority();
}
+109
View File
@@ -0,0 +1,109 @@
<?php
namespace Ddeboer\DataImport\Step;
use Ddeboer\DataImport\Exception\ValidationException;
use Symfony\Component\Validator\Constraints;
use Symfony\Component\Validator\Constraint;
use Symfony\Component\Validator\Validator\ValidatorInterface;
/**
* @author Markus Bachmann <markus.bachmann@bachi.biz>
*/
class ValidatorStep implements PriorityStep
{
/**
* @var array
*/
private $constraints = [];
/**
* @var array
*/
private $violations = [];
/**
* @var boolean
*/
private $throwExceptions = false;
/**
* @var integer
*/
private $line = 1;
/**
* @var ValidatorInterface
*/
private $validator;
/**
* @param ValidatorInterface $validator
*/
public function __construct(ValidatorInterface $validator)
{
$this->validator = $validator;
}
/**
* @param string $field
* @param Constraint $constraint
*
* @return $this
*/
public function add($field, Constraint $constraint)
{
if (!isset($this->constraints[$field])) {
$this->constraints[$field] = [];
}
$this->constraints[$field][] = $constraint;
return $this;
}
/**
* @param boolean $flag
*/
public function throwExceptions($flag = true)
{
$this->throwExceptions = $flag;
}
/**
* @return array
*/
public function getViolations()
{
return $this->violations;
}
/**
* {@inheritdoc}
*/
public function process(&$item)
{
$constraints = new Constraints\Collection($this->constraints);
$list = $this->validator->validate($item, $constraints);
if (count($list) > 0) {
$this->violations[$this->line] = $list;
if ($this->throwExceptions) {
throw new ValidationException($list, $this->line);
}
}
$this->line++;
return 0 === count($list);
}
/**
* {@inheritdoc}
*/
public function getPriority()
{
return 128;
}
}
@@ -0,0 +1,48 @@
<?php
namespace Ddeboer\DataImport\Step;
use Ddeboer\DataImport\Step;
use Symfony\Component\PropertyAccess\PropertyAccessor;
/**
* @author Markus Bachmann <markus.bachmann@bachi.biz>
*/
class ValueConverterStep implements Step
{
/**
* @var array
*/
private $converters = [];
/**
* @param string $property
* @param callable $converter
*
* @return $this
*/
public function add($property, callable $converter)
{
$this->converters[$property][] = $converter;
return $this;
}
/**
* {@inheritdoc}
*/
public function process(&$item)
{
$accessor = new PropertyAccessor();
foreach ($this->converters as $property => $converters) {
foreach ($converters as $converter) {
$orgValue = $accessor->getValue($item, $property);
$value = call_user_func($converter, $orgValue);
$accessor->setValue($item,$property,$value);
}
}
return true;
}
}
@@ -0,0 +1,62 @@
<?php
namespace Ddeboer\DataImport\ValueConverter;
/**
* Converts a nested array using a converter-map
*
* @author Christoph Rosse <christoph@rosse.at>
*/
class ArrayValueConverterMap
{
/**
* @var array
*/
private $converters;
/**
* @param callable[] $converters
*/
public function __construct(array $converters)
{
$this->converters = $converters;
}
/**
* {@inheritdoc}
*/
public function __invoke($input)
{
if (!is_array($input)) {
throw new \InvalidArgumentException('Input of a ArrayValueConverterMap must be an array');
}
foreach ($input as $key => $item) {
$input[$key] = $this->convertItem($item);
}
return $input;
}
/**
* Convert an item of the array using the converter-map
*
* @param $item
*
* @return mixed
*/
protected function convertItem($item)
{
foreach ($item as $key => $value) {
if (!isset($this->converters[$key])) {
continue;
}
foreach ($this->converters[$key] as $converter) {
$item[$key] = call_user_func($converter, $item[$key]);
}
}
return $item;
}
}
@@ -0,0 +1,49 @@
<?php
namespace Ddeboer\DataImport\ValueConverter;
use Ddeboer\DataImport\Exception\UnexpectedTypeException;
/**
* Convert a value in a specific charset
*
* @author Markus Bachmann <markus.bachmann@bachi.biz>
*/
class CharsetValueConverter
{
/**
* @var string
*/
private $charset;
/**
* @var string
*/
private $inCharset;
/**
* @param string $charset Charset to convert values to
* @param string $inCharset Charset of input values
*/
public function __construct($charset, $inCharset = 'UTF-8')
{
$this->charset = $charset;
$this->inCharset = $inCharset;
}
/**
* {@inheritdoc}
*/
public function __invoke($input)
{
if (function_exists('mb_convert_encoding')) {
return mb_convert_encoding($input, $this->charset, $this->inCharset);
}
if (function_exists('iconv')) {
return iconv($this->inCharset, $this->charset, $input);
}
throw new \RuntimeException('Could not convert the charset. Please install the mbstring or iconv extension!');
}
}
@@ -0,0 +1,48 @@
<?php
namespace Ddeboer\DataImport\ValueConverter;
use Ddeboer\DataImport\Exception\UnexpectedValueException;
/**
* Convert an date time object into string
*/
class DateTimeToStringValueConverter
{
/**
* Date time format
*
* @var string
* @see http://php.net/manual/en/datetime.createfromformat.php
*/
protected $outputFormat;
/**
* @param string $outputFormat
*/
public function __construct($outputFormat = 'Y-m-d H:i:s')
{
$this->outputFormat = $outputFormat;
}
/**
* Convert string to date time object
* using specified format
*
* @param mixed $input
* @return \DateTime|string
* @throws UnexpectedValueException
*/
public function convert($input)
{
if (!$input) {
return;
}
if (!($input instanceof \DateTime)) {
throw new UnexpectedValueException('Input must be DateTime object.');
}
return $input->format($this->outputFormat);
}
}
@@ -0,0 +1,79 @@
<?php
namespace Ddeboer\DataImport\ValueConverter;
use \Ddeboer\DataImport\Exception\UnexpectedValueException;
/**
* Convert an date string into another date string
* Eg. You want to change the format of a string OR
* If no output specified, return DateTime instance
*
* @author David de Boer <david@ddeboer.nl>
*/
class DateTimeValueConverter
{
/**
* Date time format
*
* @var string
*
* @see http://php.net/manual/en/datetime.createfromformat.php
*/
protected $inputFormat;
/**
* Date time format
*
* @var string
*
* @see http://php.net/manual/en/datetime.createfromformat.php
*/
protected $outputFormat;
/**
* @param string $inputFormat
* @param string $outputFormat
*/
public function __construct($inputFormat = null, $outputFormat = null)
{
$this->inputFormat = $inputFormat;
$this->outputFormat = $outputFormat;
}
/**
* Convert string to date time object then convert back to a string
* using specified format
*
* If no output format specified then return
* the \DateTime instance
*
* @param mixed $input
* @return \DateTime|string
* @throws UnexpectedValueException
*/
public function __invoke($input)
{
if (!$input) {
return;
}
if ($this->inputFormat) {
$date = \DateTime::createFromFormat($this->inputFormat, $input);
if (false === $date) {
throw new UnexpectedValueException(
$input . ' is not a valid date/time according to format ' . $this->inputFormat
);
}
} else {
$date = new \DateTime($input);
}
if ($this->outputFormat) {
return $date->format($this->outputFormat);
}
//if no output format specified we just return the \DateTime instance
return $date;
}
}
@@ -0,0 +1,36 @@
<?php
namespace Ddeboer\DataImport\ValueConverter;
use Ddeboer\DataImport\Exception\UnexpectedValueException;
/**
* @author Grégoire Paris
*/
class MappingValueConverter
{
/**
* @var array
*/
private $mapping = [];
/**
* @param array $mapping
*/
public function __construct(array $mapping)
{
$this->mapping = $mapping;
}
public function __invoke($input)
{
if (!isset($this->mapping[$input])) {
throw new UnexpectedValueException(sprintf(
'Cannot find mapping for value "%s"',
$input
));
}
return $this->mapping[$input];
}
}
@@ -0,0 +1,75 @@
<?php
namespace Ddeboer\DataImport\ValueConverter;
use Ddeboer\DataImport\Exception\UnexpectedTypeException;
use Symfony\Component\PropertyAccess\PropertyPath;
use Symfony\Component\PropertyAccess\PropertyAccess;
use Symfony\Component\PropertyAccess\PropertyAccessor;
/**
* @author Markus Bachmann <markus.bachmann@bachi.biz
*/
class ObjectConverter
{
/**
* @var string|null
*/
protected $propertyPath;
/**
* @var PropertyAccessor
*/
protected $propertyAccessor;
/**
* @param string|null $propertyPath
*/
public function __construct($propertyPath = null)
{
$this->propertyPath = $propertyPath;
$this->propertyAccessor = PropertyAccess::createPropertyAccessor();
}
/**
* Sets the property
*
* @param string $propertyPath
*/
public function setPropertyPath($propertyPath)
{
$this->propertyPath = $propertyPath;
}
/**
* Gets the property
*
* @return null|string
*/
public function getPropertyPath()
{
return $this->propertyPath;
}
/**
* {@inheritdoc}
*/
public function __invoke($input)
{
if (!is_object($input)) {
throw new UnexpectedTypeException($input, 'object');
}
if (null === $this->propertyPath && !method_exists($input, '__toString')) {
throw new \RuntimeException;
}
if (null === $this->propertyPath) {
return (string) $input;
}
$path = new PropertyPath($this->propertyPath);
return $this->propertyAccessor->getValue($input, $path);
}
}
@@ -0,0 +1,43 @@
<?php
namespace Ddeboer\DataImport\ValueConverter;
use Doctrine\Common\Persistence\ObjectRepository;
/**
* Converts a string to an object
*
* @author Markus Bachmann <markus.bachmann@bachi.biz>
*/
class StringToObjectConverter
{
/**
* @var ObjectRepository
*/
private $repository;
/**
* @var string
*/
private $property;
/**
* @param ObjectRepository $repository
* @param string $property
*/
public function __construct(ObjectRepository $repository, $property)
{
$this->repository = $repository;
$this->property = $property;
}
/**
* {@inheritdoc}
*/
public function __invoke($input)
{
$method = 'findOneBy'.ucfirst($this->property);
return $this->repository->$method($input);
}
}
+20
View File
@@ -0,0 +1,20 @@
<?php
namespace Ddeboer\DataImport;
/**
* A mediator between a reader and one or more writers and converters
*
* @author Markus Bachmann <markus.bachmann@bachi.biz>
*/
interface Workflow
{
/**
* Process the whole import workflow
*
* @return Result Object Containing Workflow Results
*
* @throws Exception
*/
public function process();
}
@@ -0,0 +1,196 @@
<?php
namespace Ddeboer\DataImport\Workflow;
use Ddeboer\DataImport\Exception;
use Ddeboer\DataImport\Exception\UnexpectedTypeException;
use Ddeboer\DataImport\Reader;
use Ddeboer\DataImport\Result;
use Ddeboer\DataImport\Step;
use Ddeboer\DataImport\Step\PriorityStep;
use Ddeboer\DataImport\Workflow;
use Ddeboer\DataImport\Writer;
use Psr\Log\LoggerInterface;
use Psr\Log\LoggerAwareInterface;
use Psr\Log\LoggerAwareTrait;
use Psr\Log\NullLogger;
/**
* A mediator between a reader and one or more writers and converters
*
* @author David de Boer <david@ddeboer.nl>
*/
class StepAggregator implements Workflow, LoggerAwareInterface
{
use LoggerAwareTrait;
/**
* @var Reader
*/
private $reader;
/**
* Identifier for the Import/Export
*
* @var string|null
*/
private $name = null;
/**
* @var boolean
*/
private $skipItemOnFailure = false;
/**
* @var \SplPriorityQueue
*/
private $steps;
/**
* @var Writer[]
*/
private $writers = [];
/**
* @var boolean
*/
protected $shouldStop = false;
/**
* @param Reader $reader
* @param string $name
*/
public function __construct(Reader $reader, $name = null)
{
$this->name = $name;
$this->reader = $reader;
// Defaults
$this->logger = new NullLogger();
$this->steps = new \SplPriorityQueue();
}
/**
* Add a step to the current workflow
*
* @param Step $step
* @param integer|null $priority
*
* @return $this
*/
public function addStep(Step $step, $priority = null)
{
$priority = null === $priority && $step instanceof PriorityStep ? $step->getPriority() : $priority;
$priority = null === $priority ? 0 : $priority;
$this->steps->insert($step, $priority);
return $this;
}
/**
* Add a new writer to the current workflow
*
* @param Writer $writer
*
* @return $this
*/
public function addWriter(Writer $writer)
{
array_push($this->writers, $writer);
return $this;
}
/**
* {@inheritdoc}
*/
public function process()
{
$count = 0;
$exceptions = new \SplObjectStorage();
$startTime = new \DateTime;
foreach ($this->writers as $writer) {
$writer->prepare();
}
if (is_callable('pcntl_signal')) {
pcntl_signal(SIGTERM, array($this, 'stop'));
pcntl_signal(SIGINT, array($this, 'stop'));
}
// Read all items
foreach ($this->reader as $index => $item) {
if (is_callable('pcntl_signal_dispatch')) {
pcntl_signal_dispatch();
}
if ($this->shouldStop) {
break;
}
try {
foreach (clone $this->steps as $step) {
if (false === $step->process($item)) {
continue 2;
}
}
if (!is_array($item) && !($item instanceof \ArrayAccess && $item instanceof \Traversable)) {
throw new UnexpectedTypeException($item, 'array');
}
foreach ($this->writers as $writer) {
$writer->writeItem($item);
}
} catch(Exception $e) {
if (!$this->skipItemOnFailure) {
throw $e;
}
$exceptions->attach($e, $index);
$this->logger->error($e->getMessage());
}
$count++;
}
foreach ($this->writers as $writer) {
$writer->finish();
}
return new Result($this->name, $startTime, new \DateTime, $count, $exceptions);
}
/**
* Stops processing and force return Result from process() function
*/
public function stop()
{
$this->shouldStop = true;
}
/**
* Sets the value which determines whether the item should be skipped when error occures
*
* @param boolean $skipItemOnFailure When true skip current item on process exception and log the error
*
* @return $this
*/
public function setSkipItemOnFailure($skipItemOnFailure)
{
$this->skipItemOnFailure = $skipItemOnFailure;
return $this;
}
/**
* @return string
*/
public function getName()
{
return $this->name;
}
}
+28
View File
@@ -0,0 +1,28 @@
<?php
namespace Ddeboer\DataImport;
/**
* Persists data in a storage medium, such as a database, CSV or XML file, etc.
*
* @author David de Boer <david@ddeboer.nl>
*/
interface Writer
{
/**
* Prepare the writer before writing the items
*/
public function prepare();
/**
* Write one data item
*
* @param array $item The data item with converted values
*/
public function writeItem(array $item);
/**
* Wrap up the writer after all items have been written
*/
public function finish();
}
@@ -0,0 +1,105 @@
<?php
namespace Ddeboer\DataImport\Writer;
use Ddeboer\DataImport\Writer;
/**
* Base class to write into streams
*
* @author Benoît Burnichon <bburnichon@gmail.com>
*/
abstract class AbstractStreamWriter implements Writer
{
use WriterTemplate;
/**
* @var resource
*/
private $stream;
/**
* @var boolean
*/
private $closeStreamOnFinish = true;
/**
* @param resource $stream
*/
public function __construct($stream = null)
{
if (null !== $stream) {
$this->setStream($stream);
}
}
/**
* Set the stream resource
*
* @param resource $stream
*
* @throws \InvalidArgumentException
*
* @return $this
*/
public function setStream($stream)
{
if (! is_resource($stream) || ! 'stream' == get_resource_type($stream)) {
throw new \InvalidArgumentException(sprintf(
'Expects argument to be a stream resource, got %s',
is_resource($stream) ? get_resource_type($stream) : gettype($stream)
));
}
$this->stream = $stream;
return $this;
}
/**
* @return resource
*/
public function getStream()
{
if (null === $this->stream) {
$this->setStream(fopen('php://temp', 'rb+'));
$this->setCloseStreamOnFinish(false);
}
return $this->stream;
}
/**
* {@inheritdoc}
*/
public function finish()
{
if (is_resource($this->stream) && $this->getCloseStreamOnFinish()) {
fclose($this->stream);
}
}
/**
* Should underlying stream be closed on finish?
*
* @param boolean $closeStreamOnFinish
*
* @return boolean
*/
public function setCloseStreamOnFinish($closeStreamOnFinish = true)
{
$this->closeStreamOnFinish = (bool) $closeStreamOnFinish;
return $this;
}
/**
* Is Stream closed on finish?
*
* @return boolean
*/
public function getCloseStreamOnFinish()
{
return $this->closeStreamOnFinish;
}
}
+44
View File
@@ -0,0 +1,44 @@
<?php
namespace Ddeboer\DataImport\Writer;
use Ddeboer\DataImport\Writer;
/**
* This class writes an item into an array that was passed by reference
*
* @author David de Boer <david@ddeboer.nl>
*/
class ArrayWriter implements Writer
{
use WriterTemplate;
/**
* @var array
*/
protected $data;
/**
* @param array $data
*/
public function __construct(array &$data)
{
$this->data = &$data;
}
/**
* {@inheritdoc}
*/
public function prepare()
{
$this->data = [];
}
/**
* {@inheritdoc}
*/
public function writeItem(array $item)
{
$this->data[] = $item;
}
}
+58
View File
@@ -0,0 +1,58 @@
<?php
namespace Ddeboer\DataImport\Writer;
use Ddeboer\DataImport\Writer;
/**
* @author Markus Bachmann <markus.bachmann@bachi.biz>
*/
class BatchWriter implements Writer
{
private $delegate;
private $size;
private $queue;
public function __construct(Writer $delegate, $size = 20)
{
$this->delegate = $delegate;
$this->size = $size;
}
public function prepare()
{
$this->delegate->prepare();
$this->queue = new \SplQueue();
$this->queue->setIteratorMode(\SplDoublyLinkedList::IT_MODE_DELETE);
}
public function writeItem(array $item)
{
$this->queue->push($item);
if (count($this->queue) >= $this->size) {
$this->flush();
}
}
public function finish()
{
$this->flush();
$this->delegate->finish();
}
private function flush()
{
foreach ($this->queue as $item) {
$this->delegate->writeItem($item);
}
if ($this->delegate instanceof FlushableWriter) {
$this->delegate->flush();
}
}
}
@@ -0,0 +1,36 @@
<?php
namespace Ddeboer\DataImport\Writer;
use Ddeboer\DataImport\Writer;
/**
* Writes using a callback or closure
*
* @author Markus Bachmann <markus.bachmann@bachi.biz>
*/
class CallbackWriter implements Writer
{
use WriterTemplate;
/**
* @var callable
*/
private $callback;
/**
* @param callable $callback
*/
public function __construct(callable $callback)
{
$this->callback = $callback;
}
/**
* {@inheritdoc}
*/
public function writeItem(array $item)
{
call_user_func($this->callback, $item);
}
}
@@ -0,0 +1,102 @@
<?php
namespace Ddeboer\DataImport\Writer;
use Ddeboer\DataImport\Reader\CountableReader;
use Ddeboer\DataImport\Writer;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Helper\ProgressBar;
/**
* Writes output to the Symfony2 console
*
* @author David de Boer <david@ddeboer.nl>
*/
class ConsoleProgressWriter implements Writer
{
/**
* @var OutputInterface
*/
protected $output;
/**
* @var ProgressBar
*/
protected $progress;
/**
* @var string
*/
protected $verbosity;
/**
* @var CountableReader
*/
protected $reader;
/**
* @var integer
*/
protected $redrawFrequency;
/**
* @param OutputInterface $output
* @param CountableReader $reader
* @param string $verbosity
* @param integer $redrawFrequency
*/
public function __construct(
OutputInterface $output,
CountableReader $reader,
$verbosity = 'debug',
$redrawFrequency = 1
) {
$this->output = $output;
$this->reader = $reader;
$this->verbosity = $verbosity;
$this->redrawFrequency = $redrawFrequency;
}
/**
* {@inheritdoc}
*/
public function prepare()
{
$this->progress = new ProgressBar($this->output, $this->reader->count());
$this->progress->setFormat($this->verbosity);
$this->progress->setRedrawFrequency($this->redrawFrequency);
$this->progress->start();
}
/**
* {@inheritdoc}
*/
public function writeItem(array $item)
{
$this->progress->advance();
}
/**
* {@inheritdoc}
*/
public function finish()
{
$this->progress->finish();
}
/**
* @return string
*/
public function getVerbosity()
{
return $this->verbosity;
}
/**
* @return integer
*/
public function getRedrawFrequency()
{
return $this->redrawFrequency;
}
}
@@ -0,0 +1,70 @@
<?php
namespace Ddeboer\DataImport\Writer;
use Ddeboer\DataImport\Writer;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Helper\Table;
/**
* @author Igor Mukhin <igor.mukhin@gmail.com>
*/
class ConsoleTableWriter implements Writer
{
use WriterTemplate;
/**
* @var OutputInterface
*/
private $output;
/**
* @var Table
*/
private $table;
/**
* @var array
*/
private $firstItem;
/**
* @param OutputInterface $output
* @param Table $table
*/
public function __construct(OutputInterface $output, Table $table) {
$this->output = $output;
$this->table = $table;
}
/**
* {@inheritdoc}
*/
public function writeItem(array $item) {
// Save first item to get keys to display at header
if (is_null($this->firstItem)) {
$this->firstItem = $item;
}
$this->table->addRow($item);
}
/**
* {@inheritdoc}
*/
public function finish() {
$this->table->setHeaders(array_keys($this->firstItem));
$this->table->render();
$this->firstItem = null;
}
/**
* @return Table
*/
public function getTable()
{
return $this->table;
}
}
+72
View File
@@ -0,0 +1,72 @@
<?php
namespace Ddeboer\DataImport\Writer;
/**
* Writes to a CSV file
*
* @author David de Boer <david@ddeboer.nl>
*/
class CsvWriter extends AbstractStreamWriter
{
/**
* @var string
*/
private $delimiter = ';';
/**
* @var string
*/
private $enclosure = '"';
/**
* @var boolean
*/
private $utf8Encoding = false;
private $row = 1;
/**
* @var boolean
*/
protected $prependHeaderRow;
/**
* @param string $delimiter The delimiter
* @param string $enclosure The enclosure
* @param resource $stream
* @param boolean $utf8Encoding
*/
public function __construct($delimiter = ';', $enclosure = '"', $stream = null, $utf8Encoding = false, $prependHeaderRow = false)
{
parent::__construct($stream);
$this->delimiter = $delimiter;
$this->enclosure = $enclosure;
$this->utf8Encoding = $utf8Encoding;
$this->prependHeaderRow = $prependHeaderRow;
}
/**
* {@inheritdoc}
*/
public function prepare()
{
if ($this->utf8Encoding) {
fprintf($this->getStream(), chr(0xEF) . chr(0xBB) . chr(0xBF));
}
}
/**
* {@inheritdoc}
*/
public function writeItem(array $item)
{
if ($this->prependHeaderRow && 1 == $this->row++) {
$headers = array_keys($item);
fputcsv($this->getStream(), $headers, $this->delimiter, $this->enclosure);
}
fputcsv($this->getStream(), $item, $this->delimiter, $this->enclosure);
}
}
+306
View File
@@ -0,0 +1,306 @@
<?php
namespace Ddeboer\DataImport\Writer;
use Ddeboer\DataImport\Writer;
use Doctrine\DBAL\Logging\SQLLogger;
use Doctrine\ORM\EntityManagerInterface;
use Doctrine\ORM\EntityRepository;
use Doctrine\ORM\Mapping\ClassMetadata;
/**
* A bulk Doctrine writer
*
* See also the {@link http://www.doctrine-project.org/docs/orm/2.1/en/reference/batch-processing.html Doctrine documentation}
* on batch processing.
*
* @author David de Boer <david@ddeboer.nl>
*/
class DoctrineWriter implements Writer, FlushableWriter
{
/**
* @var EntityManagerInterface
*/
protected $entityManager;
/**
* @var string
*/
protected $entityName;
/**
* @var EntityRepository
*/
protected $entityRepository;
/**
* @var ClassMetadata
*/
protected $entityMetadata;
/**
* Original Doctrine logger
*
* @var SQLLogger
*/
protected $originalLogger;
/**
* Whether to truncate the table first
*
* @var boolean
*/
protected $truncate = true;
/**
* List of fields used to lookup an entity
*
* @var array
*/
protected $lookupFields = array();
/**
* @param EntityManagerInterface $entityManager
* @param string $entityName
* @param string|array $index Field or fields to find current entities by
*/
public function __construct(EntityManagerInterface $entityManager, $entityName, $index = null)
{
$this->entityManager = $entityManager;
$this->entityRepository = $entityManager->getRepository($entityName);
$this->entityMetadata = $entityManager->getClassMetadata($entityName);
//translate entityName in case a namespace alias is used
$this->entityName = $this->entityMetadata->getName();
if ($index) {
if (is_array($index)) {
$this->lookupFields = $index;
} else {
$this->lookupFields = [$index];
}
}
}
/**
* @return boolean
*/
public function getTruncate()
{
return $this->truncate;
}
/**
* Set whether to truncate the table first
*
* @param boolean $truncate
*
* @return $this
*/
public function setTruncate($truncate)
{
$this->truncate = $truncate;
return $this;
}
/**
* Disable truncation
*
* @return $this
*/
public function disableTruncate()
{
$this->truncate = false;
return $this;
}
/**
* Disable Doctrine logging
*
* @return $this
*/
public function prepare()
{
$this->disableLogging();
if (true === $this->truncate) {
$this->truncateTable();
}
}
/**
* Return a new instance of the entity
*
* @return object
*/
protected function getNewInstance()
{
$className = $this->entityMetadata->getName();
if (class_exists($className) === false) {
throw new \Exception('Unable to create new instance of ' . $className);
}
return new $className;
}
/**
* Call a setter of the entity
*
* @param object $entity
* @param mixed $value
* @param string $setter
*/
protected function setValue($entity, $value, $setter)
{
if (method_exists($entity, $setter)) {
$entity->$setter($value);
}
}
/**
* Re-enable Doctrine logging
*
* @return $this
*/
public function finish()
{
$this->flush();
$this->reEnableLogging();
}
/**
* {@inheritdoc}
*/
public function writeItem(array $item)
{
$entity = $this->findOrCreateItem($item);
$this->loadAssociationObjectsToEntity($item, $entity);
$this->updateEntity($item, $entity);
$this->entityManager->persist($entity);
}
/**
* @param array $item
* @param object $entity
*/
protected function updateEntity(array $item, $entity)
{
$fieldNames = array_merge($this->entityMetadata->getFieldNames(), $this->entityMetadata->getAssociationNames());
foreach ($fieldNames as $fieldName) {
$value = null;
if (isset($item[$fieldName])) {
$value = $item[$fieldName];
} elseif (method_exists($item, 'get' . ucfirst($fieldName))) {
$value = $item->{'get' . ucfirst($fieldName)};
}
if (null === $value) {
continue;
}
if (!($value instanceof \DateTime)
|| $value != $this->entityMetadata->getFieldValue($entity, $fieldName)
) {
$setter = 'set' . ucfirst($fieldName);
$this->setValue($entity, $value, $setter);
}
}
}
/**
* Add the associated objects in case the item have for persist its relation
*
* @param array $item
* @param object $entity
*/
protected function loadAssociationObjectsToEntity(array $item, $entity)
{
foreach ($this->entityMetadata->getAssociationMappings() as $associationMapping) {
$value = null;
if (isset($item[$associationMapping['fieldName']]) && !is_object($item[$associationMapping['fieldName']])) {
$value = $this->entityManager->getReference($associationMapping['targetEntity'], $item[$associationMapping['fieldName']]);
}
if (null === $value) {
continue;
}
$setter = 'set' . ucfirst($associationMapping['fieldName']);
$this->setValue($entity, $value, $setter);
}
}
/**
* Truncate the database table for this writer
*/
protected function truncateTable()
{
$tableName = $this->entityMetadata->table['name'];
$connection = $this->entityManager->getConnection();
$query = $connection->getDatabasePlatform()->getTruncateTableSQL($tableName, true);
$connection->executeQuery($query);
}
/**
* Disable Doctrine logging
*/
protected function disableLogging()
{
$config = $this->entityManager->getConnection()->getConfiguration();
$this->originalLogger = $config->getSQLLogger();
$config->setSQLLogger(null);
}
/**
* Re-enable Doctrine logging
*/
protected function reEnableLogging()
{
$config = $this->entityManager->getConnection()->getConfiguration();
$config->setSQLLogger($this->originalLogger);
}
/**
* Finds existing entity or create a new instance
*
* @param array $item
*/
protected function findOrCreateItem(array $item)
{
$entity = null;
// If the table was not truncated to begin with, find current entity
// first
if (false === $this->truncate) {
if (!empty($this->lookupFields)) {
$lookupConditions = array();
foreach ($this->lookupFields as $fieldName) {
$lookupConditions[$fieldName] = $item[$fieldName];
}
$entity = $this->entityRepository->findOneBy(
$lookupConditions
);
} else {
$entity = $this->entityRepository->find(current($item));
}
}
if (!$entity) {
return $this->getNewInstance();
}
return $entity;
}
/**
* Flush and clear the entity manager
*/
public function flush()
{
$this->entityManager->flush();
$this->entityManager->clear($this->entityName);
}
}
+117
View File
@@ -0,0 +1,117 @@
<?php
namespace Ddeboer\DataImport\Writer;
use Ddeboer\DataImport\Writer;
use PHPExcel;
use PHPExcel_IOFactory;
/**
* Writes to an Excel file
*
* @author David de Boer <david@ddeboer.nl>
*/
class ExcelWriter implements Writer
{
/**
* @var string
*/
protected $filename;
/**
* @var null|string
*/
protected $sheet;
/**
* @var string
*/
protected $type;
/**
* @var boolean
*/
protected $prependHeaderRow;
/**
* @var PHPExcel
*/
protected $excel;
/**
* @var integer
*/
protected $row = 1;
/**
* @param \SplFileObject $file File
* @param string $sheet Sheet title (optional)
* @param string $type Excel file type (defaults to Excel2007)
* @param boolean $prependHeaderRow
*/
public function __construct(\SplFileObject $file, $sheet = null, $type = 'Excel2007', $prependHeaderRow = false)
{
$this->filename = $file->getPathname();
$this->sheet = $sheet;
$this->type = $type;
$this->prependHeaderRow = $prependHeaderRow;
}
/**
* {@inheritdoc}
*/
public function prepare()
{
$reader = PHPExcel_IOFactory::createReader($this->type);
if ($reader->canRead($this->filename)) {
$this->excel = $reader->load($this->filename);
} else {
$this->excel = new PHPExcel();
if(null !== $this->sheet && !$this->excel->sheetNameExists($this->sheet))
{
$this->excel->removeSheetByIndex(0);
}
}
if (null !== $this->sheet) {
if (!$this->excel->sheetNameExists($this->sheet)) {
$this->excel->createSheet()->setTitle($this->sheet);
}
$this->excel->setActiveSheetIndexByName($this->sheet);
}
}
/**
* {@inheritdoc}
*/
public function writeItem(array $item)
{
$count = count($item);
if ($this->prependHeaderRow && 1 == $this->row) {
$headers = array_keys($item);
for ($i = 0; $i < $count; $i++) {
$this->excel->getActiveSheet()->setCellValueByColumnAndRow($i, $this->row, $headers[$i]);
}
$this->row++;
}
$values = array_values($item);
for ($i = 0; $i < $count; $i++) {
$this->excel->getActiveSheet()->setCellValueByColumnAndRow($i, $this->row, $values[$i]);
}
$this->row++;
}
/**
* {@inheritdoc}
*/
public function finish()
{
$writer = \PHPExcel_IOFactory::createWriter($this->excel, $this->type);
$writer->save($this->filename);
}
}
@@ -0,0 +1,8 @@
<?php
namespace Ddeboer\DataImport\Writer;
interface FlushableWriter
{
public function flush();
}
+106
View File
@@ -0,0 +1,106 @@
<?php
namespace Ddeboer\DataImport\Writer;
use Ddeboer\DataImport\Exception\WriterException;
use Ddeboer\DataImport\Writer;
/**
* Write data into a specific database table using a PDO instance.
*
* IMPORTANT: If your PDO instance does not have ERRMODE_EXCEPTION any write failure will be silent or logged to
* stderr only. It is strongly recomended you enable Exceptions with:
*
* $pdo->setAttribute(\PDO::ATTR_ERRMODE, \PDO::ERRMODE_EXCEPTION);
*
* @author Stefan Warman
*/
class PdoWriter implements Writer, FlushableWriter
{
/**
* @var \PDO
*/
protected $pdo;
/**
* @var string
*/
protected $tableName;
/**
* @var \PDOStatement
*/
protected $statement;
/**
* @var array
*/
private $stack;
/**
* Note if your table name is a reserved word for your target DB you should quote it in the appropriate way e.g.
* for MySQL enclose the name in `backticks`.
*
* @param \PDO $pdo
* @param string $tableName
*/
public function __construct(\PDO $pdo, $tableName)
{
$this->pdo = $pdo;
$this->tableName = $tableName;
if (\PDO::ERRMODE_EXCEPTION !== $this->pdo->getAttribute(\PDO::ATTR_ERRMODE)) {
throw new WriterException('Please set the pdo error mode to PDO::ERRMODE_EXCEPTION');
}
}
public function prepare()
{
$this->stack = [];
$this->statement = null;
}
/**
* {@inheritdoc}
*/
public function writeItem(array $item)
{
if (null === $this->statement) {
try {
$this->statement = $this->pdo->prepare(sprintf(
'INSERT INTO %s (%s) VALUES (%s)',
$this->tableName,
implode(',', array_keys($item)),
substr(str_repeat('?,', count($item)), 0, -1)
));
} catch (\PDOException $e) {
throw new WriterException('Failed to send query', null, $e);
}
}
$this->stack[] = array_values($item);
}
public function finish()
{
$this->flush();
return $this;
}
public function flush()
{
$this->pdo->beginTransaction();
try {
foreach ($this->stack as $data) {
$this->statement->execute($data);
}
$this->stack = [];
$this->pdo->commit();
} catch (\PDOException $e) {
throw new WriterException('Failed to write to database', null, $e);
}
}
}
@@ -0,0 +1,140 @@
<?php
namespace Ddeboer\DataImport\Writer;
/**
* Class allowing multiple writers to write in same stream
*
* @author Benoît Burnichon <bburnichon@gmail.com>
*/
class StreamMergeWriter extends AbstractStreamWriter
{
/**
* @var string
*/
private $discriminantField = 'discr';
/**
* @var AbstractStreamWriter[]
*/
private $writers = [];
/**
* Set discriminant field
*
* @param string $discriminantField
*
* @return $this
*/
public function setDiscriminantField($discriminantField)
{
$this->discriminantField = (string) $discriminantField;
return $this;
}
/**
* Get discriminant Field
*
* @return string
*/
public function getDiscriminantField()
{
return $this->discriminantField;
}
/**
* {@inheritdoc}
*/
public function writeItem(array $item)
{
if ((isset($item[$this->discriminantField])
|| array_key_exists($this->discriminantField, $item))
&& $this->hasStreamWriter($key = $item[$this->discriminantField])
) {
$writer = $this->getStreamWriter($key);
$writer->writeItem($item);
}
}
/**
* Set stream writers
*
* @param AbstractStreamWriter[] $writers
*
* @return $this
*/
public function setStreamWriters(array $writers)
{
foreach ($writers as $key => $writer) {
$this->setStreamWriter($key, $writer);
}
return $this;
}
/**
* @param string $key
* @param AbstractStreamWriter $writer
*
* @return $this
*/
public function setStreamWriter($key, AbstractStreamWriter $writer)
{
$writer->setStream($this->getStream());
$writer->setCloseStreamOnFinish(false);
$this->writers[$key] = $writer;
return $this;
}
/**
* Get a previously registered Writer
*
* @param string $key
*
* @return AbstractStreamWriter
*/
public function getStreamWriter($key)
{
return $this->writers[$key];
}
/**
* Get list of registered Writers
*
* @return AbstractStreamWriter[]
*/
public function getStreamWriters()
{
return $this->writers;
}
/**
* Is a writer registered for key?
*
* @param string $key
*
* @return boolean
*/
public function hasStreamWriter($key)
{
return isset($this->writers[$key]);
}
/**
* Set a stream
*
* @param resource $stream
*/
public function setStream($stream)
{
parent::setStream($stream);
foreach ($this->getStreamWriters() as $writer) {
$writer->setStream($stream);
}
return $this;
}
}
@@ -0,0 +1,27 @@
<?php
namespace Ddeboer\DataImport\Writer;
/**
* This template can be overridden in concrete implementations
*
* @author David de Boer <david@ddeboer.nl>
*/
trait WriterTemplate
{
/**
* {@inheritdoc}
*/
public function prepare()
{
}
/**
* {@inheritdoc}
*/
public function finish()
{
}
}