This commit is contained in:
Xes
2025-08-14 22:41:49 +02:00
parent 2de81ccc46
commit 8ce45119b6
39774 changed files with 4309466 additions and 0 deletions

View File

@@ -0,0 +1,13 @@
root = true
[*]
end_of_line = lf
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
indent_style = space
indent_size = 4
[*.yml]
indent_style = space
indent_size = 2

4
vendor/ddeboer/data-import/.gitignore vendored Normal file
View File

@@ -0,0 +1,4 @@
build/
vendor/
composer.lock
phpspec.yml

View File

@@ -0,0 +1,2 @@
tools:
external_code_coverage: true

35
vendor/ddeboer/data-import/.travis.yml vendored Normal file
View File

@@ -0,0 +1,35 @@
language: php
sudo: false
php:
- 5.4
- 5.5
- 5.6
- 7.0
- hhvm
matrix:
allow_failures:
- php: 7.0
- php: hhvm
include:
- php: 5.4
env:
- COMPOSER_FLAGS="--prefer-stable --prefer-lowest"
- PHPUNIT_FLAGS="--coverage-text --coverage-clover=coverage.clover"
- COVERAGE=true
cache:
directories:
- $HOME/.composer/cache
install:
- if [[ "$TRAVIS_PHP_VERSION" == "hhvm" ]]; then composer require mongofill/mongofill=dev-master --no-update; fi
- travis_retry composer update --no-progress --no-plugins ${COMPOSER_FLAGS}
script: phpunit ${PHPUNIT_FLAGS}
after_script:
- if [[ "$COVERAGE" = true ]]; then wget https://scrutinizer-ci.com/ocular.phar; fi
- if [[ "$COVERAGE" = true ]]; then php ocular.phar code-coverage:upload --format=php-clover coverage.clover; fi

19
vendor/ddeboer/data-import/LICENSE vendored Normal file
View File

@@ -0,0 +1,19 @@
Copyright (C) 2012-2014 David de Boer <david@ddeboer.nl>
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

1291
vendor/ddeboer/data-import/README.md vendored Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,52 @@
{
"name": "ddeboer/data-import",
"description": "Import data from, and export data to, a range of file formats and media",
"keywords": [ "data", "import", "export", "csv", "excel", "doctrine" ],
"license": "MIT",
"authors": [
{
"name": "David de Boer",
"email": "david@ddeboer.nl"
},
{
"name": "The community",
"homepage": "https://github.com/ddeboer/data-import/graphs/contributors"
}
],
"support": {
"issues": "https://github.com/ddeboer/data-import/issues",
"source": "https://github.com/ddeboer/data-import"
},
"require": {
"php": ">=5.4.0",
"psr/log": "~1.0"
},
"require-dev": {
"ext-iconv": "*",
"ext-mbstring": "*",
"ext-sqlite3": "*",
"symfony/property-access": "~2.5",
"phpoffice/phpexcel": "*",
"doctrine/dbal": "~2.4",
"doctrine/orm": "~2.4",
"symfony/console": "~2.8|~3.0",
"symfony/validator": "~2.8|~3.0",
"phpspec/phpspec": "~2.1",
"henrikbjorn/phpspec-code-coverage" : "~1.0"
},
"suggest": {
"ext-iconv": "For the CharsetValueConverter",
"ext-mbstring": "For the CharsetValueConverter",
"symfony/validator": "to use the ValidatorFilter",
"doctrine/dbal": "If you want to use the DbalReader",
"phpoffice/phpexcel": "If you want to use the ExcelReader",
"symfony/console": "If you want to use the ConsoleProgressWriter",
"symfony/property-access": "If you want to use the ObjectConverter"
},
"autoload": {
"psr-4": {
"Ddeboer\\DataImport\\": "src/",
"Ddeboer\\DataImport\\Tests\\": "tests/"
}
}
}

View File

@@ -0,0 +1,10 @@
suites:
data_import_suite:
namespace: Ddeboer\DataImport
psr4_prefix: Ddeboer\DataImport
extensions:
- PhpSpec\Extension\CodeCoverageExtension
formatter.name: pretty
code_coverage:
format: clover
output: build/coverage.xml

View File

@@ -0,0 +1,5 @@
suites:
data_import_suite:
namespace: Ddeboer\DataImport
psr4_prefix: Ddeboer\DataImport
formatter.name: pretty

View File

@@ -0,0 +1,18 @@
<?xml version="1.0" encoding="UTF-8"?>
<phpunit bootstrap="./vendor/autoload.php" colors="true" convertErrorsToExceptions="true">
<testsuites>
<testsuite name="ddeboer/data-import">
<directory suffix="Test.php">./tests</directory>
</testsuite>
</testsuites>
<filter>
<whitelist>
<directory suffix=".php">./src</directory>
<exclude>
<directory suffix="Interface.php">./src</directory>
</exclude>
</whitelist>
</filter>
</phpunit>

View File

@@ -0,0 +1,28 @@
<?php
namespace spec\Ddeboer\DataImport\Exception;
use PhpSpec\ObjectBehavior;
class DuplicateHeadersExceptionSpec extends ObjectBehavior
{
function let()
{
$this->beConstructedWith(['header1', 'header2']);
}
function it_is_initializable()
{
$this->shouldHaveType('Ddeboer\DataImport\Exception\DuplicateHeadersException');
}
function it_is_a_reader_exception()
{
$this->shouldHaveType('Ddeboer\DataImport\Exception\ReaderException');
}
function it_has_a_message()
{
$this->getMessage()->shouldReturn('File contains duplicate headers: header1, header2');
}
}

View File

@@ -0,0 +1,19 @@
<?php
namespace spec\Ddeboer\DataImport\Exception;
use PhpSpec\ObjectBehavior;
class MappingExceptionSpec extends ObjectBehavior
{
function it_is_initializable()
{
$this->shouldHaveType('Ddeboer\DataImport\Exception\MappingException');
}
function it_is_an_exception()
{
$this->shouldHaveType('Exception');
$this->shouldImplement('Ddeboer\DataImport\Exception');
}
}

View File

@@ -0,0 +1,19 @@
<?php
namespace spec\Ddeboer\DataImport\Exception;
use PhpSpec\ObjectBehavior;
class ReaderExceptionSpec extends ObjectBehavior
{
function it_is_initializable()
{
$this->shouldHaveType('Ddeboer\DataImport\Exception\ReaderException');
}
function it_is_an_exception()
{
$this->shouldHaveType('Exception');
$this->shouldImplement('Ddeboer\DataImport\Exception');
}
}

View File

@@ -0,0 +1,19 @@
<?php
namespace spec\Ddeboer\DataImport\Exception;
use PhpSpec\ObjectBehavior;
class SourceNotFoundExceptionSpec extends ObjectBehavior
{
function it_is_initializable()
{
$this->shouldHaveType('Ddeboer\DataImport\Exception\SourceNotFoundException');
}
function it_is_an_exception()
{
$this->shouldHaveType('Exception');
$this->shouldImplement('Ddeboer\DataImport\Exception');
}
}

View File

@@ -0,0 +1,35 @@
<?php
namespace spec\Ddeboer\DataImport\Exception;
use PhpSpec\ObjectBehavior;
class UnexpectedTypeExceptionSpec extends ObjectBehavior
{
public function let()
{
$this->beConstructedWith(123, 'string');
}
function it_is_initializable()
{
$this->shouldHaveType('Ddeboer\DataImport\Exception\UnexpectedTypeException');
}
function it_is_an_unexpected_value_exception()
{
$this->shouldHaveType('Ddeboer\DataImport\Exception\UnexpectedValueException');
}
function it_has_a_message_with_scalar_type()
{
$this->getMessage()->shouldReturn('Expected argument of type "string", "integer" given');
}
function it_has_a_message_with_object_type()
{
$this->beConstructedWith(new \stdClass, 'string');
$this->getMessage()->shouldReturn('Expected argument of type "string", "stdClass" given');
}
}

View File

@@ -0,0 +1,18 @@
<?php
namespace spec\Ddeboer\DataImport\Exception;
use PhpSpec\ObjectBehavior;
class UnexpectedValueExceptionSpec extends ObjectBehavior
{
function it_is_initializable()
{
$this->shouldHaveType('Ddeboer\DataImport\Exception\UnexpectedValueException');
}
function it_is_an_exception()
{
$this->shouldImplement('Ddeboer\DataImport\Exception');
}
}

View File

@@ -0,0 +1,35 @@
<?php
namespace spec\Ddeboer\DataImport\Exception;
use PhpSpec\ObjectBehavior;
use Symfony\Component\Validator\ConstraintViolationListInterface;
class ValidationExceptionSpec extends ObjectBehavior
{
function let(ConstraintViolationListInterface $list)
{
$this->beConstructedWith($list, 1);
}
function it_is_initializable()
{
$this->shouldHaveType('Ddeboer\DataImport\Exception\ValidationException');
}
function it_is_an_exception()
{
$this->shouldHaveType('Exception');
$this->shouldImplement('Ddeboer\DataImport\Exception');
}
function it_has_a_list_of_violations(ConstraintViolationListInterface $list)
{
$this->getViolations()->shouldReturn($list);
}
function it_has_a_line_number()
{
$this->getLineNumber()->shouldReturn(1);
}
}

View File

@@ -0,0 +1,19 @@
<?php
namespace spec\Ddeboer\DataImport\Exception;
use PhpSpec\ObjectBehavior;
class WriterExceptionSpec extends ObjectBehavior
{
function it_is_initializable()
{
$this->shouldHaveType('Ddeboer\DataImport\Exception\WriterException');
}
function it_is_an_exception()
{
$this->shouldHaveType('Exception');
$this->shouldImplement('Ddeboer\DataImport\Exception');
}
}

View File

@@ -0,0 +1,41 @@
<?php
namespace spec\Ddeboer\DataImport\Filter;
use Ddeboer\DataImport\ValueConverter\DateTimeValueConverter;
use PhpSpec\ObjectBehavior;
class DateTimeThresholdFilterSpec extends ObjectBehavior
{
function let(DateTimeValueConverter $valueConverter)
{
$this->beConstructedWith($valueConverter);
}
function it_is_initializable()
{
$this->shouldHaveType('Ddeboer\DataImport\Filter\DateTimeThresholdFilter');
}
function it_throws_an_exception_when_no_threshold_is_set()
{
$this->shouldThrow('LogicException')->during__invoke([]);
}
function it_accepts_a_threshold(\DateTime $dateTime)
{
$this->setThreshold($dateTime);
}
function it_filters_an_item_based_on_a_time_column(DateTimeValueConverter $valueConverter)
{
$item = [
'updated_at' => '1970-01-01'
];
$valueConverter->__invoke($item['updated_at'])->willReturn(new \DateTime('1970-01-01'));
$this->beConstructedWith($valueConverter, new \DateTime());
$this->__invoke($item)->shouldReturn(false);
}
}

View File

@@ -0,0 +1,36 @@
<?php
namespace spec\Ddeboer\DataImport\Filter;
use PhpSpec\ObjectBehavior;
use Prophecy\Argument;
class OffsetFilterSpec extends ObjectBehavior
{
function it_is_initializable()
{
$this->shouldHaveType('Ddeboer\DataImport\Filter\OffsetFilter');
}
function it_does_not_limit_by_default()
{
$this->__invoke(['content'])->shouldReturn(true);
}
function it_limits_until_the_start_offset()
{
$this->beConstructedWith(1);
$this->__invoke(['content'])->shouldReturn(false);
$this->__invoke(['content'])->shouldReturn(true);
}
function it_limits_when_max_is_reached()
{
$this->beConstructedWith(0, 2);
$this->__invoke(['content'])->shouldReturn(true);
$this->__invoke(['content'])->shouldReturn(true);
$this->__invoke(['content'])->shouldReturn(false);
}
}

View File

@@ -0,0 +1,82 @@
<?php
namespace spec\Ddeboer\DataImport\Filter;
use Symfony\Component\Validator\Constraint;
use Symfony\Component\Validator\ConstraintViolationList;
use Symfony\Component\Validator\Validator\ValidatorInterface;
use PhpSpec\ObjectBehavior;
use Prophecy\Argument;
class ValidatorFilterSpec extends ObjectBehavior
{
/**
* @var array
*/
protected $item1 = ['key1' => 'value1'];
/**
* @var array
*/
protected $item2 = [
'key1' => 'value1',
'key2' => 'value2'
];
function let(ValidatorInterface $validator)
{
$this->beConstructedWith($validator);
}
function it_is_initializable()
{
$this->shouldHaveType('Ddeboer\DataImport\Filter\ValidatorFilter');
}
function it_validates_an_item(ValidatorInterface $validator, Constraint $constraint, ConstraintViolationList $list)
{
$list->count()->willReturn(0);
$validator->validate($this->item1, Argument::type('Symfony\Component\Validator\Constraints\Collection'))->willReturn($list);
$this->add('key1', $constraint);
$this->__invoke($this->item1);
}
function it_validates_an_item_non_strictly(ValidatorInterface $validator, Constraint $constraint, ConstraintViolationList $list)
{
$list->count()->willReturn(0);
$validator->validate($this->item1, Argument::type('Symfony\Component\Validator\Constraints\Collection'))->willReturn($list);
$this->setStrict(false);
$this->add('key1', $constraint);
$this->__invoke($this->item1);
$this->__invoke($this->item2);
}
function it_validates_an_item_and_the_validation_fails(ValidatorInterface $validator, Constraint $constraint, ConstraintViolationList $list)
{
$list->count()->willReturn(1);
$validator->validate($this->item1, Argument::type('Symfony\Component\Validator\Constraints\Collection'))->willReturn($list);
$this->add('key1', $constraint);
$this->__invoke($this->item1);
$this->getViolations()->shouldReturn([1 => $list]);
}
function it_validates_an_item_and_the_validation_fails_with_exception(ValidatorInterface $validator, Constraint $constraint, ConstraintViolationList $list)
{
$list->count()->willReturn(1);
$validator->validate($this->item1, Argument::type('Symfony\Component\Validator\Constraints\Collection'))->willReturn($list);
$this->throwExceptions(true);
$this->add('key1', $constraint);
$this->shouldThrow('Ddeboer\DataImport\Exception\ValidationException')->during__invoke($this->item1);
$this->getViolations()->shouldReturn([1 => $list]);
}
}

View File

@@ -0,0 +1,20 @@
<?php
namespace spec\Ddeboer\DataImport\Reader\Factory;
use PhpSpec\ObjectBehavior;
class CsvReaderFactorySpec extends ObjectBehavior
{
function it_is_initializable()
{
$this->shouldHaveType('Ddeboer\DataImport\Reader\Factory\CsvReaderFactory');
}
function it_creates_a_reader()
{
$file = new \SplFileObject(tempnam(sys_get_temp_dir(), null));
$this->getReader($file)->shouldHaveType('Ddeboer\DataImport\Reader\CsvReader');
}
}

View File

@@ -0,0 +1,24 @@
<?php
namespace spec\Ddeboer\DataImport\Reader\Factory;
use Doctrine\DBAL\Connection;
use PhpSpec\ObjectBehavior;
class DbalReaderFactorySpec extends ObjectBehavior
{
function let(Connection $dbal)
{
$this->beConstructedWith($dbal);
}
function it_is_initializable()
{
$this->shouldHaveType('Ddeboer\DataImport\Reader\Factory\DbalReaderFactory');
}
function it_creates_a_reader()
{
$this->getReader('SQL', [])->shouldHaveType('Ddeboer\DataImport\Reader\DbalReader');
}
}

View File

@@ -0,0 +1,24 @@
<?php
namespace spec\Ddeboer\DataImport\Reader\Factory;
use Doctrine\Common\Persistence\ObjectManager;
use PhpSpec\ObjectBehavior;
class DoctrineReaderFactorySpec extends ObjectBehavior
{
function let(ObjectManager $objectManager)
{
$this->beConstructedWith($objectManager);
}
function it_is_initializable()
{
$this->shouldHaveType('Ddeboer\DataImport\Reader\Factory\DoctrineReaderFactory');
}
function it_creates_a_reader()
{
$this->getReader('Entity')->shouldHaveType('Ddeboer\DataImport\Reader\DoctrineReader');
}
}

View File

@@ -0,0 +1,20 @@
<?php
namespace spec\Ddeboer\DataImport\Reader\Factory;
use PhpSpec\ObjectBehavior;
class ExcelReaderFactorySpec extends ObjectBehavior
{
function it_is_initializable()
{
$this->shouldHaveType('Ddeboer\DataImport\Reader\Factory\ExcelReaderFactory');
}
function it_creates_a_reader()
{
$file = new \SplFileObject(tempnam(sys_get_temp_dir(), null));
$this->getReader($file)->shouldHaveType('Ddeboer\DataImport\Reader\ExcelReader');
}
}

View File

@@ -0,0 +1,66 @@
<?php
namespace spec\Ddeboer\DataImport;
use PhpSpec\ObjectBehavior;
use Prophecy\Argument;
class ResultSpec extends ObjectBehavior
{
function let(\DateTime $startTime, \DateTime $endTime, \SplObjectStorage $exceptions, \DateInterval $elapsed)
{
$startTime->diff($endTime)->willReturn($elapsed);
$exceptions->count()->willReturn(4);
$this->beConstructedWith('name', $startTime, $endTime, 10, $exceptions);
}
function it_is_initializable()
{
$this->shouldHaveType('Ddeboer\DataImport\Result');
}
function it_has_a_name()
{
$this->getName()->shouldReturn('name');
}
function it_has_a_start_time(\DateTime $startTime)
{
$this->getStartTime()->shouldReturn($startTime);
}
function it_has_a_end_time(\DateTime $endTime)
{
$this->getEndTime()->shouldReturn($endTime);
}
function it_has_an_elapsed_time(\DateInterval $elapsed)
{
$this->getElapsed()->shouldReturn($elapsed);
}
function it_has_an_error_count()
{
$this->getErrorCount()->shouldReturn(4);
}
function it_has_a_success_count()
{
$this->getSuccessCount()->shouldReturn(6);
}
function it_has_a_total_processed_count()
{
$this->getTotalProcessedCount()->shouldReturn(10);
}
function it_checks_if_it_has_errors()
{
$this->hasErrors()->shouldReturn(true);
}
function it_has_exceptions(\SplObjectStorage $exceptions)
{
$this->getExceptions()->shouldReturn($exceptions);
}
}

View File

@@ -0,0 +1,13 @@
<?php
namespace Ddeboer\DataImport;
/**
* Base exception
*
* @author David de Boer <david@ddeboer.nl>
*/
interface Exception
{
}

View File

@@ -0,0 +1,17 @@
<?php
namespace Ddeboer\DataImport\Exception;
/**
* @author David de Boer <david@ddeboer.nl>
*/
class DuplicateHeadersException extends ReaderException
{
/**
* @param array $duplicates
*/
public function __construct(array $duplicates)
{
parent::__construct(sprintf('File contains duplicate headers: %s', implode($duplicates, ', ')));
}
}

View File

@@ -0,0 +1,15 @@
<?php
namespace Ddeboer\DataImport\Exception;
use Ddeboer\DataImport\Exception;
/**
* Description of MappingException
*
* @author gnat
*/
class MappingException extends \Exception implements Exception
{
}

View File

@@ -0,0 +1,11 @@
<?php
namespace Ddeboer\DataImport\Exception;
/**
* @author David de Boer <david@ddeboer.nl>
*/
class ReaderException extends UnexpectedValueException
{
}

View File

@@ -0,0 +1,13 @@
<?php
namespace Ddeboer\DataImport\Exception;
use Ddeboer\DataImport\Exception;
/**
* @author Markus Bachmann <markus.bachmann@bachi.biz>
*/
class SourceNotFoundException extends \Exception implements Exception
{
}

View File

@@ -0,0 +1,22 @@
<?php
namespace Ddeboer\DataImport\Exception;
/**
* @author David de Boer <david@ddeboer.nl>
*/
class UnexpectedTypeException extends UnexpectedValueException
{
/**
* @param mixed $value
* @param string $expectedType
*/
public function __construct($value, $expectedType)
{
parent::__construct(sprintf(
'Expected argument of type "%s", "%s" given',
$expectedType,
is_object($value) ? get_class($value) : gettype($value)
));
}
}

View File

@@ -0,0 +1,14 @@
<?php
namespace Ddeboer\DataImport\Exception;
use Ddeboer\DataImport\Exception;
/**
* @author David de Boer <david@ddeboer.nl>
*/
class UnexpectedValueException extends \UnexpectedValueException implements Exception
{
}

View File

@@ -0,0 +1,48 @@
<?php
namespace Ddeboer\DataImport\Exception;
use Ddeboer\DataImport\Exception;
use Symfony\Component\Validator\ConstraintViolationListInterface;
/**
* @author Markus Bachmann <markus.bachmann@bachi.biz>
*/
class ValidationException extends \Exception implements Exception
{
/**
* @var ConstraintViolationListInterface
*/
private $violations;
/**
* @var integer
*/
private $lineNumber;
/**
* @param ConstraintViolationListInterface $list
* @param integer $line
*/
public function __construct(ConstraintViolationListInterface $list, $line)
{
$this->violations = $list;
$this->lineNumber = $line;
}
/**
* @return ConstraintViolationListInterface
*/
public function getViolations()
{
return $this->violations;
}
/**
* @return integer
*/
public function getLineNumber()
{
return $this->lineNumber;
}
}

View File

@@ -0,0 +1,13 @@
<?php
namespace Ddeboer\DataImport\Exception;
use Ddeboer\DataImport\Exception;
/**
* @author David de Boer <david@ddeboer.nl>
*/
class WriterException extends \Exception implements Exception
{
}

View File

@@ -0,0 +1,84 @@
<?php
namespace Ddeboer\DataImport\Filter;
use Ddeboer\DataImport\ValueConverter\DateTimeValueConverter;
/**
* This filter can be used to filter out some items from a specific date.
*
* Useful to do incremental imports
*
* @author Grégoire Paris
*/
class DateTimeThresholdFilter
{
/**
* Threshold dates strictly before this date will be filtered out
*
* @var \DateTime|null
*/
protected $threshold;
/**
* Used to convert the values in the time column
*
* @var DateTimeValueConverter
*/
protected $valueConverter;
/**
* The name of the column that should contain the value the filter will compare the threshold with
*
* @var string
*/
protected $timeColumnName = 'updated_at';
/**
* @var integer
*/
protected $priority = 512;
/**
* @param DateTimeValueConverter $valueConverter
* @param \DateTime|null $threshold
* @param string $timeColumnName
* @param integer $priority
*/
public function __construct(
DateTimeValueConverter $valueConverter,
\DateTime $threshold = null,
$timeColumnName = 'updated_at',
$priority = 512
) {
$this->valueConverter = $valueConverter;
$this->threshold = $threshold;
$this->timeColumnName = $timeColumnName;
$this->priority = $priority;
}
/**
* {@inheritdoc}
*/
public function __invoke(array $item)
{
if ($this->threshold == null) {
throw new \LogicException('Make sure you set a threshold');
}
$threshold = call_user_func($this->valueConverter, $item[$this->timeColumnName]);
return $threshold >= $this->threshold;
}
/**
* Useful if you build a filter service, and want to set the threshold
* dynamically afterwards.
*
* @param \DateTime $value
*/
public function setThreshold(\DateTime $value)
{
$this->threshold = $value;
}
}

View File

@@ -0,0 +1,82 @@
<?php
namespace Ddeboer\DataImport\Filter;
/**
* This filter can be used to filter out some items from the beginning and/or
* end of the items.
*
* @author Ville Mattila <ville@eventio.fi>
*/
class OffsetFilter
{
/**
* @var integer
*/
protected $offset = 0;
/**
* @var integer|null
*/
protected $limit = null;
/**
* @var integer
*/
protected $offsetCount = 0;
/**
* @var integer
*/
protected $sliceCount = 0;
/**
* @var boolean
*/
protected $maxLimitHit = false;
/**
* @param integer $offset 0-based index of the item to start read from
* @param integer|null $limit Maximum count of items to read. null = no limit
*/
public function __construct($offset = 0, $limit = null)
{
$this->offset = $offset;
$this->limit = $limit;
}
/**
* {@inheritdoc}
*/
public function __invoke(array $item)
{
// In case we've already filtered up to limited
if ($this->maxLimitHit) {
return false;
}
$this->offsetCount++;
// We have not reached the start offset
if ($this->offsetCount < $this->offset + 1) {
return false;
}
// There is no maximum limit, so we'll return always true
if (null === $this->limit) {
return true;
}
$this->sliceCount++;
if ($this->sliceCount < $this->limit) {
return true;
} elseif ($this->sliceCount == $this->limit) {
$this->maxLimitHit = true;
return true;
}
return false;
}
}

View File

@@ -0,0 +1,117 @@
<?php
namespace Ddeboer\DataImport\Filter;
use Symfony\Component\Validator\Validator\ValidatorInterface;
use Symfony\Component\Validator\Constraint;
use Symfony\Component\Validator\Constraints;
use Ddeboer\DataImport\Exception\ValidationException;
/**
* @author Markus Bachmann <markus.bachmann@bachi.biz>
*/
class ValidatorFilter
{
/**
* @var ValidatorInterface
*/
private $validator;
/**
* @var boolean
*/
private $throwExceptions = false;
/**
* @var integer
*/
private $line = 1;
/**
* @var boolean
*/
private $strict = true;
/**
* @var array
*/
private $constraints = [];
/**
* @var array
*/
private $violations = [];
/**
* @param ValidatorInterface $validator
*/
public function __construct(ValidatorInterface $validator)
{
$this->validator = $validator;
}
/**
* @param string $field
* @param Constraint $constraint
*/
public function add($field, Constraint $constraint)
{
if (!isset($this->constraints[$field])) {
$this->constraints[$field] = [];
}
$this->constraints[$field][] = $constraint;
}
/**
* @param boolean $flag
*/
public function throwExceptions($flag = true)
{
$this->throwExceptions = $flag;
}
/**
* @param boolean $strict
*/
public function setStrict($strict)
{
$this->strict = $strict;
}
/**
* @return array
*/
public function getViolations()
{
return $this->violations;
}
/**
* @param array $item
*
* @return boolean
*/
public function __invoke(array $item)
{
if (!$this->strict) {
// Only validate properties which have an constaint.
$temp = array_intersect(array_keys($item), array_keys($this->constraints));
$item = array_intersect_key($item, array_flip($temp));
}
$constraints = new Constraints\Collection($this->constraints);
$list = $this->validator->validate($item, $constraints);
$currentLine = $this->line++;
if (count($list) > 0) {
$this->violations[$currentLine] = $list;
if ($this->throwExceptions) {
throw new ValidationException($list, $currentLine);
}
}
return 0 === count($list);
}
}

View File

@@ -0,0 +1,18 @@
<?php
namespace Ddeboer\DataImport;
/**
* Iterator that reads data to be imported
*
* @author David de Boer <david@ddeboer.nl>
*/
interface Reader extends \Iterator
{
/**
* Get the field (column, property) names
*
* @return array
*/
public function getFields();
}

View File

@@ -0,0 +1,24 @@
<?php
namespace Ddeboer\DataImport\Reader;
/**
* Reads an array
*
* @author David de Boer <david@ddeboer.nl>
*/
class ArrayReader extends \ArrayIterator implements CountableReader
{
/**
* {@inheritdoc}
*/
public function getFields()
{
// Examine first row
if ($this->count() > 0) {
return array_keys($this[0]);
}
return [];
}
}

View File

@@ -0,0 +1,31 @@
<?php
namespace Ddeboer\DataImport\Reader;
/**
* Use a class implementing both \Iterator and \Countable as a reader
*
* This class uses count() on iterators implementing \Countable interface
* and iterator_count in any further cases
*
* Be careful! iterator_count iterates through the whole iterator loading every data into the memory (for example from streams)
* It is not recommended for very big datasets.
*
* @author Márk Sági-Kazár <mark.sagikazar@gmail.com>
*/
class CountableIteratorReader extends IteratorReader implements CountableReader
{
/**
* {@inheritdoc}
*/
public function count()
{
$iterator = $this->getInnerIterator();
if ($iterator instanceof \Countable) {
return count($iterator);
}
return iterator_count($iterator);
}
}

View File

@@ -0,0 +1,15 @@
<?php
namespace Ddeboer\DataImport\Reader;
use Ddeboer\DataImport\Reader;
/**
* Reader that provides the count of total items
*
* @author David de Boer <david@ddeboer.nl>
*/
interface CountableReader extends Reader, \Countable
{
// Don't add count() to interface: see https://github.com/ddeboer/data-import/pull/5
}

View File

@@ -0,0 +1,414 @@
<?php
namespace Ddeboer\DataImport\Reader;
use Ddeboer\DataImport\Exception\DuplicateHeadersException;
/**
* Reads a CSV file, using as little memory as possible
*
* @author David de Boer <david@ddeboer.nl>
*/
class CsvReader implements CountableReader, \SeekableIterator
{
const DUPLICATE_HEADERS_INCREMENT = 1;
const DUPLICATE_HEADERS_MERGE = 2;
/**
* Number of the row that contains the column names
*
* @var integer
*/
protected $headerRowNumber;
/**
* CSV file
*
* @var \SplFileObject
*/
protected $file;
/**
* Column headers as read from the CSV file
*
* @var array
*/
protected $columnHeaders = [];
/**
* Number of column headers, stored and re-used for performance
*
* In case of duplicate headers, this is always the number of unmerged headers.
*
* @var integer
*/
protected $headersCount;
/**
* Total number of rows in the CSV file
*
* @var integer
*/
protected $count;
/**
* Faulty CSV rows
*
* @var array
*/
protected $errors = [];
/**
* Strict parsing - skip any lines mismatching header length
*
* @var boolean
*/
protected $strict = true;
/**
* How to handle duplicate headers
*
* @var integer
*/
protected $duplicateHeadersFlag;
/**
* @param \SplFileObject $file
* @param string $delimiter
* @param string $enclosure
* @param string $escape
*/
public function __construct(\SplFileObject $file, $delimiter = ',', $enclosure = '"', $escape = '\\')
{
ini_set('auto_detect_line_endings', true);
$this->file = $file;
$this->file->setFlags(
\SplFileObject::READ_CSV |
\SplFileObject::SKIP_EMPTY |
\SplFileObject::READ_AHEAD |
\SplFileObject::DROP_NEW_LINE
);
$this->file->setCsvControl(
$delimiter,
$enclosure,
$escape
);
}
/**
* Return the current row as an array
*
* If a header row has been set, an associative array will be returned
*
* @return array
*/
public function current()
{
// If the CSV has no column headers just return the line
if (empty($this->columnHeaders)) {
return $this->file->current();
}
// Since the CSV has column headers use them to construct an associative array for the columns in this line
do {
$line = $this->file->current();
// In non-strict mode pad/slice the line to match the column headers
if (!$this->isStrict()) {
if ($this->headersCount > count($line)) {
$line = array_pad($line, $this->headersCount, null); // Line too short
} else {
$line = array_slice($line, 0, $this->headersCount); // Line too long
}
}
// See if values for duplicate headers should be merged
if (self::DUPLICATE_HEADERS_MERGE === $this->duplicateHeadersFlag) {
$line = $this->mergeDuplicates($line);
}
// Count the number of elements in both: they must be equal.
if (count($this->columnHeaders) === count($line)) {
return array_combine(array_keys($this->columnHeaders), $line);
}
// They are not equal, so log the row as error and skip it.
if ($this->valid()) {
$this->errors[$this->key()] = $line;
$this->next();
}
} while($this->valid());
return null;
}
/**
* Get column headers
*
* @return array
*/
public function getColumnHeaders()
{
return array_keys($this->columnHeaders);
}
/**
* Set column headers
*
* @param array $columnHeaders
*/
public function setColumnHeaders(array $columnHeaders)
{
$this->columnHeaders = array_count_values($columnHeaders);
$this->headersCount = count($columnHeaders);
}
/**
* Set header row number
*
* @param integer $rowNumber Number of the row that contains column header names
* @param integer $duplicates How to handle duplicates (optional). One of:
* - CsvReader::DUPLICATE_HEADERS_INCREMENT;
* increments duplicates (dup, dup1, dup2 etc.)
* - CsvReader::DUPLICATE_HEADERS_MERGE; merges
* values for duplicate headers into an array
* (dup => [value1, value2, value3])
*
* @throws DuplicateHeadersException If duplicate headers are encountered
* and no duplicate handling has been
* specified
*/
public function setHeaderRowNumber($rowNumber, $duplicates = null)
{
$this->duplicateHeadersFlag = $duplicates;
$this->headerRowNumber = $rowNumber;
$headers = $this->readHeaderRow($rowNumber);
$this->setColumnHeaders($headers);
}
/**
* Rewind the file pointer
*
* If a header row has been set, the pointer is set just below the header
* row. That way, when you iterate over the rows, that header row is
* skipped.
*/
public function rewind()
{
$this->file->rewind();
if (null !== $this->headerRowNumber) {
$this->file->seek($this->headerRowNumber + 1);
}
}
/**
* {@inheritdoc}
*/
public function count()
{
if (null === $this->count) {
$position = $this->key();
$this->count = iterator_count($this);
$this->seek($position);
}
return $this->count;
}
/**
* {@inheritdoc}
*/
public function next()
{
$this->file->next();
}
/**
* {@inheritdoc}
*/
public function valid()
{
return $this->file->valid();
}
/**
* {@inheritdoc}
*/
public function key()
{
return $this->file->key();
}
/**
* {@inheritdoc}
*/
public function seek($pointer)
{
$this->file->seek($pointer);
}
/**
* {@inheritdoc}
*/
public function getFields()
{
return $this->getColumnHeaders();
}
/**
* Get a row
*
* @param integer $number Row number
*
* @return array
*/
public function getRow($number)
{
$this->seek($number);
return $this->current();
}
/**
* Get rows that have an invalid number of columns
*
* @return array
*/
public function getErrors()
{
if (0 === $this->key()) {
// Iterator has not yet been processed, so do that now
foreach ($this as $row) { /* noop */ }
}
return $this->errors;
}
/**
* Does the reader contain any invalid rows?
*
* @return boolean
*/
public function hasErrors()
{
return count($this->getErrors()) > 0;
}
/**
* Should the reader use strict parsing?
*
* @return boolean
*/
public function isStrict()
{
return $this->strict;
}
/**
* Set strict parsing
*
* @param boolean $strict
*/
public function setStrict($strict)
{
$this->strict = $strict;
}
/**
* Read header row from CSV file
*
* @param integer $rowNumber Row number
*
* @return array
*
* @throws DuplicateHeadersException
*/
protected function readHeaderRow($rowNumber)
{
$this->file->seek($rowNumber);
$headers = $this->file->current();
// Test for duplicate column headers
$diff = array_diff_assoc($headers, array_unique($headers));
if (count($diff) > 0) {
switch ($this->duplicateHeadersFlag) {
case self::DUPLICATE_HEADERS_INCREMENT:
$headers = $this->incrementHeaders($headers);
// Fall through
case self::DUPLICATE_HEADERS_MERGE:
break;
default:
throw new DuplicateHeadersException($diff);
}
}
return $headers;
}
/**
* Add an increment to duplicate headers
*
* So the following line:
* |duplicate|duplicate|duplicate|
* |first |second |third |
*
* Yields value:
* $duplicate => 'first', $duplicate1 => 'second', $duplicate2 => 'third'
*
* @param array $headers
*
* @return array
*/
protected function incrementHeaders(array $headers)
{
$incrementedHeaders = [];
foreach (array_count_values($headers) as $header => $count) {
if ($count > 1) {
$incrementedHeaders[] = $header;
for ($i = 1; $i < $count; $i++) {
$incrementedHeaders[] = $header . $i;
}
} else {
$incrementedHeaders[] = $header;
}
}
return $incrementedHeaders;
}
/**
* Merges values for duplicate headers into an array
*
* So the following line:
* |duplicate|duplicate|duplicate|
* |first |second |third |
*
* Yields value:
* $duplicate => ['first', 'second', 'third']
*
* @param array $line
*
* @return array
*/
protected function mergeDuplicates(array $line)
{
$values = [];
$i = 0;
foreach ($this->columnHeaders as $count) {
if (1 === $count) {
$values[] = $line[$i];
} else {
$values[] = array_slice($line, $i, $count);
}
$i += $count;
}
return $values;
}
}

View File

@@ -0,0 +1,225 @@
<?php
namespace Ddeboer\DataImport\Reader;
use Doctrine\DBAL\Connection;
use Doctrine\DBAL\Statement;
/**
* Reads data through the Doctrine DBAL
*/
class DbalReader implements CountableReader
{
/**
* @var Connection
*/
private $connection;
/**
* @var array
*/
private $data;
/**
* @var Statement
*/
private $stmt;
/**
* @var string
*/
private $sql;
/**
* @var array
*/
private $params;
/**
* @var integer
*/
private $rowCount;
/**
* @var boolean
*/
private $rowCountCalculated = true;
/**
* @var string
*/
private $key;
/**
* @param Connection $connection
* @param string $sql
* @param array $params
*/
public function __construct(Connection $connection, $sql, array $params = [])
{
$this->connection = $connection;
$this->setSql($sql, $params);
}
/**
* Do calculate row count?
*
* @param boolean $calculate
*/
public function setRowCountCalculated($calculate = true)
{
$this->rowCountCalculated = (bool) $calculate;
}
/**
* Is row count calculated?
*
* @return boolean
*/
public function isRowCountCalculated()
{
return $this->rowCountCalculated;
}
/**
* {@inheritdoc}
*/
public function getFields()
{
if (null === $this->data) {
$this->rewind();
}
if (false === $this->data) {
return [];
}
return array_keys((array) $this->data);
}
/**
* Set Query string with Parameters
*
* @param string $sql
* @param array $params
*/
public function setSql($sql, array $params = [])
{
$this->sql = (string) $sql;
$this->setSqlParameters($params);
}
/**
* Set SQL parameters
*
* @param array $params
*/
public function setSqlParameters(array $params)
{
$this->params = $params;
$this->stmt = null;
$this->rowCount = null;
}
/**
* {@inheritdoc}
*/
public function current()
{
if (null === $this->data) {
$this->rewind();
}
return $this->data;
}
/**
* {@inheritdoc}
*/
public function next()
{
$this->key++;
$this->data = $this->stmt->fetch(\PDO::FETCH_ASSOC);
}
/**
* {@inheritdoc}
*/
public function key()
{
return $this->key;
}
/**
* {@inheritdoc}
*/
public function valid()
{
if (null === $this->data) {
$this->rewind();
}
return (false !== $this->data);
}
/**
* {@inheritdoc}
*/
public function rewind()
{
if (null === $this->stmt) {
$this->stmt = $this->prepare($this->sql, $this->params);
}
if (0 !== $this->key) {
$this->stmt->execute();
$this->data = $this->stmt->fetch(\PDO::FETCH_ASSOC);
$this->key = 0;
}
}
/**
* {@inheritdoc}
*/
public function count()
{
if (null === $this->rowCount) {
if ($this->rowCountCalculated) {
$this->doCalcRowCount();
} else {
if (null === $this->stmt) {
$this->rewind();
}
$this->rowCount = $this->stmt->rowCount();
}
}
return $this->rowCount;
}
private function doCalcRowCount()
{
$statement = $this->prepare(sprintf('SELECT COUNT(*) FROM (%s) AS count', $this->sql), $this->params);
$statement->execute();
$this->rowCount = (int) $statement->fetchColumn(0);
}
/**
* Prepare given statement
*
* @param string $sql
* @param array $params
*
* @return Statement
*/
private function prepare($sql, array $params)
{
$statement = $this->connection->prepare($sql);
foreach ($params as $key => $value) {
$statement->bindValue($key, $value);
}
return $statement;
}
}

View File

@@ -0,0 +1,108 @@
<?php
namespace Ddeboer\DataImport\Reader;
use Doctrine\Common\Persistence\ObjectManager;
use Doctrine\ORM\Internal\Hydration\IterableResult;
use Doctrine\ORM\Query;
/**
* Reads entities through the Doctrine ORM
*
* @author David de Boer <david@ddeboer.nl>
*/
class DoctrineReader implements CountableReader
{
/**
* @var ObjectManager
*/
protected $objectManager;
/**
* @var string
*/
protected $objectName;
/**
* @var IterableResult
*/
protected $iterableResult;
/**
* @param ObjectManager $objectManager
* @param string $objectName e.g. YourBundle:YourEntity
*/
public function __construct(ObjectManager $objectManager, $objectName)
{
$this->objectManager = $objectManager;
$this->objectName = $objectName;
}
/**
* {@inheritdoc}
*/
public function getFields()
{
return $this->objectManager->getClassMetadata($this->objectName)
->getFieldNames();
}
/**
* {@inheritdoc}
*/
public function current()
{
return current($this->iterableResult->current());
}
/**
* {@inheritdoc}
*/
public function next()
{
$this->iterableResult->next();
}
/**
* {@inheritdoc}
*/
public function key()
{
return $this->iterableResult->key();
}
/**
* {@inheritdoc}
*/
public function valid()
{
return $this->iterableResult->valid();
}
/**
* {@inheritdoc}
*/
public function rewind()
{
if (!$this->iterableResult) {
$query = $this->objectManager->createQuery(
sprintf('SELECT o FROM %s o', $this->objectName)
);
$this->iterableResult = $query->iterate([], Query::HYDRATE_ARRAY);
}
$this->iterableResult->rewind();
}
/**
* {@inheritdoc}
*/
public function count()
{
$query = $this->objectManager->createQuery(
sprintf('SELECT COUNT(o) FROM %s o', $this->objectName)
);
return $query->getSingleScalarResult();
}
}

View File

@@ -0,0 +1,206 @@
<?php
namespace Ddeboer\DataImport\Reader;
/**
* Reads Excel files with the help of PHPExcel
*
* PHPExcel must be installed.
*
* @author David de Boer <david@ddeboer.nl>
*
* @link http://phpexcel.codeplex.com/
* @link https://github.com/logiQ/PHPExcel
*/
class ExcelReader implements CountableReader, \SeekableIterator
{
/**
* @var array
*/
protected $worksheet;
/**
* @var integer
*/
protected $headerRowNumber;
/**
* @var integer
*/
protected $pointer = 0;
/**
* @var array
*/
protected $columnHeaders;
/**
* Total number of rows
*
* @var integer
*/
protected $count;
/**
* @param \SplFileObject $file Excel file
* @param integer $headerRowNumber Optional number of header row
* @param integer $activeSheet Index of active sheet to read from
* @param boolean $readOnly If set to false, the reader take care of the excel formatting (slow)
*/
public function __construct(\SplFileObject $file, $headerRowNumber = null, $activeSheet = null, $readOnly = true)
{
$reader = \PHPExcel_IOFactory::createReaderForFile($file->getPathName());
$reader->setReadDataOnly($readOnly);
/** @var \PHPExcel $excel */
$excel = $reader->load($file->getPathname());
if (null !== $activeSheet) {
$excel->setActiveSheetIndex($activeSheet);
}
$this->worksheet = $excel->getActiveSheet()->toArray();
if (null !== $headerRowNumber) {
$this->setHeaderRowNumber($headerRowNumber);
}
}
/**
* Return the current row as an array
*
* If a header row has been set, an associative array will be returned
*
* @return array
*/
public function current()
{
$row = $this->worksheet[$this->pointer];
// If the CSV has column headers, use them to construct an associative
// array for the columns in this line
if (!empty($this->columnHeaders)) {
// Count the number of elements in both: they must be equal.
// If not, ignore the row
if (count($this->columnHeaders) == count($row)) {
return array_combine(array_values($this->columnHeaders), $row);
}
} else {
// Else just return the column values
return $row;
}
}
/**
* Get column headers
*
* @return array
*/
public function getColumnHeaders()
{
return $this->columnHeaders;
}
/**
* Set column headers
*
* @param array $columnHeaders
*/
public function setColumnHeaders(array $columnHeaders)
{
$this->columnHeaders = $columnHeaders;
}
/**
* Rewind the file pointer
*
* If a header row has been set, the pointer is set just below the header
* row. That way, when you iterate over the rows, that header row is
* skipped.
*/
public function rewind()
{
if (null === $this->headerRowNumber) {
$this->pointer = 0;
} else {
$this->pointer = $this->headerRowNumber + 1;
}
}
/**
* Set header row number
*
* @param integer $rowNumber Number of the row that contains column header names
*/
public function setHeaderRowNumber($rowNumber)
{
$this->headerRowNumber = $rowNumber;
$this->columnHeaders = $this->worksheet[$rowNumber];
}
/**
* {@inheritdoc}
*/
public function next()
{
$this->pointer++;
}
/**
* {@inheritdoc}
*/
public function valid()
{
return isset($this->worksheet[$this->pointer]);
}
/**
* {@inheritdoc}
*/
public function key()
{
return $this->pointer;
}
/**
* {@inheritdoc}
*/
public function seek($pointer)
{
$this->pointer = $pointer;
}
/**
* {@inheritdoc}
*/
public function count()
{
$count = count($this->worksheet);
if (null !== $this->headerRowNumber) {
$count--;
}
return $count;
}
/**
* {@inheritdoc}
*/
public function getFields()
{
return $this->columnHeaders;
}
/**
* Get a row
*
* @param integer $number
*
* @return array
*/
public function getRow($number)
{
$this->seek($number);
return $this->current();
}
}

View File

@@ -0,0 +1,76 @@
<?php
namespace Ddeboer\DataImport\Reader\Factory;
use Ddeboer\DataImport\Reader\CsvReader;
/**
* Factory that creates CsvReaders
*
* @author David de Boer <david@ddeboer.nl>
*/
class CsvReaderFactory
{
/**
* @var integer
*/
protected $headerRowNumber;
/**
* @var boolean
*/
protected $strict;
/**
* @var string
*/
protected $delimiter;
/**
* @var string
*/
protected $enclosure;
/**
* @var string
*/
protected $escape;
/**
* @param integer $headerRowNumber
* @param boolean $strict
* @param string $delimiter
* @param string $enclosure
* @param string $escape
*/
public function __construct(
$headerRowNumber = null,
$strict = true,
$delimiter = ',',
$enclosure = '"',
$escape = '\\'
) {
$this->headerRowNumber = $headerRowNumber;
$this->strict = $strict;
$this->delimiter = $delimiter;
$this->enclosure = $enclosure;
$this->escape = $escape;
}
/**
* @param \SplFileObject $file
*
* @return CsvReader
*/
public function getReader(\SplFileObject $file)
{
$reader = new CsvReader($file, $this->delimiter, $this->enclosure, $this->escape);
if (null !== $this->headerRowNumber) {
$reader->setHeaderRowNumber($this->headerRowNumber);
}
$reader->setStrict($this->strict);
return $reader;
}
}

View File

@@ -0,0 +1,38 @@
<?php
namespace Ddeboer\DataImport\Reader\Factory;
use Ddeboer\DataImport\Reader\DbalReader;
use Doctrine\DBAL\Connection;
/**
* Factory that creates DbalReaders
*
* @author David de Boer <david@ddeboer.nl>
*/
class DbalReaderFactory
{
/**
* @var Connection
*/
protected $connection;
/**
* @param Connection $connection
*/
public function __construct(Connection $connection)
{
$this->connection = $connection;
}
/**
* @param string $sql
* @param array $params
*
* @return DbalReader
*/
public function getReader($sql, array $params = [])
{
return new DbalReader($this->connection, $sql, $params);
}
}

View File

@@ -0,0 +1,37 @@
<?php
namespace Ddeboer\DataImport\Reader\Factory;
use Ddeboer\DataImport\Reader\DoctrineReader;
use Doctrine\Common\Persistence\ObjectManager;
/**
* Factory that creates DoctrineReaders
*
* @author David de Boer <david@ddeboer.nl>
*/
class DoctrineReaderFactory
{
/**
* @var ObjectManager
*/
protected $objectManager;
/**
* @param ObjectManager $objectManager
*/
public function __construct(ObjectManager $objectManager)
{
$this->objectManager = $objectManager;
}
/**
* @param string $object
*
* @return DoctrineReader
*/
public function getReader($object)
{
return new DoctrineReader($this->objectManager, $object);
}
}

View File

@@ -0,0 +1,43 @@
<?php
namespace Ddeboer\DataImport\Reader\Factory;
use Ddeboer\DataImport\Reader\ExcelReader;
/**
* Factory that creates ExcelReaders
*
* @author David de Boer <david@ddeboer.nl>
*/
class ExcelReaderFactory
{
/**
* @var integer
*/
protected $headerRowNumber;
/**
* @var integer
*/
protected $activeSheet;
/**
* @param integer $headerRowNumber
* @param integer $activeSheet
*/
public function __construct($headerRowNumber = null, $activeSheet = null)
{
$this->headerRowNumber = $headerRowNumber;
$this->activeSheet = $activeSheet;
}
/**
* @param \SplFileObject $file
*
* @return ExcelReader
*/
public function getReader(\SplFileObject $file)
{
return new ExcelReader($file, $this->headerRowNumber, $this->activeSheet);
}
}

View File

@@ -0,0 +1,21 @@
<?php
namespace Ddeboer\DataImport\Reader;
use Ddeboer\DataImport\Reader;
/**
* Use an iterator as a reader
*
* @author Márk Sági-Kazár <mark.sagikazar@gmail.com>
*/
class IteratorReader extends \IteratorIterator implements Reader
{
/**
* {@inheritdoc}
*/
public function getFields()
{
return array_keys($this->current());
}
}

View File

@@ -0,0 +1,183 @@
<?php
namespace Ddeboer\DataImport\Reader;
use Ddeboer\DataImport\Reader;
use Ddeboer\DataImport\Exception\ReaderException;
/**
* Takes multiple readers for processing in the same workflow
*
* @author Adam Paterson <hello@adampaterson.co.uk>
* @author Aydin Hassan <aydin@hotmail.co.uk>
*/
class OneToManyReader implements CountableReader
{
/**
* @var Reader
*/
protected $leftReader;
/**
* @var Reader
*/
protected $rightReader;
/**
* @var string
*/
protected $leftJoinField;
/**
* @var string
*/
protected $rightJoinField;
/**
* Key to nest the rightRows under
*
* @var string
*/
protected $nestKey;
/**
* @param Reader $leftReader
* @param Reader $rightReader
* @param string $nestKey
* @param string $leftJoinField
* @param string $rightJoinField
*/
public function __construct(
Reader $leftReader,
Reader $rightReader,
$nestKey,
$leftJoinField,
$rightJoinField = null
) {
if (is_null($rightJoinField)) {
$rightJoinField = $leftJoinField;
}
$this->leftJoinField = $leftJoinField;
$this->rightJoinField = $rightJoinField;
$this->leftReader = $leftReader;
$this->rightReader = $rightReader;
$this->nestKey = $nestKey;
}
/**
* Create an array of children in the leftRow,
* with the data returned from the right reader
* Where the ID fields Match
*
* @return array
*
* @throws ReaderException
*/
public function current()
{
$leftRow = $this->leftReader->current();
if (array_key_exists($this->nestKey, $leftRow)) {
throw new ReaderException(
sprintf(
'Left Row: "%s" Reader already contains a field named "%s". Please choose a different nest key field',
$this->key(),
$this->nestKey
)
);
}
$leftRow[$this->nestKey] = [];
$leftId = $this->getRowId($leftRow, $this->leftJoinField);
$rightRow = $this->rightReader->current();
$rightId = $this->getRowId($rightRow, $this->rightJoinField);
while ($leftId == $rightId && $this->rightReader->valid()) {
$leftRow[$this->nestKey][] = $rightRow;
$this->rightReader->next();
$rightRow = $this->rightReader->current();
if($this->rightReader->valid()) {
$rightId = $this->getRowId($rightRow, $this->rightJoinField);
}
}
return $leftRow;
}
/**
* @param array $row
* @param string $idField
*
* @return mixed
*
* @throws ReaderException
*/
protected function getRowId(array $row, $idField)
{
if (!array_key_exists($idField, $row)) {
throw new ReaderException(
sprintf(
'Row: "%s" has no field named "%s"',
$this->key(),
$idField
)
);
}
return $row[$idField];
}
/**
* {@inheritdoc}
*/
public function next()
{
$this->leftReader->next();
//right reader is iterated in current() method.
}
/**
* {@inheritdoc}
*/
public function key()
{
return $this->leftReader->key();
}
/**
* {@inheritdoc}
*/
public function valid()
{
return $this->leftReader->valid() && $this->rightReader->valid();
}
/**
* {@inheritdoc}
*/
public function rewind()
{
$this->leftReader->rewind();
$this->rightReader->rewind();
}
/**
* {@inheritdoc}
*/
public function getFields()
{
return array_merge($this->leftReader->getFields(), [$this->nestKey]);
}
/**
* {@inheritdoc}
*/
public function count()
{
return $this->leftReader->count();
}
}

View File

@@ -0,0 +1,128 @@
<?php
namespace Ddeboer\DataImport\Reader;
/**
* Reads data through PDO
*
* @author Robbie Mackay
*/
class PdoReader implements CountableReader
{
/**
* @var \PDO
*/
protected $pdo;
/**
* @var string
*/
protected $tableName;
/**
* @var \PDOStatement
*/
protected $statement;
/**
* @var array
*/
private $data;
/**
* @param \PDO $pdo
* @param string $sql
* @param array $params
*/
public function __construct(\PDO $pdo, $sql, array $params = [])
{
$this->pdo = $pdo;
$this->statement = $this->pdo->prepare($sql);
foreach ($params as $key => $value) {
$this->statement->bindValue($key, $value);
}
}
/**
* {@inheritdoc}
*/
public function getFields()
{
if ($this->statement->execute()) {
// Statement executed successfully
// Grab the first row to find keys
$row = $this->statement->fetch(\PDO::FETCH_ASSOC);
// Return field keys, or empty array no rows remain
return array_keys($row ? $row : []);
} else {
// If the statement errors return empty
return [];
}
}
/**
* {@inheritdoc}
*/
public function current()
{
return current($this->data);
}
/**
* {@inheritdoc}
*/
public function next()
{
next($this->data);
}
/**
* {@inheritdoc}
*/
public function key()
{
return key($this->data);
}
/**
* {@inheritdoc}
*/
public function valid()
{
$key = key($this->data);
return ($key !== null && $key !== false);
}
/**
* {@inheritdoc}
*/
public function rewind()
{
$this->loadData();
reset($this->data);
}
/**
* {@inheritdoc}
*/
public function count()
{
$this->loadData();
return count($this->data);
}
/**
* Load data if it hasn't been loaded yet
*/
protected function loadData()
{
if (null === $this->data) {
$this->statement->execute();
$this->data = $this->statement->fetchAll(\PDO::FETCH_ASSOC);
}
}
}

View File

@@ -0,0 +1,146 @@
<?php
namespace Ddeboer\DataImport;
use Ddeboer\DataImport\Exception\ExceptionInterface;
/**
* Simple Container for Workflow Results
*
* @author Aydin Hassan <aydin@hotmail.co.uk>
*/
class Result
{
/**
* Identifier given to the import/export
*
* @var string
*/
protected $name;
/**
* @var \DateTime
*/
protected $startTime;
/**
* @var \DateTime
*/
protected $endTime;
/**
* @var \DateInterval
*/
protected $elapsed;
/**
* @var integer
*/
protected $errorCount = 0;
/**
* @var integer
*/
protected $successCount = 0;
/**
* @var integer
*/
protected $totalProcessedCount = 0;
/**
* @var \SplObjectStorage
*/
protected $exceptions;
/**
* @param string $name
* @param \DateTime $startTime
* @param \DateTime $endTime
* @param integer $totalCount
* @param \SplObjectStorage $exceptions
*/
public function __construct($name, \DateTime $startTime, \DateTime $endTime, $totalCount, \SplObjectStorage $exceptions)
{
$this->name = $name;
$this->startTime = $startTime;
$this->endTime = $endTime;
$this->elapsed = $startTime->diff($endTime);
$this->totalProcessedCount = $totalCount;
$this->errorCount = count($exceptions);
$this->successCount = $totalCount - $this->errorCount;
$this->exceptions = $exceptions;
}
/**
* @return string
*/
public function getName()
{
return $this->name;
}
/**
* @return \DateTime
*/
public function getStartTime()
{
return $this->startTime;
}
/**
* @return \DateTime
*/
public function getEndTime()
{
return $this->endTime;
}
/**
* @return \DateInterval
*/
public function getElapsed()
{
return $this->elapsed;
}
/**
* @return integer
*/
public function getErrorCount()
{
return $this->errorCount;
}
/**
* @return integer
*/
public function getSuccessCount()
{
return $this->successCount;
}
/**
* @return integer
*/
public function getTotalProcessedCount()
{
return $this->totalProcessedCount;
}
/**
* @return boolean
*/
public function hasErrors()
{
return $this->errorCount > 0;
}
/**
* @return \SplObjectStorage
*/
public function getExceptions()
{
return $this->exceptions;
}
}

18
vendor/ddeboer/data-import/src/Step.php vendored Normal file
View File

@@ -0,0 +1,18 @@
<?php
namespace Ddeboer\DataImport;
/**
* @author Markus Bachmann <markus.bachmann@bachi.biz>
*/
interface Step
{
/**
* Any processing done on each item in the data stack
*
* @param mixed &$item
*
* @return boolean False return value means the item should be skipped
*/
public function process(&$item);
}

View File

@@ -0,0 +1,51 @@
<?php
namespace Ddeboer\DataImport\Step;
use Ddeboer\DataImport\Step;
use Ddeboer\DataImport\Exception\UnexpectedTypeException;
/**
* @author Markus Bachmann <markus.bachmann@bachi.biz>
*/
class ConverterStep implements Step
{
/**
* @var callable[]
*/
private $converters;
/**
* @param array $converters
*/
public function __construct(array $converters = [])
{
foreach ($converters as $converter) {
$this->add($converter);
}
}
/**
* @param callable $converter
*
* @return $this
*/
public function add(callable $converter)
{
$this->converters[] = $converter;
return $this;
}
/**
* {@inheritdoc}
*/
public function process(&$item)
{
foreach ($this->converters as $converter) {
$item = call_user_func($converter, $item);
}
return true;
}
}

View File

@@ -0,0 +1,48 @@
<?php
namespace Ddeboer\DataImport\Step;
use Ddeboer\DataImport\Step;
/**
* @author Markus Bachmann <markus.bachmann@bachi.biz>
*/
class FilterStep implements Step
{
/**
* @var \SplPriorityQueue
*/
private $filters;
public function __construct()
{
$this->filters = new \SplPriorityQueue();
}
/**
* @param callable $filter
* @param integer $priority
*
* @return $this
*/
public function add(callable $filter, $priority = null)
{
$this->filters->insert($filter, $priority);
return $this;
}
/**
* {@inheritdoc}
*/
public function process(&$item)
{
foreach (clone $this->filters as $filter) {
if (false === call_user_func($filter, $item)) {
return false;
}
}
return true;
}
}

View File

@@ -0,0 +1,75 @@
<?php
namespace Ddeboer\DataImport\Step;
use Ddeboer\DataImport\Exception\MappingException;
use Ddeboer\DataImport\Step;
use Symfony\Component\PropertyAccess\Exception\NoSuchPropertyException;
use Symfony\Component\PropertyAccess\Exception\UnexpectedTypeException;
use Symfony\Component\PropertyAccess\PropertyAccessor;
/**
* @author Markus Bachmann <markus.bachmann@bachi.biz>
*/
class MappingStep implements Step
{
/**
* @var array
*/
private $mappings = [];
/**
* @var PropertyAccessor
*/
private $accessor;
/**
* @param array $mappings
* @param PropertyAccessor $accessor
*/
public function __construct(array $mappings = [], PropertyAccessor $accessor = null)
{
$this->mappings = $mappings;
$this->accessor = $accessor ?: new PropertyAccessor();
}
/**
* @param string $from
* @param string $to
*
* @return $this
*/
public function map($from, $to)
{
$this->mappings[$from] = $to;
return $this;
}
/**
* {@inheritdoc}
*
* @throws MappingException
*/
public function process(&$item)
{
try {
foreach ($this->mappings as $from => $to) {
$value = $this->accessor->getValue($item, $from);
$this->accessor->setValue($item, $to, $value);
$from = str_replace(['[',']'], '', $from);
// Check if $item is an array, because properties can't be unset.
// So we don't call unset for objects to prevent side affects.
if (is_array($item) && isset($item[$from])) {
unset($item[$from]);
}
}
} catch (NoSuchPropertyException $exception) {
throw new MappingException('Unable to map item', null, $exception);
} catch (UnexpectedTypeException $exception) {
throw new MappingException('Unable to map item', null, $exception);
}
}
}

View File

@@ -0,0 +1,16 @@
<?php
namespace Ddeboer\DataImport\Step;
use Ddeboer\DataImport\Step;
/**
* @author Markus Bachmann <markus.bachmann@bachi.biz>
*/
interface PriorityStep extends Step
{
/**
* @return integer
*/
public function getPriority();
}

View File

@@ -0,0 +1,109 @@
<?php
namespace Ddeboer\DataImport\Step;
use Ddeboer\DataImport\Exception\ValidationException;
use Symfony\Component\Validator\Constraints;
use Symfony\Component\Validator\Constraint;
use Symfony\Component\Validator\Validator\ValidatorInterface;
/**
* @author Markus Bachmann <markus.bachmann@bachi.biz>
*/
class ValidatorStep implements PriorityStep
{
/**
* @var array
*/
private $constraints = [];
/**
* @var array
*/
private $violations = [];
/**
* @var boolean
*/
private $throwExceptions = false;
/**
* @var integer
*/
private $line = 1;
/**
* @var ValidatorInterface
*/
private $validator;
/**
* @param ValidatorInterface $validator
*/
public function __construct(ValidatorInterface $validator)
{
$this->validator = $validator;
}
/**
* @param string $field
* @param Constraint $constraint
*
* @return $this
*/
public function add($field, Constraint $constraint)
{
if (!isset($this->constraints[$field])) {
$this->constraints[$field] = [];
}
$this->constraints[$field][] = $constraint;
return $this;
}
/**
* @param boolean $flag
*/
public function throwExceptions($flag = true)
{
$this->throwExceptions = $flag;
}
/**
* @return array
*/
public function getViolations()
{
return $this->violations;
}
/**
* {@inheritdoc}
*/
public function process(&$item)
{
$constraints = new Constraints\Collection($this->constraints);
$list = $this->validator->validate($item, $constraints);
if (count($list) > 0) {
$this->violations[$this->line] = $list;
if ($this->throwExceptions) {
throw new ValidationException($list, $this->line);
}
}
$this->line++;
return 0 === count($list);
}
/**
* {@inheritdoc}
*/
public function getPriority()
{
return 128;
}
}

View File

@@ -0,0 +1,48 @@
<?php
namespace Ddeboer\DataImport\Step;
use Ddeboer\DataImport\Step;
use Symfony\Component\PropertyAccess\PropertyAccessor;
/**
* @author Markus Bachmann <markus.bachmann@bachi.biz>
*/
class ValueConverterStep implements Step
{
/**
* @var array
*/
private $converters = [];
/**
* @param string $property
* @param callable $converter
*
* @return $this
*/
public function add($property, callable $converter)
{
$this->converters[$property][] = $converter;
return $this;
}
/**
* {@inheritdoc}
*/
public function process(&$item)
{
$accessor = new PropertyAccessor();
foreach ($this->converters as $property => $converters) {
foreach ($converters as $converter) {
$orgValue = $accessor->getValue($item, $property);
$value = call_user_func($converter, $orgValue);
$accessor->setValue($item,$property,$value);
}
}
return true;
}
}

View File

@@ -0,0 +1,62 @@
<?php
namespace Ddeboer\DataImport\ValueConverter;
/**
* Converts a nested array using a converter-map
*
* @author Christoph Rosse <christoph@rosse.at>
*/
class ArrayValueConverterMap
{
/**
* @var array
*/
private $converters;
/**
* @param callable[] $converters
*/
public function __construct(array $converters)
{
$this->converters = $converters;
}
/**
* {@inheritdoc}
*/
public function __invoke($input)
{
if (!is_array($input)) {
throw new \InvalidArgumentException('Input of a ArrayValueConverterMap must be an array');
}
foreach ($input as $key => $item) {
$input[$key] = $this->convertItem($item);
}
return $input;
}
/**
* Convert an item of the array using the converter-map
*
* @param $item
*
* @return mixed
*/
protected function convertItem($item)
{
foreach ($item as $key => $value) {
if (!isset($this->converters[$key])) {
continue;
}
foreach ($this->converters[$key] as $converter) {
$item[$key] = call_user_func($converter, $item[$key]);
}
}
return $item;
}
}

View File

@@ -0,0 +1,49 @@
<?php
namespace Ddeboer\DataImport\ValueConverter;
use Ddeboer\DataImport\Exception\UnexpectedTypeException;
/**
* Convert a value in a specific charset
*
* @author Markus Bachmann <markus.bachmann@bachi.biz>
*/
class CharsetValueConverter
{
/**
* @var string
*/
private $charset;
/**
* @var string
*/
private $inCharset;
/**
* @param string $charset Charset to convert values to
* @param string $inCharset Charset of input values
*/
public function __construct($charset, $inCharset = 'UTF-8')
{
$this->charset = $charset;
$this->inCharset = $inCharset;
}
/**
* {@inheritdoc}
*/
public function __invoke($input)
{
if (function_exists('mb_convert_encoding')) {
return mb_convert_encoding($input, $this->charset, $this->inCharset);
}
if (function_exists('iconv')) {
return iconv($this->inCharset, $this->charset, $input);
}
throw new \RuntimeException('Could not convert the charset. Please install the mbstring or iconv extension!');
}
}

View File

@@ -0,0 +1,48 @@
<?php
namespace Ddeboer\DataImport\ValueConverter;
use Ddeboer\DataImport\Exception\UnexpectedValueException;
/**
* Convert an date time object into string
*/
class DateTimeToStringValueConverter
{
/**
* Date time format
*
* @var string
* @see http://php.net/manual/en/datetime.createfromformat.php
*/
protected $outputFormat;
/**
* @param string $outputFormat
*/
public function __construct($outputFormat = 'Y-m-d H:i:s')
{
$this->outputFormat = $outputFormat;
}
/**
* Convert string to date time object
* using specified format
*
* @param mixed $input
* @return \DateTime|string
* @throws UnexpectedValueException
*/
public function convert($input)
{
if (!$input) {
return;
}
if (!($input instanceof \DateTime)) {
throw new UnexpectedValueException('Input must be DateTime object.');
}
return $input->format($this->outputFormat);
}
}

View File

@@ -0,0 +1,79 @@
<?php
namespace Ddeboer\DataImport\ValueConverter;
use \Ddeboer\DataImport\Exception\UnexpectedValueException;
/**
* Convert an date string into another date string
* Eg. You want to change the format of a string OR
* If no output specified, return DateTime instance
*
* @author David de Boer <david@ddeboer.nl>
*/
class DateTimeValueConverter
{
/**
* Date time format
*
* @var string
*
* @see http://php.net/manual/en/datetime.createfromformat.php
*/
protected $inputFormat;
/**
* Date time format
*
* @var string
*
* @see http://php.net/manual/en/datetime.createfromformat.php
*/
protected $outputFormat;
/**
* @param string $inputFormat
* @param string $outputFormat
*/
public function __construct($inputFormat = null, $outputFormat = null)
{
$this->inputFormat = $inputFormat;
$this->outputFormat = $outputFormat;
}
/**
* Convert string to date time object then convert back to a string
* using specified format
*
* If no output format specified then return
* the \DateTime instance
*
* @param mixed $input
* @return \DateTime|string
* @throws UnexpectedValueException
*/
public function __invoke($input)
{
if (!$input) {
return;
}
if ($this->inputFormat) {
$date = \DateTime::createFromFormat($this->inputFormat, $input);
if (false === $date) {
throw new UnexpectedValueException(
$input . ' is not a valid date/time according to format ' . $this->inputFormat
);
}
} else {
$date = new \DateTime($input);
}
if ($this->outputFormat) {
return $date->format($this->outputFormat);
}
//if no output format specified we just return the \DateTime instance
return $date;
}
}

View File

@@ -0,0 +1,36 @@
<?php
namespace Ddeboer\DataImport\ValueConverter;
use Ddeboer\DataImport\Exception\UnexpectedValueException;
/**
* @author Grégoire Paris
*/
class MappingValueConverter
{
/**
* @var array
*/
private $mapping = [];
/**
* @param array $mapping
*/
public function __construct(array $mapping)
{
$this->mapping = $mapping;
}
public function __invoke($input)
{
if (!isset($this->mapping[$input])) {
throw new UnexpectedValueException(sprintf(
'Cannot find mapping for value "%s"',
$input
));
}
return $this->mapping[$input];
}
}

View File

@@ -0,0 +1,75 @@
<?php
namespace Ddeboer\DataImport\ValueConverter;
use Ddeboer\DataImport\Exception\UnexpectedTypeException;
use Symfony\Component\PropertyAccess\PropertyPath;
use Symfony\Component\PropertyAccess\PropertyAccess;
use Symfony\Component\PropertyAccess\PropertyAccessor;
/**
* @author Markus Bachmann <markus.bachmann@bachi.biz
*/
class ObjectConverter
{
/**
* @var string|null
*/
protected $propertyPath;
/**
* @var PropertyAccessor
*/
protected $propertyAccessor;
/**
* @param string|null $propertyPath
*/
public function __construct($propertyPath = null)
{
$this->propertyPath = $propertyPath;
$this->propertyAccessor = PropertyAccess::createPropertyAccessor();
}
/**
* Sets the property
*
* @param string $propertyPath
*/
public function setPropertyPath($propertyPath)
{
$this->propertyPath = $propertyPath;
}
/**
* Gets the property
*
* @return null|string
*/
public function getPropertyPath()
{
return $this->propertyPath;
}
/**
* {@inheritdoc}
*/
public function __invoke($input)
{
if (!is_object($input)) {
throw new UnexpectedTypeException($input, 'object');
}
if (null === $this->propertyPath && !method_exists($input, '__toString')) {
throw new \RuntimeException;
}
if (null === $this->propertyPath) {
return (string) $input;
}
$path = new PropertyPath($this->propertyPath);
return $this->propertyAccessor->getValue($input, $path);
}
}

View File

@@ -0,0 +1,43 @@
<?php
namespace Ddeboer\DataImport\ValueConverter;
use Doctrine\Common\Persistence\ObjectRepository;
/**
* Converts a string to an object
*
* @author Markus Bachmann <markus.bachmann@bachi.biz>
*/
class StringToObjectConverter
{
/**
* @var ObjectRepository
*/
private $repository;
/**
* @var string
*/
private $property;
/**
* @param ObjectRepository $repository
* @param string $property
*/
public function __construct(ObjectRepository $repository, $property)
{
$this->repository = $repository;
$this->property = $property;
}
/**
* {@inheritdoc}
*/
public function __invoke($input)
{
$method = 'findOneBy'.ucfirst($this->property);
return $this->repository->$method($input);
}
}

View File

@@ -0,0 +1,20 @@
<?php
namespace Ddeboer\DataImport;
/**
* A mediator between a reader and one or more writers and converters
*
* @author Markus Bachmann <markus.bachmann@bachi.biz>
*/
interface Workflow
{
/**
* Process the whole import workflow
*
* @return Result Object Containing Workflow Results
*
* @throws Exception
*/
public function process();
}

View File

@@ -0,0 +1,196 @@
<?php
namespace Ddeboer\DataImport\Workflow;
use Ddeboer\DataImport\Exception;
use Ddeboer\DataImport\Exception\UnexpectedTypeException;
use Ddeboer\DataImport\Reader;
use Ddeboer\DataImport\Result;
use Ddeboer\DataImport\Step;
use Ddeboer\DataImport\Step\PriorityStep;
use Ddeboer\DataImport\Workflow;
use Ddeboer\DataImport\Writer;
use Psr\Log\LoggerInterface;
use Psr\Log\LoggerAwareInterface;
use Psr\Log\LoggerAwareTrait;
use Psr\Log\NullLogger;
/**
* A mediator between a reader and one or more writers and converters
*
* @author David de Boer <david@ddeboer.nl>
*/
class StepAggregator implements Workflow, LoggerAwareInterface
{
use LoggerAwareTrait;
/**
* @var Reader
*/
private $reader;
/**
* Identifier for the Import/Export
*
* @var string|null
*/
private $name = null;
/**
* @var boolean
*/
private $skipItemOnFailure = false;
/**
* @var \SplPriorityQueue
*/
private $steps;
/**
* @var Writer[]
*/
private $writers = [];
/**
* @var boolean
*/
protected $shouldStop = false;
/**
* @param Reader $reader
* @param string $name
*/
public function __construct(Reader $reader, $name = null)
{
$this->name = $name;
$this->reader = $reader;
// Defaults
$this->logger = new NullLogger();
$this->steps = new \SplPriorityQueue();
}
/**
* Add a step to the current workflow
*
* @param Step $step
* @param integer|null $priority
*
* @return $this
*/
public function addStep(Step $step, $priority = null)
{
$priority = null === $priority && $step instanceof PriorityStep ? $step->getPriority() : $priority;
$priority = null === $priority ? 0 : $priority;
$this->steps->insert($step, $priority);
return $this;
}
/**
* Add a new writer to the current workflow
*
* @param Writer $writer
*
* @return $this
*/
public function addWriter(Writer $writer)
{
array_push($this->writers, $writer);
return $this;
}
/**
* {@inheritdoc}
*/
public function process()
{
$count = 0;
$exceptions = new \SplObjectStorage();
$startTime = new \DateTime;
foreach ($this->writers as $writer) {
$writer->prepare();
}
if (is_callable('pcntl_signal')) {
pcntl_signal(SIGTERM, array($this, 'stop'));
pcntl_signal(SIGINT, array($this, 'stop'));
}
// Read all items
foreach ($this->reader as $index => $item) {
if (is_callable('pcntl_signal_dispatch')) {
pcntl_signal_dispatch();
}
if ($this->shouldStop) {
break;
}
try {
foreach (clone $this->steps as $step) {
if (false === $step->process($item)) {
continue 2;
}
}
if (!is_array($item) && !($item instanceof \ArrayAccess && $item instanceof \Traversable)) {
throw new UnexpectedTypeException($item, 'array');
}
foreach ($this->writers as $writer) {
$writer->writeItem($item);
}
} catch(Exception $e) {
if (!$this->skipItemOnFailure) {
throw $e;
}
$exceptions->attach($e, $index);
$this->logger->error($e->getMessage());
}
$count++;
}
foreach ($this->writers as $writer) {
$writer->finish();
}
return new Result($this->name, $startTime, new \DateTime, $count, $exceptions);
}
/**
* Stops processing and force return Result from process() function
*/
public function stop()
{
$this->shouldStop = true;
}
/**
* Sets the value which determines whether the item should be skipped when error occures
*
* @param boolean $skipItemOnFailure When true skip current item on process exception and log the error
*
* @return $this
*/
public function setSkipItemOnFailure($skipItemOnFailure)
{
$this->skipItemOnFailure = $skipItemOnFailure;
return $this;
}
/**
* @return string
*/
public function getName()
{
return $this->name;
}
}

View File

@@ -0,0 +1,28 @@
<?php
namespace Ddeboer\DataImport;
/**
* Persists data in a storage medium, such as a database, CSV or XML file, etc.
*
* @author David de Boer <david@ddeboer.nl>
*/
interface Writer
{
/**
* Prepare the writer before writing the items
*/
public function prepare();
/**
* Write one data item
*
* @param array $item The data item with converted values
*/
public function writeItem(array $item);
/**
* Wrap up the writer after all items have been written
*/
public function finish();
}

View File

@@ -0,0 +1,105 @@
<?php
namespace Ddeboer\DataImport\Writer;
use Ddeboer\DataImport\Writer;
/**
* Base class to write into streams
*
* @author Benoît Burnichon <bburnichon@gmail.com>
*/
abstract class AbstractStreamWriter implements Writer
{
use WriterTemplate;
/**
* @var resource
*/
private $stream;
/**
* @var boolean
*/
private $closeStreamOnFinish = true;
/**
* @param resource $stream
*/
public function __construct($stream = null)
{
if (null !== $stream) {
$this->setStream($stream);
}
}
/**
* Set the stream resource
*
* @param resource $stream
*
* @throws \InvalidArgumentException
*
* @return $this
*/
public function setStream($stream)
{
if (! is_resource($stream) || ! 'stream' == get_resource_type($stream)) {
throw new \InvalidArgumentException(sprintf(
'Expects argument to be a stream resource, got %s',
is_resource($stream) ? get_resource_type($stream) : gettype($stream)
));
}
$this->stream = $stream;
return $this;
}
/**
* @return resource
*/
public function getStream()
{
if (null === $this->stream) {
$this->setStream(fopen('php://temp', 'rb+'));
$this->setCloseStreamOnFinish(false);
}
return $this->stream;
}
/**
* {@inheritdoc}
*/
public function finish()
{
if (is_resource($this->stream) && $this->getCloseStreamOnFinish()) {
fclose($this->stream);
}
}
/**
* Should underlying stream be closed on finish?
*
* @param boolean $closeStreamOnFinish
*
* @return boolean
*/
public function setCloseStreamOnFinish($closeStreamOnFinish = true)
{
$this->closeStreamOnFinish = (bool) $closeStreamOnFinish;
return $this;
}
/**
* Is Stream closed on finish?
*
* @return boolean
*/
public function getCloseStreamOnFinish()
{
return $this->closeStreamOnFinish;
}
}

View File

@@ -0,0 +1,44 @@
<?php
namespace Ddeboer\DataImport\Writer;
use Ddeboer\DataImport\Writer;
/**
* This class writes an item into an array that was passed by reference
*
* @author David de Boer <david@ddeboer.nl>
*/
class ArrayWriter implements Writer
{
use WriterTemplate;
/**
* @var array
*/
protected $data;
/**
* @param array $data
*/
public function __construct(array &$data)
{
$this->data = &$data;
}
/**
* {@inheritdoc}
*/
public function prepare()
{
$this->data = [];
}
/**
* {@inheritdoc}
*/
public function writeItem(array $item)
{
$this->data[] = $item;
}
}

View File

@@ -0,0 +1,58 @@
<?php
namespace Ddeboer\DataImport\Writer;
use Ddeboer\DataImport\Writer;
/**
* @author Markus Bachmann <markus.bachmann@bachi.biz>
*/
class BatchWriter implements Writer
{
private $delegate;
private $size;
private $queue;
public function __construct(Writer $delegate, $size = 20)
{
$this->delegate = $delegate;
$this->size = $size;
}
public function prepare()
{
$this->delegate->prepare();
$this->queue = new \SplQueue();
$this->queue->setIteratorMode(\SplDoublyLinkedList::IT_MODE_DELETE);
}
public function writeItem(array $item)
{
$this->queue->push($item);
if (count($this->queue) >= $this->size) {
$this->flush();
}
}
public function finish()
{
$this->flush();
$this->delegate->finish();
}
private function flush()
{
foreach ($this->queue as $item) {
$this->delegate->writeItem($item);
}
if ($this->delegate instanceof FlushableWriter) {
$this->delegate->flush();
}
}
}

View File

@@ -0,0 +1,36 @@
<?php
namespace Ddeboer\DataImport\Writer;
use Ddeboer\DataImport\Writer;
/**
* Writes using a callback or closure
*
* @author Markus Bachmann <markus.bachmann@bachi.biz>
*/
class CallbackWriter implements Writer
{
use WriterTemplate;
/**
* @var callable
*/
private $callback;
/**
* @param callable $callback
*/
public function __construct(callable $callback)
{
$this->callback = $callback;
}
/**
* {@inheritdoc}
*/
public function writeItem(array $item)
{
call_user_func($this->callback, $item);
}
}

View File

@@ -0,0 +1,102 @@
<?php
namespace Ddeboer\DataImport\Writer;
use Ddeboer\DataImport\Reader\CountableReader;
use Ddeboer\DataImport\Writer;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Helper\ProgressBar;
/**
* Writes output to the Symfony2 console
*
* @author David de Boer <david@ddeboer.nl>
*/
class ConsoleProgressWriter implements Writer
{
/**
* @var OutputInterface
*/
protected $output;
/**
* @var ProgressBar
*/
protected $progress;
/**
* @var string
*/
protected $verbosity;
/**
* @var CountableReader
*/
protected $reader;
/**
* @var integer
*/
protected $redrawFrequency;
/**
* @param OutputInterface $output
* @param CountableReader $reader
* @param string $verbosity
* @param integer $redrawFrequency
*/
public function __construct(
OutputInterface $output,
CountableReader $reader,
$verbosity = 'debug',
$redrawFrequency = 1
) {
$this->output = $output;
$this->reader = $reader;
$this->verbosity = $verbosity;
$this->redrawFrequency = $redrawFrequency;
}
/**
* {@inheritdoc}
*/
public function prepare()
{
$this->progress = new ProgressBar($this->output, $this->reader->count());
$this->progress->setFormat($this->verbosity);
$this->progress->setRedrawFrequency($this->redrawFrequency);
$this->progress->start();
}
/**
* {@inheritdoc}
*/
public function writeItem(array $item)
{
$this->progress->advance();
}
/**
* {@inheritdoc}
*/
public function finish()
{
$this->progress->finish();
}
/**
* @return string
*/
public function getVerbosity()
{
return $this->verbosity;
}
/**
* @return integer
*/
public function getRedrawFrequency()
{
return $this->redrawFrequency;
}
}

View File

@@ -0,0 +1,70 @@
<?php
namespace Ddeboer\DataImport\Writer;
use Ddeboer\DataImport\Writer;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Helper\Table;
/**
* @author Igor Mukhin <igor.mukhin@gmail.com>
*/
class ConsoleTableWriter implements Writer
{
use WriterTemplate;
/**
* @var OutputInterface
*/
private $output;
/**
* @var Table
*/
private $table;
/**
* @var array
*/
private $firstItem;
/**
* @param OutputInterface $output
* @param Table $table
*/
public function __construct(OutputInterface $output, Table $table) {
$this->output = $output;
$this->table = $table;
}
/**
* {@inheritdoc}
*/
public function writeItem(array $item) {
// Save first item to get keys to display at header
if (is_null($this->firstItem)) {
$this->firstItem = $item;
}
$this->table->addRow($item);
}
/**
* {@inheritdoc}
*/
public function finish() {
$this->table->setHeaders(array_keys($this->firstItem));
$this->table->render();
$this->firstItem = null;
}
/**
* @return Table
*/
public function getTable()
{
return $this->table;
}
}

View File

@@ -0,0 +1,72 @@
<?php
namespace Ddeboer\DataImport\Writer;
/**
* Writes to a CSV file
*
* @author David de Boer <david@ddeboer.nl>
*/
class CsvWriter extends AbstractStreamWriter
{
/**
* @var string
*/
private $delimiter = ';';
/**
* @var string
*/
private $enclosure = '"';
/**
* @var boolean
*/
private $utf8Encoding = false;
private $row = 1;
/**
* @var boolean
*/
protected $prependHeaderRow;
/**
* @param string $delimiter The delimiter
* @param string $enclosure The enclosure
* @param resource $stream
* @param boolean $utf8Encoding
*/
public function __construct($delimiter = ';', $enclosure = '"', $stream = null, $utf8Encoding = false, $prependHeaderRow = false)
{
parent::__construct($stream);
$this->delimiter = $delimiter;
$this->enclosure = $enclosure;
$this->utf8Encoding = $utf8Encoding;
$this->prependHeaderRow = $prependHeaderRow;
}
/**
* {@inheritdoc}
*/
public function prepare()
{
if ($this->utf8Encoding) {
fprintf($this->getStream(), chr(0xEF) . chr(0xBB) . chr(0xBF));
}
}
/**
* {@inheritdoc}
*/
public function writeItem(array $item)
{
if ($this->prependHeaderRow && 1 == $this->row++) {
$headers = array_keys($item);
fputcsv($this->getStream(), $headers, $this->delimiter, $this->enclosure);
}
fputcsv($this->getStream(), $item, $this->delimiter, $this->enclosure);
}
}

View File

@@ -0,0 +1,306 @@
<?php
namespace Ddeboer\DataImport\Writer;
use Ddeboer\DataImport\Writer;
use Doctrine\DBAL\Logging\SQLLogger;
use Doctrine\ORM\EntityManagerInterface;
use Doctrine\ORM\EntityRepository;
use Doctrine\ORM\Mapping\ClassMetadata;
/**
* A bulk Doctrine writer
*
* See also the {@link http://www.doctrine-project.org/docs/orm/2.1/en/reference/batch-processing.html Doctrine documentation}
* on batch processing.
*
* @author David de Boer <david@ddeboer.nl>
*/
class DoctrineWriter implements Writer, FlushableWriter
{
/**
* @var EntityManagerInterface
*/
protected $entityManager;
/**
* @var string
*/
protected $entityName;
/**
* @var EntityRepository
*/
protected $entityRepository;
/**
* @var ClassMetadata
*/
protected $entityMetadata;
/**
* Original Doctrine logger
*
* @var SQLLogger
*/
protected $originalLogger;
/**
* Whether to truncate the table first
*
* @var boolean
*/
protected $truncate = true;
/**
* List of fields used to lookup an entity
*
* @var array
*/
protected $lookupFields = array();
/**
* @param EntityManagerInterface $entityManager
* @param string $entityName
* @param string|array $index Field or fields to find current entities by
*/
public function __construct(EntityManagerInterface $entityManager, $entityName, $index = null)
{
$this->entityManager = $entityManager;
$this->entityRepository = $entityManager->getRepository($entityName);
$this->entityMetadata = $entityManager->getClassMetadata($entityName);
//translate entityName in case a namespace alias is used
$this->entityName = $this->entityMetadata->getName();
if ($index) {
if (is_array($index)) {
$this->lookupFields = $index;
} else {
$this->lookupFields = [$index];
}
}
}
/**
* @return boolean
*/
public function getTruncate()
{
return $this->truncate;
}
/**
* Set whether to truncate the table first
*
* @param boolean $truncate
*
* @return $this
*/
public function setTruncate($truncate)
{
$this->truncate = $truncate;
return $this;
}
/**
* Disable truncation
*
* @return $this
*/
public function disableTruncate()
{
$this->truncate = false;
return $this;
}
/**
* Disable Doctrine logging
*
* @return $this
*/
public function prepare()
{
$this->disableLogging();
if (true === $this->truncate) {
$this->truncateTable();
}
}
/**
* Return a new instance of the entity
*
* @return object
*/
protected function getNewInstance()
{
$className = $this->entityMetadata->getName();
if (class_exists($className) === false) {
throw new \Exception('Unable to create new instance of ' . $className);
}
return new $className;
}
/**
* Call a setter of the entity
*
* @param object $entity
* @param mixed $value
* @param string $setter
*/
protected function setValue($entity, $value, $setter)
{
if (method_exists($entity, $setter)) {
$entity->$setter($value);
}
}
/**
* Re-enable Doctrine logging
*
* @return $this
*/
public function finish()
{
$this->flush();
$this->reEnableLogging();
}
/**
* {@inheritdoc}
*/
public function writeItem(array $item)
{
$entity = $this->findOrCreateItem($item);
$this->loadAssociationObjectsToEntity($item, $entity);
$this->updateEntity($item, $entity);
$this->entityManager->persist($entity);
}
/**
* @param array $item
* @param object $entity
*/
protected function updateEntity(array $item, $entity)
{
$fieldNames = array_merge($this->entityMetadata->getFieldNames(), $this->entityMetadata->getAssociationNames());
foreach ($fieldNames as $fieldName) {
$value = null;
if (isset($item[$fieldName])) {
$value = $item[$fieldName];
} elseif (method_exists($item, 'get' . ucfirst($fieldName))) {
$value = $item->{'get' . ucfirst($fieldName)};
}
if (null === $value) {
continue;
}
if (!($value instanceof \DateTime)
|| $value != $this->entityMetadata->getFieldValue($entity, $fieldName)
) {
$setter = 'set' . ucfirst($fieldName);
$this->setValue($entity, $value, $setter);
}
}
}
/**
* Add the associated objects in case the item have for persist its relation
*
* @param array $item
* @param object $entity
*/
protected function loadAssociationObjectsToEntity(array $item, $entity)
{
foreach ($this->entityMetadata->getAssociationMappings() as $associationMapping) {
$value = null;
if (isset($item[$associationMapping['fieldName']]) && !is_object($item[$associationMapping['fieldName']])) {
$value = $this->entityManager->getReference($associationMapping['targetEntity'], $item[$associationMapping['fieldName']]);
}
if (null === $value) {
continue;
}
$setter = 'set' . ucfirst($associationMapping['fieldName']);
$this->setValue($entity, $value, $setter);
}
}
/**
* Truncate the database table for this writer
*/
protected function truncateTable()
{
$tableName = $this->entityMetadata->table['name'];
$connection = $this->entityManager->getConnection();
$query = $connection->getDatabasePlatform()->getTruncateTableSQL($tableName, true);
$connection->executeQuery($query);
}
/**
* Disable Doctrine logging
*/
protected function disableLogging()
{
$config = $this->entityManager->getConnection()->getConfiguration();
$this->originalLogger = $config->getSQLLogger();
$config->setSQLLogger(null);
}
/**
* Re-enable Doctrine logging
*/
protected function reEnableLogging()
{
$config = $this->entityManager->getConnection()->getConfiguration();
$config->setSQLLogger($this->originalLogger);
}
/**
* Finds existing entity or create a new instance
*
* @param array $item
*/
protected function findOrCreateItem(array $item)
{
$entity = null;
// If the table was not truncated to begin with, find current entity
// first
if (false === $this->truncate) {
if (!empty($this->lookupFields)) {
$lookupConditions = array();
foreach ($this->lookupFields as $fieldName) {
$lookupConditions[$fieldName] = $item[$fieldName];
}
$entity = $this->entityRepository->findOneBy(
$lookupConditions
);
} else {
$entity = $this->entityRepository->find(current($item));
}
}
if (!$entity) {
return $this->getNewInstance();
}
return $entity;
}
/**
* Flush and clear the entity manager
*/
public function flush()
{
$this->entityManager->flush();
$this->entityManager->clear($this->entityName);
}
}

View File

@@ -0,0 +1,117 @@
<?php
namespace Ddeboer\DataImport\Writer;
use Ddeboer\DataImport\Writer;
use PHPExcel;
use PHPExcel_IOFactory;
/**
* Writes to an Excel file
*
* @author David de Boer <david@ddeboer.nl>
*/
class ExcelWriter implements Writer
{
/**
* @var string
*/
protected $filename;
/**
* @var null|string
*/
protected $sheet;
/**
* @var string
*/
protected $type;
/**
* @var boolean
*/
protected $prependHeaderRow;
/**
* @var PHPExcel
*/
protected $excel;
/**
* @var integer
*/
protected $row = 1;
/**
* @param \SplFileObject $file File
* @param string $sheet Sheet title (optional)
* @param string $type Excel file type (defaults to Excel2007)
* @param boolean $prependHeaderRow
*/
public function __construct(\SplFileObject $file, $sheet = null, $type = 'Excel2007', $prependHeaderRow = false)
{
$this->filename = $file->getPathname();
$this->sheet = $sheet;
$this->type = $type;
$this->prependHeaderRow = $prependHeaderRow;
}
/**
* {@inheritdoc}
*/
public function prepare()
{
$reader = PHPExcel_IOFactory::createReader($this->type);
if ($reader->canRead($this->filename)) {
$this->excel = $reader->load($this->filename);
} else {
$this->excel = new PHPExcel();
if(null !== $this->sheet && !$this->excel->sheetNameExists($this->sheet))
{
$this->excel->removeSheetByIndex(0);
}
}
if (null !== $this->sheet) {
if (!$this->excel->sheetNameExists($this->sheet)) {
$this->excel->createSheet()->setTitle($this->sheet);
}
$this->excel->setActiveSheetIndexByName($this->sheet);
}
}
/**
* {@inheritdoc}
*/
public function writeItem(array $item)
{
$count = count($item);
if ($this->prependHeaderRow && 1 == $this->row) {
$headers = array_keys($item);
for ($i = 0; $i < $count; $i++) {
$this->excel->getActiveSheet()->setCellValueByColumnAndRow($i, $this->row, $headers[$i]);
}
$this->row++;
}
$values = array_values($item);
for ($i = 0; $i < $count; $i++) {
$this->excel->getActiveSheet()->setCellValueByColumnAndRow($i, $this->row, $values[$i]);
}
$this->row++;
}
/**
* {@inheritdoc}
*/
public function finish()
{
$writer = \PHPExcel_IOFactory::createWriter($this->excel, $this->type);
$writer->save($this->filename);
}
}

View File

@@ -0,0 +1,8 @@
<?php
namespace Ddeboer\DataImport\Writer;
interface FlushableWriter
{
public function flush();
}

View File

@@ -0,0 +1,106 @@
<?php
namespace Ddeboer\DataImport\Writer;
use Ddeboer\DataImport\Exception\WriterException;
use Ddeboer\DataImport\Writer;
/**
* Write data into a specific database table using a PDO instance.
*
* IMPORTANT: If your PDO instance does not have ERRMODE_EXCEPTION any write failure will be silent or logged to
* stderr only. It is strongly recomended you enable Exceptions with:
*
* $pdo->setAttribute(\PDO::ATTR_ERRMODE, \PDO::ERRMODE_EXCEPTION);
*
* @author Stefan Warman
*/
class PdoWriter implements Writer, FlushableWriter
{
/**
* @var \PDO
*/
protected $pdo;
/**
* @var string
*/
protected $tableName;
/**
* @var \PDOStatement
*/
protected $statement;
/**
* @var array
*/
private $stack;
/**
* Note if your table name is a reserved word for your target DB you should quote it in the appropriate way e.g.
* for MySQL enclose the name in `backticks`.
*
* @param \PDO $pdo
* @param string $tableName
*/
public function __construct(\PDO $pdo, $tableName)
{
$this->pdo = $pdo;
$this->tableName = $tableName;
if (\PDO::ERRMODE_EXCEPTION !== $this->pdo->getAttribute(\PDO::ATTR_ERRMODE)) {
throw new WriterException('Please set the pdo error mode to PDO::ERRMODE_EXCEPTION');
}
}
public function prepare()
{
$this->stack = [];
$this->statement = null;
}
/**
* {@inheritdoc}
*/
public function writeItem(array $item)
{
if (null === $this->statement) {
try {
$this->statement = $this->pdo->prepare(sprintf(
'INSERT INTO %s (%s) VALUES (%s)',
$this->tableName,
implode(',', array_keys($item)),
substr(str_repeat('?,', count($item)), 0, -1)
));
} catch (\PDOException $e) {
throw new WriterException('Failed to send query', null, $e);
}
}
$this->stack[] = array_values($item);
}
public function finish()
{
$this->flush();
return $this;
}
public function flush()
{
$this->pdo->beginTransaction();
try {
foreach ($this->stack as $data) {
$this->statement->execute($data);
}
$this->stack = [];
$this->pdo->commit();
} catch (\PDOException $e) {
throw new WriterException('Failed to write to database', null, $e);
}
}
}

View File

@@ -0,0 +1,140 @@
<?php
namespace Ddeboer\DataImport\Writer;
/**
* Class allowing multiple writers to write in same stream
*
* @author Benoît Burnichon <bburnichon@gmail.com>
*/
class StreamMergeWriter extends AbstractStreamWriter
{
/**
* @var string
*/
private $discriminantField = 'discr';
/**
* @var AbstractStreamWriter[]
*/
private $writers = [];
/**
* Set discriminant field
*
* @param string $discriminantField
*
* @return $this
*/
public function setDiscriminantField($discriminantField)
{
$this->discriminantField = (string) $discriminantField;
return $this;
}
/**
* Get discriminant Field
*
* @return string
*/
public function getDiscriminantField()
{
return $this->discriminantField;
}
/**
* {@inheritdoc}
*/
public function writeItem(array $item)
{
if ((isset($item[$this->discriminantField])
|| array_key_exists($this->discriminantField, $item))
&& $this->hasStreamWriter($key = $item[$this->discriminantField])
) {
$writer = $this->getStreamWriter($key);
$writer->writeItem($item);
}
}
/**
* Set stream writers
*
* @param AbstractStreamWriter[] $writers
*
* @return $this
*/
public function setStreamWriters(array $writers)
{
foreach ($writers as $key => $writer) {
$this->setStreamWriter($key, $writer);
}
return $this;
}
/**
* @param string $key
* @param AbstractStreamWriter $writer
*
* @return $this
*/
public function setStreamWriter($key, AbstractStreamWriter $writer)
{
$writer->setStream($this->getStream());
$writer->setCloseStreamOnFinish(false);
$this->writers[$key] = $writer;
return $this;
}
/**
* Get a previously registered Writer
*
* @param string $key
*
* @return AbstractStreamWriter
*/
public function getStreamWriter($key)
{
return $this->writers[$key];
}
/**
* Get list of registered Writers
*
* @return AbstractStreamWriter[]
*/
public function getStreamWriters()
{
return $this->writers;
}
/**
* Is a writer registered for key?
*
* @param string $key
*
* @return boolean
*/
public function hasStreamWriter($key)
{
return isset($this->writers[$key]);
}
/**
* Set a stream
*
* @param resource $stream
*/
public function setStream($stream)
{
parent::setStream($stream);
foreach ($this->getStreamWriters() as $writer) {
$writer->setStream($stream);
}
return $this;
}
}

View File

@@ -0,0 +1,27 @@
<?php
namespace Ddeboer\DataImport\Writer;
/**
* This template can be overridden in concrete implementations
*
* @author David de Boer <david@ddeboer.nl>
*/
trait WriterTemplate
{
/**
* {@inheritdoc}
*/
public function prepare()
{
}
/**
* {@inheritdoc}
*/
public function finish()
{
}
}

View File

@@ -0,0 +1,63 @@
<?php
namespace Ddeboer\DataImport\Filter;
use Ddeboer\DataImport\Filter\DateTimeThresholdFilter;
use Ddeboer\DataImport\ValueConverter\DateTimeValueConverter;
class DateTimeFilterTest extends \PHPUnit_Framework_TestCase
{
public function setUp()
{
$this->items = array(
'a' => array('updated_at' => '-3 day'),
'b' => array('updated_at' => '-2 day'),
'c' => array('updated_at' => '-1 day'),
'd' => array('updated_at' => 'today'),
'e' => array('updated_at' => 'now'),
'f' => array('updated_at' => '+1 day'),
);
}
private function applyFilter(DateTimeThresholdFilter $filter, array $items)
{
return array_filter($items, array($filter, '__invoke'));
}
/**
* @expectedException \LogicException
* @expectedExceptionMessage Make sure you set a threshold
*/
public function testDefaultFilter()
{
$this->applyFilter(
new DateTimeThresholdFilter(new DateTimeValueConverter()),
$this->items
);
}
public function testFilter()
{
$resultItems = $this->applyFilter(new DateTimeThresholdFilter(
new DateTimeValueConverter(),
new \DateTime('today')
), $this->items);
$this->assertEquals(
array('d', 'e', 'f'),
array_keys($resultItems)
);
}
public function testSetter()
{
$filter = new DateTimeThresholdFilter(new DateTimeValueConverter());
$filter->setThreshold(new \DateTime('today'));
$resultItems = $this->applyFilter($filter, $this->items);
$this->assertEquals(
array('d', 'e', 'f'),
array_keys($resultItems)
);
}
}

View File

@@ -0,0 +1,51 @@
<?php
namespace Ddeboer\DataImport\Filter;
use Ddeboer\DataImport\Filter\OffsetFilter;
/**
* @author Ville Mattila <ville@eventio.fi>
*/
class OffsetFilterTest extends \PHPUnit_Framework_TestCase
{
private function applyFilter(OffsetFilter $filter, array $items) {
$result = array();
foreach ($items as $item) {
if (true === call_user_func($filter, array($item))) {
$result[] = $item;
}
}
return $result;
}
public function testDefaultFilter()
{
$items = array('first','second','third','fourth');
$resultItems = $this->applyFilter(new OffsetFilter(), $items);
$this->assertEquals($resultItems, $items);
}
public function testStartOffset()
{
$items = array('first','second','third','fourth');
$resultItems = $this->applyFilter(new OffsetFilter(1), $items);
$this->assertEquals($resultItems, array('second','third','fourth'));
}
public function testMaxCount()
{
$items = array('first','second','third','fourth');
$resultItems = $this->applyFilter(new OffsetFilter(0, 2), $items);
$this->assertEquals($resultItems, array('first','second'));
}
public function testOffsetWithMaxCount()
{
$items = array('first','second','third','fourth');
$resultItems = $this->applyFilter(new OffsetFilter(1, 1), $items);
$this->assertEquals($resultItems, array('second'));
}
}

View File

@@ -0,0 +1,108 @@
<?php
namespace Ddeboer\DataImport\Tests\Filter;
use Ddeboer\DataImport\Exception\ValidationException;
use Ddeboer\DataImport\Filter\ValidatorFilter;
use Symfony\Component\Validator\Constraints;
use Symfony\Component\Validator\ConstraintViolationList;
class ValidationFilterTest extends \PHPUnit_Framework_TestCase
{
protected function setUp()
{
$this->validator = $this->getMock('Symfony\\Component\\Validator\\Validator\\ValidatorInterface');
$this->filter = new ValidatorFilter($this->validator);
}
public function testFilterWithValid()
{
$item = array('foo' => 'bar');
$list = new ConstraintViolationList();
$this->validator->expects($this->once())
->method('validate')
->willReturn($list);
$this->assertTrue(call_user_func($this->filter, $item));
}
public function testFilterWithInvalidItem()
{
$item = array('foo' => 'bar');
$violation = $this->getMock('Symfony\\Component\\Validator\\ConstraintViolationInterface');
$list = new ConstraintViolationList(array($violation));
$this->validator->expects($this->once())
->method('validate')
->willReturn($list);
$this->assertFalse(call_user_func($this->filter, $item));
$this->assertEquals(array(1 => $list), $this->filter->getViolations());
}
public function testStopOnFirstError()
{
$this->filter->throwExceptions();
$item = array('foo' => 'bar');
$violation = $this->getMock('Symfony\\Component\\Validator\\ConstraintViolationInterface');
$list = new ConstraintViolationList(array($violation));
$this->validator->expects($this->once())
->method('validate')
->willReturn($list);
try {
call_user_func($this->filter, $item);
$this->fail('ValidationException should be thrown');
} catch (ValidationException $e) {
$this->assertSame(1, $e->getLineNumber());
$this->assertEquals($list, $e->getViolations());
}
}
public function testFilterNonStrict()
{
$this->filter->setStrict(false);
$item = array('foo' => true, 'bar' => true);
$this->filter->add('foo', new Constraints\IsTrue());
$this->assertTrue(call_user_func($this->filter, $item));
}
public function testFilterLineNumbers()
{
$this->filter->throwExceptions();
$item = array('foo' => 'bar');
$violation = $this->getMock('Symfony\\Component\\Validator\\ConstraintViolationInterface');
$list = new ConstraintViolationList(array($violation));
$this->validator->expects($this->exactly(2))
->method('validate')
->willReturn($list);
try {
$this->assertTrue(call_user_func($this->filter, $item));
$this->fail('ValidationException should be thrown (1)');
} catch (ValidationException $e) {
$this->assertSame(1, $e->getLineNumber());
$this->assertEquals($list, $e->getViolations());
}
try {
$this->assertTrue(call_user_func($this->filter, $item));
$this->fail('ValidationException should be thrown (2)');
} catch (ValidationException $e) {
$this->assertSame(2, $e->getLineNumber());
$this->assertEquals($list, $e->getViolations());
}
}
}

View File

@@ -0,0 +1,43 @@
<?php
namespace Ddeboer\DataImport\Tests\Fixtures\Entity;
class TestEntity
{
private $firstProperty;
private $secondProperty;
private $firstAssociation;
public function getFirstProperty()
{
return $this->firstProperty;
}
public function setFirstProperty($firstProperty)
{
$this->firstProperty = $firstProperty;
}
public function getSecondProperty()
{
return $this->secondProperty;
}
public function setSecondProperty($secondProperty)
{
$this->secondProperty = $secondProperty;
}
public function getFirstAssociation()
{
return $this->firstAssociation;
}
public function setFirstAssociation($firstAssociation)
{
$this->firstAssociation = $firstAssociation;
}
}

View File

@@ -0,0 +1,30 @@
<?php
namespace Ddeboer\DataImport\Tests\Fixtures\Entity;
/**
* @Entity()
*/
class User
{
/** @Id @GeneratedValue @Column(type="integer") **/
private $id;
/** @Column(type="string") */
private $username;
public function getId()
{
return $this->id;
}
public function getUsername()
{
return $this->username;
}
public function setUsername($username)
{
$this->username = $username;
}
}

View File

@@ -0,0 +1 @@
50,123,"Description"
Can't render this file because it contains an unexpected character in line 1 and column 20.

View File

@@ -0,0 +1,4 @@
id,number,description
50,123,"Description"
6,456,"Another description"
7,7890,"Some more info"
1 id number description
2 50 123 Description
3 6 456 Another description
4 7 7890 Some more info

Binary file not shown.

View File

@@ -0,0 +1,2 @@
id,description,description,description,details,details,last
50,"First","Second","Third","Details1","Details2","Last one"
1 id description description description details details last
2 50 First Second Third Details1 Details2 Last one

View File

@@ -0,0 +1,5 @@
id,number,description
50,123,"Description"
123,test
7,7890,Some more info,"too many columns"
123,66,Valid
1 id number description
2 50 123 Description
3 123 test
4 7 7890 Some more info too many columns
5 123 66 Valid

View File

@@ -0,0 +1 @@
50,123,"Description"
Can't render this file because it contains an unexpected character in line 1 and column 20.

View File

@@ -0,0 +1,4 @@
id,number,description
50,123,"Description"
6,456
7,7890,"Some more info"
1 id number description
2 50 123 Description
3 6 456
4 7 7890 Some more info

View File

@@ -0,0 +1,4 @@
id,number,description
50,123,"Description"
6,456,"Another description","Some more info"
7,7890,"Yet another description"
1 id number description
2 50 123 Description
3 6 456 Another description Some more info
4 7 7890 Yet another description

Binary file not shown.

View File

@@ -0,0 +1,3 @@
50,123,"Description"
6,456,"Another description"
7,7890,"Some more info"
1 50 123 Description
2 6 456 Another description
3 7 7890 Some more info

Some files were not shown because too many files have changed in this diff Show More